From fe7348f052b93561f6698758d23a2a944f9b0459 Mon Sep 17 00:00:00 2001 From: Emillio Mariscal Date: Tue, 27 Feb 2024 14:37:24 -0300 Subject: [PATCH 1/3] Refactor, working on relations --- setup/bootstrap.sh | 2 +- setup/db/underpass.sql | 17 +++++++ src/bootstrap/bootstrap.cc | 68 ++++++++++++++------------ src/bootstrap/bootstrap.hh | 29 +++++++++--- src/raw/queryraw.cc | 18 ++++--- src/raw/queryraw.hh | 5 +- src/replicator/threads.cc | 46 +++++++++++------- src/underpass.cc | 5 +- src/underpassconfig.hh | 4 ++ src/validate/queryvalidate.cc | 89 +++++++++++++++++++++++++++++++++++ src/validate/queryvalidate.hh | 7 +++ 11 files changed, 224 insertions(+), 66 deletions(-) diff --git a/setup/bootstrap.sh b/setup/bootstrap.sh index 4c4a6b0e..db0cd30d 100644 --- a/setup/bootstrap.sh +++ b/setup/bootstrap.sh @@ -107,7 +107,7 @@ then PGPASSWORD=$PASS psql --host $HOST --user $USER --port $PORT $DB < db/indexes.sql echo "Configuring Underpass ..." - python3 poly2geojson.py $COUNTRY.poly + python3 ../utils/poly2geojson.py $COUNTRY.poly if "$use_docker"; then docker cp $COUNTRY.geojson underpass:/etc/underpass/priority.geojson diff --git a/setup/db/underpass.sql b/setup/db/underpass.sql index bae39f2f..5e8e2781 100644 --- a/setup/db/underpass.sql +++ b/setup/db/underpass.sql @@ -94,11 +94,28 @@ CREATE TABLE IF NOT EXISTS public.nodes ( uid int8 ); +CREATE TABLE IF NOT EXISTS public.rels ( + osm_id int8, + changeset int8, + geom public.geometry(Geometry,4326), + tags JSONB, + members int8[], + timestamp timestamp with time zone, + version int, + "user" text, + uid int8 +); + CREATE TABLE IF NOT EXISTS public.way_refs ( way_id int8, node_id int8 ); +CREATE TABLE IF NOT EXISTS public.rel_members ( + rel_id int8, + way_id int8 +); + CREATE UNIQUE INDEX nodes_id_idx ON public.nodes (osm_id DESC); CREATE UNIQUE INDEX ways_poly_id_idx ON public.ways_poly (osm_id DESC); CREATE UNIQUE INDEX ways_line_id_idx ON public.ways_line(osm_id DESC); diff --git a/src/bootstrap/bootstrap.cc b/src/bootstrap/bootstrap.cc index 13b4c6b1..474d2483 100644 --- a/src/bootstrap/bootstrap.cc +++ b/src/bootstrap/bootstrap.cc @@ -45,10 +45,10 @@ using namespace logger; namespace bootstrap { -const int PAGE_SIZE = 100; +Bootstrap::Bootstrap(void) {} std::string -allTasksQueries(std::shared_ptr> tasks) { +Bootstrap::allTasksQueries(std::shared_ptr> tasks) { std::string queries = ""; for (auto it = tasks->begin(); it != tasks->end(); ++it) { queries += it->query ; @@ -56,10 +56,10 @@ allTasksQueries(std::shared_ptr> tasks) { return queries; } -void startProcessingWays(const underpassconfig::UnderpassConfig &config) { - +void +Bootstrap::start(const underpassconfig::UnderpassConfig &config) { std::cout << "Connecting to the database ..." << std::endl; - auto db = std::make_shared(); + db = std::make_shared(); if (!db->connect(config.underpass_db_url)) { std::cout << "Could not connect to Underpass DB, aborting bootstrapping thread!" << std::endl; return; @@ -81,10 +81,23 @@ void startProcessingWays(const underpassconfig::UnderpassConfig &config) { log_debug("Couldn't load plugin! %1%", e.what()); exit(0); } - auto validator = creator(); - auto queryvalidate = std::make_shared(db); - auto queryraw = std::make_shared(db); + validator = creator(); + queryvalidate = std::make_shared(db); + queryraw = std::make_shared(db); + page_size = config.bootstrap_page_size; + concurrency = config.concurrency; + norefs = config.norefs; + + processWays(); + // processNodes(); + // processRels(); + +} + +void +Bootstrap::processWays() { + std::vector tables = { QueryRaw::polyTable, QueryRaw::lineTable @@ -94,27 +107,25 @@ void startProcessingWays(const underpassconfig::UnderpassConfig &config) { std::cout << std::endl << "Counting geometries ... " << std::endl; long int total = queryraw->getWaysCount(*table_it); long int count = 0; - int num_chunks = total / PAGE_SIZE; + int num_chunks = total / page_size; std::cout << "Total: " << total << std::endl; - std::cout << "Threads: " << config.concurrency << std::endl; - std::cout << "Page size: " << PAGE_SIZE << std::endl; + std::cout << "Threads: " << concurrency << std::endl; + std::cout << "Page size: " << page_size << std::endl; long lastid = 0; - int concurrentTasks = config.concurrency; + int concurrentTasks = concurrency; int taskIndex = 0; - std::chrono::steady_clock::time_point begin; - std::chrono::steady_clock::time_point end; for (int chunkIndex = 1; chunkIndex <= (num_chunks/concurrentTasks); chunkIndex++) { int percentage = (count * 100) / total; auto ways = std::make_shared>(); - if (!config.norefs) { - ways = queryraw->getWaysFromDB(lastid, config.concurrency * PAGE_SIZE, *table_it); + if (!norefs) { + ways = queryraw->getWaysFromDB(lastid, concurrency * page_size, *table_it); } else { - ways = queryraw->getWaysFromDBWithoutRefs(lastid, config.concurrency * PAGE_SIZE, *table_it); + ways = queryraw->getWaysFromDBWithoutRefs(lastid, concurrency * page_size, *table_it); } auto tasks = std::make_shared>(concurrentTasks); @@ -122,15 +133,13 @@ void startProcessingWays(const underpassconfig::UnderpassConfig &config) { for (int taskIndex = 0; taskIndex < concurrentTasks; taskIndex++) { auto taskWays = std::make_shared>(); WayTask wayTask { - std::ref(validator), - std::ref(queryvalidate), - config, taskIndex, std::ref(tasks), std::ref(ways), }; std::cout << "\r" << "Processing " << *table_it << ": " << count << "/" << total << " (" << percentage << "%)"; - boost::asio::post(pool, boost::bind(threadBootstrapTask, wayTask)); + + boost::asio::post(pool, boost::bind(&Bootstrap::threadBootstrapTask, this, wayTask)); } pool.join(); @@ -148,14 +157,11 @@ void startProcessingWays(const underpassconfig::UnderpassConfig &config) { // This thread get started for every page of way void -threadBootstrapTask(WayTask wayTask) +Bootstrap::threadBootstrapTask(WayTask wayTask) { #ifdef TIMING_DEBUG boost::timer::auto_cpu_timer timer("bootstrap::threadBootstrapTask(wayTask): took %w seconds\n"); #endif - auto plugin = wayTask.plugin; - auto queryvalidate = wayTask.queryvalidate; - auto config = wayTask.config; auto taskIndex = wayTask.taskIndex; auto tasks = wayTask.tasks; auto ways = wayTask.ways; @@ -163,16 +169,15 @@ threadBootstrapTask(WayTask wayTask) BootstrapTask task; int processed = 0; + auto wayval = std::make_shared>>(); + // Proccesing ways - for (int i = 0; i < PAGE_SIZE; ++i) { + for (int i = 0; i < page_size; ++i) { if (i * taskIndex < ways->size()) { auto way = ways->at(i * (taskIndex + 1)); - auto status = plugin->checkWay(way, "building"); - for (auto status_it = status->status.begin(); status_it != status->status.end(); ++status_it) { - task.query += queryvalidate->applyChange(*status, *status_it); - } + wayval->push_back(validator->checkWay(way, "building")); // Fill the way_refs table - if (!config.norefs) { + if (!norefs) { for (auto ref = way.refs.begin(); ref != way.refs.end(); ++ref) { task.query += "INSERT INTO way_refs (way_id, node_id) VALUES (" + std::to_string(way.id) + "," + std::to_string(*ref) + "); "; } @@ -180,6 +185,7 @@ threadBootstrapTask(WayTask wayTask) ++processed; } } + queryvalidate->ways(wayval, task.query); task.processed = processed; const std::lock_guard lock(tasks_change_mutex); (*tasks)[taskIndex] = task; diff --git a/src/bootstrap/bootstrap.hh b/src/bootstrap/bootstrap.hh index 8210ad99..3758ea7c 100644 --- a/src/bootstrap/bootstrap.hh +++ b/src/bootstrap/bootstrap.hh @@ -21,6 +21,7 @@ #include "raw/queryraw.hh" #include "underpassconfig.hh" #include "validate/validate.hh" +#include using namespace queryvalidate; using namespace queryraw; @@ -37,18 +38,34 @@ struct BootstrapTask { }; struct WayTask { - std::shared_ptr plugin; - std::shared_ptr queryvalidate; - underpassconfig::UnderpassConfig config; int taskIndex; std::shared_ptr> tasks; std::shared_ptr> ways; }; -void startProcessingWays(const underpassconfig::UnderpassConfig &config); +class Bootstrap { + public: + Bootstrap(void); + ~Bootstrap(void){}; + + static const std::string polyTable; + static const std::string lineTable; + + void start(const underpassconfig::UnderpassConfig &config); + void processWays(); -// This thread get started for every page of way -void threadBootstrapTask(WayTask wayTask); + // This thread get started for every page of way + void threadBootstrapTask(WayTask wayTask); + std::string allTasksQueries(std::shared_ptr> tasks); + + std::shared_ptr validator; + std::shared_ptr queryvalidate; + std::shared_ptr queryraw; + std::shared_ptr db; + bool norefs; + unsigned int concurrency; + unsigned int page_size; +}; static std::mutex tasks_change_mutex; diff --git a/src/raw/queryraw.cc b/src/raw/queryraw.cc index c75738ab..d1d03fd0 100644 --- a/src/raw/queryraw.cc +++ b/src/raw/queryraw.cc @@ -24,8 +24,6 @@ /// includes querying existing data in the database, as well as /// updating the database. -// TODO: add support for relations/multipolygon - // This is generated by autoconf #ifdef HAVE_CONFIG_H #include "unconfig.h" @@ -233,6 +231,12 @@ QueryRaw::applyChange(const OsmWay &way) const return query; } +std::string +QueryRaw::applyChange(const OsmRelation &relation) const +{ + +} + std::vector arrayStrToVector(std::string &refs_str) { refs_str.erase(0, 1); refs_str.erase(refs_str.size() - 1); @@ -261,10 +265,8 @@ void QueryRaw::getNodeCache(std::shared_ptr osmchanges, const mul if (way->action != osmobjects::remove) { // Save referenced nodes for later use for (auto rit = std::begin(way->refs); rit != std::end(way->refs); ++rit) { - if (way->action != osmobjects::remove) { - if (!osmchanges->nodecache.count(*rit)) { - referencedNodeIds += std::to_string(*rit) + ","; - } + if (!osmchanges->nodecache.count(*rit)) { + referencedNodeIds += std::to_string(*rit) + ","; } } } else { @@ -337,6 +339,10 @@ void QueryRaw::getNodeCache(std::shared_ptr osmchanges, const mul } } + // Build relation multipolyon geometries + // TODO + + } void diff --git a/src/raw/queryraw.hh b/src/raw/queryraw.hh index 20c99eda..4396bb71 100644 --- a/src/raw/queryraw.hh +++ b/src/raw/queryraw.hh @@ -64,14 +64,15 @@ class QueryRaw { std::string applyChange(const OsmNode &node) const; /// Build query for processed Way std::string applyChange(const OsmWay &way) const; - /// Build query for updating modified nodes in ways geometries - // std::string applyChange(const std::shared_ptr>> nodes) const; + /// Build query for processed Relation + std::string applyChange(const OsmRelation &relation) const; /// Get nodes for filling Node cache void getNodeCache(std::shared_ptr osmchanges, const multipolygon_t &poly); /// Get nodes for filling Node cache from ways refs void getNodeCacheFromWays(std::shared_ptr> ways, std::map &nodecache) const; // Get ways by refs std::list> getWaysByNodesRefs(std::string &nodeIds) const; + // DB connection std::shared_ptr dbconn; // Get ways count diff --git a/src/replicator/threads.cc b/src/replicator/threads.cc index 670d2565..9187ab98 100644 --- a/src/replicator/threads.cc +++ b/src/replicator/threads.cc @@ -479,6 +479,7 @@ threadOsmChange(OsmChangeTask osmChangeTask) auto removed_nodes = std::make_shared>(); auto removed_ways = std::make_shared>(); + auto removed_relations = std::make_shared>(); auto validation_removals = std::make_shared>(); // Raw data and validation @@ -524,6 +525,25 @@ threadOsmChange(OsmChangeTask osmChangeTask) } } + // Relations + // for (auto rit = std::begin(change->relations); rit != std::end(change->relations); ++rit) { + // osmobjects::OsmRelation *relation = rit->get(); + + // if (relation->action != osmobjects::remove && !relation->priority) { + // continue; + // } + + // // Remove deleted relations from validation table + // if (!config->disable_validation && relation->action == osmobjects::remove) { + // removed_relations->push_back(relation->id); + // } + + // // Update relations, ignore new ones outside priority area + // if (!config->disable_raw) { + // task.query += queryraw->applyChange(*relation); + // } + // } + } } @@ -532,30 +552,20 @@ threadOsmChange(OsmChangeTask osmChangeTask) // Validate ways auto wayval = osmchanges->validateWays(poly, plugin); - for (auto it = wayval->begin(); it != wayval->end(); ++it) { + queryvalidate->ways(wayval, task.query, validation_removals); - if (it->get()->status.size() > 0) { - for (auto status_it = it->get()->status.begin(); status_it != it->get()->status.end(); ++status_it) { - task.query += queryvalidate->applyChange(*it->get(), *status_it); - } - if (!it->get()->hasStatus(overlapping)) { - task.query += queryvalidate->updateValidation(it->get()->osm_id, overlapping, "building"); - } - if (!it->get()->hasStatus(duplicate)) { - task.query += queryvalidate->updateValidation(it->get()->osm_id, duplicate, "building"); - } - if (!it->get()->hasStatus(badgeom)) { - task.query += queryvalidate->updateValidation(it->get()->osm_id, badgeom, "building"); - } - } else { - validation_removals->push_back(it->get()->osm_id); - } - } + // Validate nodes + auto nodeval = osmchanges->validateNodes(poly, plugin); + queryvalidate->nodes(nodeval, task.query, validation_removals); + + // Validate relations + // task.query += queryvalidate->rels(wayval, task.query, validation_removals); // Remove validation entries for removed objects task.query += queryvalidate->updateValidation(validation_removals); task.query += queryvalidate->updateValidation(removed_nodes); task.query += queryvalidate->updateValidation(removed_ways); + // task.query += queryvalidate->updateValidation(removed_relations); } diff --git a/src/underpass.cc b/src/underpass.cc index b16bec4c..723db754 100644 --- a/src/underpass.cc +++ b/src/underpass.cc @@ -329,8 +329,9 @@ main(int argc, char *argv[]) if (vm.count("bootstrap")){ std::thread bootstrapThread; - std::cout << "Starting bootstrapping proccess ..." << std::endl; - bootstrapThread = std::thread(bootstrap::startProcessingWays, std::ref(config)); + std::cout << "Starting bootstrapping process ..." << std::endl; + auto boostrapper = bootstrap::Bootstrap(); + bootstrapThread = std::thread(&bootstrap::Bootstrap::start, &boostrapper, std::ref(config)); log_info("Waiting..."); if (bootstrapThread.joinable()) { bootstrapThread.join(); diff --git a/src/underpassconfig.hh b/src/underpassconfig.hh index 45446e14..85f7abf7 100644 --- a/src/underpassconfig.hh +++ b/src/underpassconfig.hh @@ -110,6 +110,9 @@ struct UnderpassConfig { if (yaml.contains_key("underpass_db_url")) { underpass_db_url = yamlConfig.get_value("underpass_db_url"); } + if (yaml.contains_key("bootstrap_page_size")) { + bootstrap_page_size = std::stoul(yamlConfig.get_value("bootstrap_page_size")); + } if (yaml.contains_key("planet_servers")) { std::vector planet_servers_config = yamlConfig.get_values("planet_servers"); for (auto it = planet_servers_config.begin(); it != planet_servers_config.end(); ++it) { @@ -155,6 +158,7 @@ struct UnderpassConfig { std::string datadir; std::vector planet_servers; unsigned int concurrency = 1; + unsigned int bootstrap_page_size = 100; frequency_t frequency = frequency_t::minutely; ptime start_time = not_a_date_time; ///< Starting time for changesets and OSM changes import diff --git a/src/validate/queryvalidate.cc b/src/validate/queryvalidate.cc index 499beae6..afdd11db 100644 --- a/src/validate/queryvalidate.cc +++ b/src/validate/queryvalidate.cc @@ -117,6 +117,17 @@ QueryValidate::updateValidation(long osm_id, const valerror_t &status, const std return query; } +std::string +QueryValidate::updateValidation(long osm_id, const valerror_t &status) const +{ + std::string format = "DELETE FROM validation WHERE osm_id = %d and status = '%s';"; + boost::format fmt(format); + fmt % osm_id; + fmt % status_list[status]; + std::string query = fmt.str(); + return query; +} + std::string QueryValidate::applyChange(const ValidateStatus &validation, const valerror_t &status) const { @@ -171,6 +182,84 @@ QueryValidate::applyChange(const ValidateStatus &validation, const valerror_t &s return query; } + +void +QueryValidate::ways( + std::shared_ptr>> wayval, + std::string &task_query +) { + for (auto it = wayval->begin(); it != wayval->end(); ++it) { + if (it->get()->status.size() > 0) { + for (auto status_it = it->get()->status.begin(); status_it != it->get()->status.end(); ++status_it) { + task_query += applyChange(*it->get(), *status_it); + } + } + } +} + +void +QueryValidate::ways( + std::shared_ptr>> wayval, + std::string &task_query, + std::shared_ptr> validation_removals +) { + for (auto it = wayval->begin(); it != wayval->end(); ++it) { + if (it->get()->status.size() > 0) { + for (auto status_it = it->get()->status.begin(); status_it != it->get()->status.end(); ++status_it) { + task_query += applyChange(*it->get(), *status_it); + } + if (!it->get()->hasStatus(overlapping)) { + task_query += updateValidation(it->get()->osm_id, overlapping, "building"); + } + if (!it->get()->hasStatus(duplicate)) { + task_query += updateValidation(it->get()->osm_id, duplicate, "building"); + } + if (!it->get()->hasStatus(badgeom)) { + task_query += updateValidation(it->get()->osm_id, badgeom, "building"); + } + if (!it->get()->hasStatus(badvalue)) { + task_query += updateValidation(it->get()->osm_id, badvalue); + } + } else { + validation_removals->push_back(it->get()->osm_id); + } + } +} + +void +QueryValidate::nodes( + std::shared_ptr>> nodeval, + std::string &task_query +) { + for (auto it = nodeval->begin(); it != nodeval->end(); ++it) { + if (it->get()->status.size() > 0) { + for (auto status_it = it->get()->status.begin(); status_it != it->get()->status.end(); ++status_it) { + task_query += applyChange(*it->get(), *status_it); + } + } + } +} + +void +QueryValidate::nodes( + std::shared_ptr>> nodeval, + std::string &task_query, + std::shared_ptr> validation_removals +) { + for (auto it = nodeval->begin(); it != nodeval->end(); ++it) { + if (it->get()->status.size() > 0) { + for (auto status_it = it->get()->status.begin(); status_it != it->get()->status.end(); ++status_it) { + task_query += applyChange(*it->get(), *status_it); + } + if (!it->get()->hasStatus(badvalue)) { + task_query += updateValidation(it->get()->osm_id, badvalue); + } + } else { + validation_removals->push_back(it->get()->osm_id); + } + } +} + } // namespace queryvalidate // local Variables: diff --git a/src/validate/queryvalidate.hh b/src/validate/queryvalidate.hh index f3b16b5c..f3c3536f 100644 --- a/src/validate/queryvalidate.hh +++ b/src/validate/queryvalidate.hh @@ -78,6 +78,13 @@ class QueryValidate { /// Update the validation table, delete any feature that has been fixed. std::string updateValidation(std::shared_ptr> removals); std::string updateValidation(long osm_id, const valerror_t &status, const std::string &source) const; + std::string updateValidation(long osm_id, const valerror_t &status) const; + void ways(std::shared_ptr>> wayval, std::string &task_query); + void nodes(std::shared_ptr>> nodeval, std::string &task_query); + void rels(std::shared_ptr>> relval, std::string &task_query); + void ways(std::shared_ptr>> wayval, std::string &task_query, std::shared_ptr> validation_removals); + void nodes(std::shared_ptr>> nodeval, std::string &task_query, std::shared_ptr> validation_removals); + void rels(std::shared_ptr>> relval, std::string &task_query, std::shared_ptr> validation_removals); // Database connection, used for escape strings std::shared_ptr dbconn; }; From 0393150c413f9103bc17ce6a95034b9dffb33410 Mon Sep 17 00:00:00 2001 From: Emillio Mariscal Date: Tue, 27 Feb 2024 16:25:05 -0300 Subject: [PATCH 2/3] + nodes to bootstrap --- setup/db/underpass.sql | 2 +- src/bootstrap/bootstrap.cc | 97 ++++++++++++++++++++++++++++++++++++-- src/bootstrap/bootstrap.hh | 11 ++++- src/osm/osmobjects.cc | 20 -------- src/raw/queryraw.cc | 38 ++++++++++++++- src/raw/queryraw.hh | 4 +- 6 files changed, 143 insertions(+), 29 deletions(-) diff --git a/setup/db/underpass.sql b/setup/db/underpass.sql index 5e8e2781..cffcb0c6 100644 --- a/setup/db/underpass.sql +++ b/setup/db/underpass.sql @@ -94,7 +94,7 @@ CREATE TABLE IF NOT EXISTS public.nodes ( uid int8 ); -CREATE TABLE IF NOT EXISTS public.rels ( +CREATE TABLE IF NOT EXISTS public.relations ( osm_id int8, changeset int8, geom public.geometry(Geometry,4326), diff --git a/src/bootstrap/bootstrap.cc b/src/bootstrap/bootstrap.cc index 474d2483..5749309b 100644 --- a/src/bootstrap/bootstrap.cc +++ b/src/bootstrap/bootstrap.cc @@ -43,6 +43,8 @@ using namespace queryraw; using namespace underpassconfig; using namespace logger; +typedef boost::geometry::model::d2::point_xy point_t; + namespace bootstrap { Bootstrap::Bootstrap(void) {} @@ -90,7 +92,7 @@ Bootstrap::start(const underpassconfig::UnderpassConfig &config) { norefs = config.norefs; processWays(); - // processNodes(); + processNodes(); // processRels(); } @@ -105,7 +107,7 @@ Bootstrap::processWays() { for (auto table_it = tables.begin(); table_it != tables.end(); ++table_it) { std::cout << std::endl << "Counting geometries ... " << std::endl; - long int total = queryraw->getWaysCount(*table_it); + long int total = queryraw->getCount(*table_it); long int count = 0; int num_chunks = total / page_size; @@ -139,7 +141,7 @@ Bootstrap::processWays() { }; std::cout << "\r" << "Processing " << *table_it << ": " << count << "/" << total << " (" << percentage << "%)"; - boost::asio::post(pool, boost::bind(&Bootstrap::threadBootstrapTask, this, wayTask)); + boost::asio::post(pool, boost::bind(&Bootstrap::threadBootstrapWayTask, this, wayTask)); } pool.join(); @@ -155,12 +157,61 @@ Bootstrap::processWays() { } +void +Bootstrap::processNodes() { + + std::cout << std::endl << "Counting nodes ... " << std::endl; + long int total = queryraw->getCount("nodes"); + long int count = 0; + int num_chunks = total / page_size; + + std::cout << "Total: " << total << std::endl; + std::cout << "Threads: " << concurrency << std::endl; + std::cout << "Page size: " << page_size << std::endl; + long lastid = 0; + + int concurrentTasks = concurrency; + int taskIndex = 0; + + for (int chunkIndex = 1; chunkIndex <= (num_chunks/concurrentTasks); chunkIndex++) { + + int percentage = (count * 100) / total; + + auto nodes = std::make_shared>(); + nodes = queryraw->getNodesFromDB(lastid, concurrency * page_size); + + auto tasks = std::make_shared>(concurrentTasks); + boost::asio::thread_pool pool(concurrentTasks); + for (int taskIndex = 0; taskIndex < concurrentTasks; taskIndex++) { + auto taskNodes = std::make_shared>(); + NodeTask nodeTask { + taskIndex, + std::ref(tasks), + std::ref(nodes), + }; + std::cout << "\r" << "Processing nodes: " << count << "/" << total << " (" << percentage << "%)"; + + boost::asio::post(pool, boost::bind(&Bootstrap::threadBootstrapNodeTask, this, nodeTask)); + } + + pool.join(); + + db->query(allTasksQueries(tasks)); + lastid = nodes->back().id; + for (auto it = tasks->begin(); it != tasks->end(); ++it) { + count += it->processed; + } + } + std::cout << std::endl; + +} + // This thread get started for every page of way void -Bootstrap::threadBootstrapTask(WayTask wayTask) +Bootstrap::threadBootstrapWayTask(WayTask wayTask) { #ifdef TIMING_DEBUG - boost::timer::auto_cpu_timer timer("bootstrap::threadBootstrapTask(wayTask): took %w seconds\n"); + boost::timer::auto_cpu_timer timer("bootstrap::threadBootstrapWayTask(wayTask): took %w seconds\n"); #endif auto taskIndex = wayTask.taskIndex; auto tasks = wayTask.tasks; @@ -192,4 +243,40 @@ Bootstrap::threadBootstrapTask(WayTask wayTask) } +// This thread get started for every page of node +void +Bootstrap::threadBootstrapNodeTask(NodeTask nodeTask) +{ +#ifdef TIMING_DEBUG + boost::timer::auto_cpu_timer timer("bootstrap::threadBootstrapNodeTask(nodeTask): took %w seconds\n"); +#endif + auto taskIndex = nodeTask.taskIndex; + auto tasks = nodeTask.tasks; + auto nodes = nodeTask.nodes; + + BootstrapTask task; + int processed = 0; + + auto nodeval = std::make_shared>>(); + + // Proccesing nodes + std::vector node_tests = {"building", "natural", "place", "waterway"}; + for (int i = 0; i < page_size; ++i) { + if (i * taskIndex < nodes->size()) { + auto node = nodes->at(i * (taskIndex + 1)); + for (auto test_it = std::begin(node_tests); test_it != std::end(node_tests); ++test_it) { + if (node.containsKey(*test_it)) { + nodeval->push_back(validator->checkNode(node, *test_it)); + } + } + ++processed; + } + } + queryvalidate->nodes(nodeval, task.query); + task.processed = processed; + const std::lock_guard lock(tasks_change_mutex); + (*tasks)[taskIndex] = task; + +} + } \ No newline at end of file diff --git a/src/bootstrap/bootstrap.hh b/src/bootstrap/bootstrap.hh index 3758ea7c..22b78d85 100644 --- a/src/bootstrap/bootstrap.hh +++ b/src/bootstrap/bootstrap.hh @@ -43,6 +43,13 @@ struct WayTask { std::shared_ptr> ways; }; +struct NodeTask { + int taskIndex; + std::shared_ptr> tasks; + std::shared_ptr> nodes; +}; + + class Bootstrap { public: Bootstrap(void); @@ -53,9 +60,11 @@ class Bootstrap { void start(const underpassconfig::UnderpassConfig &config); void processWays(); + void processNodes(); // This thread get started for every page of way - void threadBootstrapTask(WayTask wayTask); + void threadBootstrapWayTask(WayTask wayTask); + void threadBootstrapNodeTask(NodeTask nodeTask); std::string allTasksQueries(std::shared_ptr> tasks); std::shared_ptr validator; diff --git a/src/osm/osmobjects.cc b/src/osm/osmobjects.cc index 9f9c0ae3..d1f93fdd 100644 --- a/src/osm/osmobjects.cc +++ b/src/osm/osmobjects.cc @@ -27,7 +27,6 @@ #include #include #include -//#include #ifdef LIBXML #include #endif @@ -36,11 +35,6 @@ using namespace boost::posix_time; using namespace boost::gregorian; #define BOOST_BIND_GLOBAL_PLACEHOLDERS 1 -// #include -// typedef boost::geometry::model::d2::point_xy point_t; -// typedef boost::geometry::model::polygon polygon_t; -// typedef boost::geometry::model::multi_polygon multipolygon_t; -// typedef boost::geometry::model::linestring linestring_t; #include "osm/osmobjects.hh" @@ -90,20 +84,6 @@ OsmObject::dump(void) const } }; -// This represents an ODM node. A node has point coordinates, and may -// contain tags if it's a POI. -// void -// OsmWay::makeLinestring(point_t point) -// { -// // If the first and last ref are the same, it's a closed polygon, -// // like a building. -// if (refs.begin() == refs.end()) { -// boost::geometry::append(polygon, point); -// } else { -// boost::geometry::append(linestring, point); -// } -// }; - void OsmWay::dump(void) const { diff --git a/src/raw/queryraw.cc b/src/raw/queryraw.cc index d1d03fd0..b3be52e4 100644 --- a/src/raw/queryraw.cc +++ b/src/raw/queryraw.cc @@ -436,7 +436,7 @@ QueryRaw::getWaysByNodesRefs(std::string &nodeIds) const return ways; } -int QueryRaw::getWaysCount(const std::string &tableName) { +int QueryRaw::getCount(const std::string &tableName) { std::string query = "select count(osm_id) from " + tableName; auto result = dbconn->query(query); return result[0][0].as(); @@ -488,6 +488,42 @@ QueryRaw::getWaysFromDB(long lastid, int pageSize, const std::string &tableName) return ways; } +std::shared_ptr> +QueryRaw::getNodesFromDB(long lastid, int pageSize) { + std::string nodesQuery = "SELECT osm_id, ST_AsText(geom, 4326)"; + + if (lastid > 0) { + nodesQuery += ", version, tags FROM nodes where osm_id < " + std::to_string(lastid) + " order by osm_id desc limit " + std::to_string(pageSize) + ";"; + } else { + nodesQuery += ", version, tags FROM nodes order by osm_id desc limit " + std::to_string(pageSize) + ";"; + } + + auto nodes_result = dbconn->query(nodesQuery); + // Fill vector of OsmNode objects + auto nodes = std::make_shared>(); + for (auto node_it = nodes_result.begin(); node_it != nodes_result.end(); ++node_it) { + OsmNode node; + node.id = (*node_it)[0].as(); + + point_t point; + std::string point_str = (*node_it)[1].as(); + boost::geometry::read_wkt(point_str, point); + node.setPoint(boost::geometry::get<0>(point), boost::geometry::get<1>(point)); + node.version = (*node_it)[2].as(); + auto tags = (*node_it)[3]; + if (!tags.is_null()) { + auto tags = parseTagsString((*node_it)[3].as()); + for (auto const& [key, val] : tags) + { + node.addTag(key, val); + } + } + nodes->push_back(node); + } + + return nodes; +} + std::shared_ptr> QueryRaw::getWaysFromDBWithoutRefs(long lastid, int pageSize, const std::string &tableName) { std::string waysQuery; diff --git a/src/raw/queryraw.hh b/src/raw/queryraw.hh index 4396bb71..c6e2317f 100644 --- a/src/raw/queryraw.hh +++ b/src/raw/queryraw.hh @@ -76,12 +76,14 @@ class QueryRaw { // DB connection std::shared_ptr dbconn; // Get ways count - int getWaysCount(const std::string &tableName); + int getCount(const std::string &tableName); // Build tags query std::string buildTagsQuery(std::map tags) const; // Get ways by page std::shared_ptr> getWaysFromDB(long lastid, int pageSize, const std::string &tableName); std::shared_ptr> getWaysFromDBWithoutRefs(long lastid, int pageSize, const std::string &tableName); + // Get nodes by page + std::shared_ptr> getNodesFromDB(long lastid, int pageSize); }; From ef79e72eaf2be2aaa9d128657067a95e2389c572 Mon Sep 17 00:00:00 2001 From: Emillio Mariscal Date: Tue, 27 Feb 2024 21:41:29 -0300 Subject: [PATCH 3/3] Refactor, + nodes/relations support for bootstrapping --- config/validate/building.yaml | 4 +- setup/db/underpass.sql | 4 +- src/bootstrap/bootstrap.cc | 123 +++++++++++++++++++---- src/bootstrap/bootstrap.hh | 7 ++ src/osm/osmobjects.hh | 8 +- src/raw/queryraw.cc | 159 +++++++++++++++++++++--------- src/raw/queryraw.hh | 2 + src/validate/defaultvalidation.cc | 21 ++++ src/validate/defaultvalidation.hh | 4 + src/validate/semantic.cc | 81 +++++++++++++-- src/validate/semantic.hh | 1 + src/validate/validate.hh | 8 ++ 12 files changed, 343 insertions(+), 79 deletions(-) diff --git a/config/validate/building.yaml b/config/validate/building.yaml index 98b2e609..dce36bc6 100644 --- a/config/validate/building.yaml +++ b/config/validate/building.yaml @@ -7,8 +7,8 @@ config: - badgeom_maxangle: - 91 - badvalue: - - yes + - no - incomplete: - - yes + - no diff --git a/setup/db/underpass.sql b/setup/db/underpass.sql index cffcb0c6..0542e21d 100644 --- a/setup/db/underpass.sql +++ b/setup/db/underpass.sql @@ -99,7 +99,7 @@ CREATE TABLE IF NOT EXISTS public.relations ( changeset int8, geom public.geometry(Geometry,4326), tags JSONB, - members int8[], + refs int8[], timestamp timestamp with time zone, version int, "user" text, @@ -111,7 +111,7 @@ CREATE TABLE IF NOT EXISTS public.way_refs ( node_id int8 ); -CREATE TABLE IF NOT EXISTS public.rel_members ( +CREATE TABLE IF NOT EXISTS public.rel_refs ( rel_id int8, way_id int8 ); diff --git a/src/bootstrap/bootstrap.cc b/src/bootstrap/bootstrap.cc index 5749309b..2f81f22d 100644 --- a/src/bootstrap/bootstrap.cc +++ b/src/bootstrap/bootstrap.cc @@ -93,7 +93,7 @@ Bootstrap::start(const underpassconfig::UnderpassConfig &config) { processWays(); processNodes(); - // processRels(); + processRelations(); } @@ -106,22 +106,20 @@ Bootstrap::processWays() { }; for (auto table_it = tables.begin(); table_it != tables.end(); ++table_it) { - std::cout << std::endl << "Counting geometries ... " << std::endl; + std::cout << std::endl << "Processing ways ... "; long int total = queryraw->getCount(*table_it); long int count = 0; int num_chunks = total / page_size; - std::cout << "Total: " << total << std::endl; - std::cout << "Threads: " << concurrency << std::endl; - std::cout << "Page size: " << page_size << std::endl; long lastid = 0; int concurrentTasks = concurrency; int taskIndex = 0; + int percentage = 0; - for (int chunkIndex = 1; chunkIndex <= (num_chunks/concurrentTasks); chunkIndex++) { + for (int chunkIndex = 0; chunkIndex <= (num_chunks/concurrentTasks); chunkIndex++) { - int percentage = (count * 100) / total; + percentage = (count * 100) / total; auto ways = std::make_shared>(); if (!norefs) { @@ -152,6 +150,8 @@ Bootstrap::processWays() { count += it->processed; } } + percentage = (count * 100) / total; + std::cout << "\r" << "Processing " << *table_it << ": " << count << "/" << total << " (" << percentage << "%)"; } std::cout << std::endl; @@ -160,22 +160,20 @@ Bootstrap::processWays() { void Bootstrap::processNodes() { - std::cout << std::endl << "Counting nodes ... " << std::endl; + std::cout << "Processing nodes ... "; long int total = queryraw->getCount("nodes"); long int count = 0; int num_chunks = total / page_size; - std::cout << "Total: " << total << std::endl; - std::cout << "Threads: " << concurrency << std::endl; - std::cout << "Page size: " << page_size << std::endl; long lastid = 0; int concurrentTasks = concurrency; int taskIndex = 0; + int percentage = 0; - for (int chunkIndex = 1; chunkIndex <= (num_chunks/concurrentTasks); chunkIndex++) { + for (int chunkIndex = 0; chunkIndex <= (num_chunks/concurrentTasks); chunkIndex++) { - int percentage = (count * 100) / total; + percentage = (count * 100) / total; auto nodes = std::make_shared>(); nodes = queryraw->getNodesFromDB(lastid, concurrency * page_size); @@ -190,7 +188,6 @@ Bootstrap::processNodes() { std::ref(nodes), }; std::cout << "\r" << "Processing nodes: " << count << "/" << total << " (" << percentage << "%)"; - boost::asio::post(pool, boost::bind(&Bootstrap::threadBootstrapNodeTask, this, nodeTask)); } @@ -202,6 +199,57 @@ Bootstrap::processNodes() { count += it->processed; } } + percentage = (count * 100) / total; + std::cout << "\r" << "Processing nodes: " << count << "/" << total << " (" << percentage << "%)"; + std::cout << std::endl; + +} + +void +Bootstrap::processRelations() { + + std::cout << "Processing relations ... "; + long int total = queryraw->getCount("relations"); + long int count = 0; + int num_chunks = total / page_size; + + long lastid = 0; + + int concurrentTasks = concurrency; + int taskIndex = 0; + int percentage = 0; + + for (int chunkIndex = 0; chunkIndex <= (num_chunks/concurrentTasks); chunkIndex++) { + + percentage = (count * 100) / total; + + auto relations = std::make_shared>(); + relations = queryraw->getRelationsFromDB(lastid, concurrency * page_size); + + auto tasks = std::make_shared>(concurrentTasks); + boost::asio::thread_pool pool(concurrentTasks); + for (int taskIndex = 0; taskIndex < concurrentTasks; taskIndex++) { + auto taskRelations = std::make_shared>(); + RelationTask relationTask { + taskIndex, + std::ref(tasks), + std::ref(relations), + }; + std::cout << "\r" << "Processing relations: " << count << "/" << total << " (" << percentage << "%)"; + boost::asio::post(pool, boost::bind(&Bootstrap::threadBootstrapRelationTask, this, relationTask)); + } + + pool.join(); + + db->query(allTasksQueries(tasks)); + lastid = relations->back().id; + for (auto it = tasks->begin(); it != tasks->end(); ++it) { + count += it->processed; + } + } + percentage = (count * 100) / total; + std::cout << "\r" << "Processing relations: " << count << "/" << total << " (" << percentage << "%)"; + std::cout << std::endl; } @@ -223,9 +271,9 @@ Bootstrap::threadBootstrapWayTask(WayTask wayTask) auto wayval = std::make_shared>>(); // Proccesing ways - for (int i = 0; i < page_size; ++i) { - if (i * taskIndex < ways->size()) { - auto way = ways->at(i * (taskIndex + 1)); + for (int i = taskIndex * page_size; i < (taskIndex + 1) * page_size; ++i) { + if (i < ways->size()) { + auto way = ways->at(i); wayval->push_back(validator->checkWay(way, "building")); // Fill the way_refs table if (!norefs) { @@ -261,9 +309,9 @@ Bootstrap::threadBootstrapNodeTask(NodeTask nodeTask) // Proccesing nodes std::vector node_tests = {"building", "natural", "place", "waterway"}; - for (int i = 0; i < page_size; ++i) { - if (i * taskIndex < nodes->size()) { - auto node = nodes->at(i * (taskIndex + 1)); + for (int i = taskIndex * page_size; i < (taskIndex + 1) * page_size; ++i) { + if (i < nodes->size()) { + auto node = nodes->at(i); for (auto test_it = std::begin(node_tests); test_it != std::end(node_tests); ++test_it) { if (node.containsKey(*test_it)) { nodeval->push_back(validator->checkNode(node, *test_it)); @@ -279,4 +327,39 @@ Bootstrap::threadBootstrapNodeTask(NodeTask nodeTask) } +// This thread get started for every page of relation +void +Bootstrap::threadBootstrapRelationTask(RelationTask relationTask) +{ +#ifdef TIMING_DEBUG + boost::timer::auto_cpu_timer timer("bootstrap::threadBootstrapRelationTask(relationTask): took %w seconds\n"); +#endif + auto taskIndex = relationTask.taskIndex; + auto tasks = relationTask.tasks; + auto relations = relationTask.relations; + + BootstrapTask task; + int processed = 0; + + auto relationval = std::make_shared>>(); + + // Proccesing relations + for (int i = taskIndex * page_size; i < (taskIndex + 1) * page_size; ++i) { + if (i < relations->size()) { + auto relation = relations->at(i); + // relationval->push_back(validator->checkRelation(way, "building")); + // Fill the rel_members table + // for (auto ref = relation.refs.begin(); ref != relation.refs.end(); ++ref) { + // task.query += "INSERT INTO rel_refs (rel_id, way_id) VALUES (" + std::to_string(rel.id) + "," + std::to_string(*ref) + "); "; + // } + ++processed; + } + } + // queryvalidate->relations(relval, task.query); + task.processed = processed; + const std::lock_guard lock(tasks_change_mutex); + (*tasks)[taskIndex] = task; + +} + } \ No newline at end of file diff --git a/src/bootstrap/bootstrap.hh b/src/bootstrap/bootstrap.hh index 22b78d85..4142e400 100644 --- a/src/bootstrap/bootstrap.hh +++ b/src/bootstrap/bootstrap.hh @@ -49,6 +49,11 @@ struct NodeTask { std::shared_ptr> nodes; }; +struct RelationTask { + int taskIndex; + std::shared_ptr> tasks; + std::shared_ptr> relations; +}; class Bootstrap { public: @@ -61,10 +66,12 @@ class Bootstrap { void start(const underpassconfig::UnderpassConfig &config); void processWays(); void processNodes(); + void processRelations(); // This thread get started for every page of way void threadBootstrapWayTask(WayTask wayTask); void threadBootstrapNodeTask(NodeTask nodeTask); + void threadBootstrapRelationTask(RelationTask relationTask); std::string allTasksQueries(std::shared_ptr> tasks); std::shared_ptr validator; diff --git a/src/osm/osmobjects.hh b/src/osm/osmobjects.hh index e014cb80..b090c3e6 100644 --- a/src/osm/osmobjects.hh +++ b/src/osm/osmobjects.hh @@ -42,6 +42,7 @@ typedef boost::geometry::model::d2::point_xy point_t; typedef boost::geometry::model::polygon polygon_t; typedef boost::geometry::model::multi_polygon multipolygon_t; typedef boost::geometry::model::linestring linestring_t; +typedef boost::geometry::model::multi_linestring multilinestring_t; typedef boost::geometry::model::segment segment_t; typedef boost::geometry::model::point> sphere_t; @@ -174,9 +175,6 @@ class OsmWay : public OsmObject { /// Return the number of nodes in this way int numPoints(void) { return boost::geometry::num_points(linestring); }; - /// Add a point to the way's geometric data storage - // void makeLinestring(point_t point); - /// Calculate the length of the linestring in Kilometers double getLength(void) { @@ -220,6 +218,10 @@ class OsmRelation : public OsmObject { public: OsmRelation(void) { type = relation; }; + multilinestring_t multilinestring; ///< Store the members as a linestring + multipolygon_t multipolygon; ///< Store the members as a multipolygon + point_t center; ///< Store the centroid of the relation + /// Add a member to this relation void addMember(long ref, osmtype_t _type, const std::string role) { members.push_back({ref, _type, role}); }; diff --git a/src/raw/queryraw.cc b/src/raw/queryraw.cc index b3be52e4..9decd2ec 100644 --- a/src/raw/queryraw.cc +++ b/src/raw/queryraw.cc @@ -377,20 +377,42 @@ QueryRaw::getNodeCacheFromWays(std::shared_ptr> ways, std::m } } -std::map parseTagsString(std::string input) { - std::map tags; +std::map parseJSONObjectStr(std::string input) { + std::map obj; boost::property_tree::ptree pt; try { std::istringstream jsonStream(input); boost::property_tree::read_json(jsonStream, pt); } catch (const boost::property_tree::json_parser::json_parser_error& e) { std::cerr << "Error parsing JSON: " << e.what() << std::endl; - return tags; + return obj; } for (const auto& pair : pt) { - tags[pair.first] = pair.second.get_value(); + obj[pair.first] = pair.second.get_value(); } - return tags; + return obj; +} + +std::vector> parseJSONArrayStr(std::string input) { + std::vector> arr; + boost::property_tree::ptree pt; + try { + std::istringstream jsonStream(input); + boost::property_tree::read_json(jsonStream, pt); + } catch (const boost::property_tree::json_parser::json_parser_error& e) { + std::cerr << "Error parsing JSON: " << e.what() << std::endl; + return arr; + } + + for (const auto& item : pt) { + std::map obj; + for (const auto& pair : item.second) { + obj[pair.first] = pair.second.get_value(); + } + arr.push_back(obj); + } + + return arr; } std::list> @@ -417,7 +439,7 @@ QueryRaw::getWaysByNodesRefs(std::string &nodeIds) const way->version = (*way_it)[2].as(); auto tags = (*way_it)[3]; if (!tags.is_null()) { - auto tags = parseTagsString((*way_it)[3].as()); + auto tags = parseJSONObjectStr((*way_it)[3].as()); for (auto const& [key, val] : tags) { way->addTag(key, val); @@ -442,6 +464,43 @@ int QueryRaw::getCount(const std::string &tableName) { return result[0][0].as(); } +std::shared_ptr> +QueryRaw::getNodesFromDB(long lastid, int pageSize) { + std::string nodesQuery = "SELECT osm_id, ST_AsText(geom, 4326)"; + + if (lastid > 0) { + nodesQuery += ", version, tags FROM nodes where osm_id < " + std::to_string(lastid) + " order by osm_id desc limit " + std::to_string(pageSize) + ";"; + } else { + nodesQuery += ", version, tags FROM nodes order by osm_id desc limit " + std::to_string(pageSize) + ";"; + } + + auto nodes_result = dbconn->query(nodesQuery); + // Fill vector of OsmNode objects + auto nodes = std::make_shared>(); + for (auto node_it = nodes_result.begin(); node_it != nodes_result.end(); ++node_it) { + OsmNode node; + node.id = (*node_it)[0].as(); + + point_t point; + std::string point_str = (*node_it)[1].as(); + boost::geometry::read_wkt(point_str, point); + node.setPoint(boost::geometry::get<0>(point), boost::geometry::get<1>(point)); + node.version = (*node_it)[2].as(); + auto tags = (*node_it)[3]; + if (!tags.is_null()) { + auto tags = parseJSONObjectStr((*node_it)[3].as()); + for (auto const& [key, val] : tags) + { + node.addTag(key, val); + } + } + nodes->push_back(node); + } + + return nodes; +} + + std::shared_ptr> QueryRaw::getWaysFromDB(long lastid, int pageSize, const std::string &tableName) { std::string waysQuery; @@ -475,7 +534,7 @@ QueryRaw::getWaysFromDB(long lastid, int pageSize, const std::string &tableName) way.version = (*way_it)[3].as(); auto tags = (*way_it)[4]; if (!tags.is_null()) { - auto tags = parseTagsString((*way_it)[4].as()); + auto tags = parseJSONObjectStr((*way_it)[4].as()); for (auto const& [key, val] : tags) { way.addTag(key, val); @@ -488,42 +547,6 @@ QueryRaw::getWaysFromDB(long lastid, int pageSize, const std::string &tableName) return ways; } -std::shared_ptr> -QueryRaw::getNodesFromDB(long lastid, int pageSize) { - std::string nodesQuery = "SELECT osm_id, ST_AsText(geom, 4326)"; - - if (lastid > 0) { - nodesQuery += ", version, tags FROM nodes where osm_id < " + std::to_string(lastid) + " order by osm_id desc limit " + std::to_string(pageSize) + ";"; - } else { - nodesQuery += ", version, tags FROM nodes order by osm_id desc limit " + std::to_string(pageSize) + ";"; - } - - auto nodes_result = dbconn->query(nodesQuery); - // Fill vector of OsmNode objects - auto nodes = std::make_shared>(); - for (auto node_it = nodes_result.begin(); node_it != nodes_result.end(); ++node_it) { - OsmNode node; - node.id = (*node_it)[0].as(); - - point_t point; - std::string point_str = (*node_it)[1].as(); - boost::geometry::read_wkt(point_str, point); - node.setPoint(boost::geometry::get<0>(point), boost::geometry::get<1>(point)); - node.version = (*node_it)[2].as(); - auto tags = (*node_it)[3]; - if (!tags.is_null()) { - auto tags = parseTagsString((*node_it)[3].as()); - for (auto const& [key, val] : tags) - { - node.addTag(key, val); - } - } - nodes->push_back(node); - } - - return nodes; -} - std::shared_ptr> QueryRaw::getWaysFromDBWithoutRefs(long lastid, int pageSize, const std::string &tableName) { std::string waysQuery; @@ -553,7 +576,7 @@ QueryRaw::getWaysFromDBWithoutRefs(long lastid, int pageSize, const std::string } auto tags = (*way_it)[2]; if (!tags.is_null()) { - auto tags = parseTagsString((*way_it)[2].as()); + auto tags = parseJSONObjectStr((*way_it)[2].as()); for (auto const& [key, val] : tags) { way.addTag(key, val); @@ -566,6 +589,54 @@ QueryRaw::getWaysFromDBWithoutRefs(long lastid, int pageSize, const std::string return ways; } +std::shared_ptr> +QueryRaw::getRelationsFromDB(long lastid, int pageSize) { + std::string relationsQuery = "SELECT osm_id, refs, ST_AsText(geom, 4326)"; + if (lastid > 0) { + relationsQuery += ", version, tags FROM relations where osm_id < " + std::to_string(lastid) + " order by osm_id desc limit " + std::to_string(pageSize) + ";"; + } else { + relationsQuery += ", version, tags FROM relations order by osm_id desc limit " + std::to_string(pageSize) + ";"; + } + + auto relations_result = dbconn->query(relationsQuery); + // Fill vector of OsmRelation objects + auto relations = std::make_shared>(); + for (auto rel_it = relations_result.begin(); rel_it != relations_result.end(); ++rel_it) { + OsmRelation relation; + relation.id = (*rel_it)[0].as(); + auto refs = (*rel_it)[1]; + if (!refs.is_null()) { + auto refs = parseJSONArrayStr((*rel_it)[1].as()); + for (auto ref_it = refs.begin(); ref_it != refs.end(); ++ref_it) { + if (ref_it->at("type") == "w" && (ref_it->at("role") == "inner" || ref_it->at("role") == "outer")) { + relation.addMember( + std::stoi(ref_it->at("ref")), + osmobjects::osmtype_t::way, + ref_it->at("role") + ); + } + } + std::string geometry = (*rel_it)[2].as(); + if (geometry.substr(0, 12) == "MULTIPOLYGON") { + boost::geometry::read_wkt(geometry, relation.multipolygon); + } + relation.version = (*rel_it)[3].as(); + } + auto tags = (*rel_it)[4]; + if (!tags.is_null()) { + auto tags = parseJSONObjectStr((*rel_it)[4].as()); + for (auto const& [key, val] : tags) + { + relation.addTag(key, val); + } + } + relations->push_back(relation); + } + + return relations; +} + + } // namespace queryraw // local Variables: diff --git a/src/raw/queryraw.hh b/src/raw/queryraw.hh index c6e2317f..8cabf4cf 100644 --- a/src/raw/queryraw.hh +++ b/src/raw/queryraw.hh @@ -84,6 +84,8 @@ class QueryRaw { std::shared_ptr> getWaysFromDBWithoutRefs(long lastid, int pageSize, const std::string &tableName); // Get nodes by page std::shared_ptr> getNodesFromDB(long lastid, int pageSize); + // Get relations by page + std::shared_ptr> getRelationsFromDB(long lastid, int pageSize); }; diff --git a/src/validate/defaultvalidation.cc b/src/validate/defaultvalidation.cc index 5a95502c..b4520ad4 100644 --- a/src/validate/defaultvalidation.cc +++ b/src/validate/defaultvalidation.cc @@ -77,6 +77,27 @@ DefaultValidation::checkWay(const osmobjects::OsmWay &way, const std::string &ty return status; } +// This checks a relation. A relation should always have some tags. +std::shared_ptr +DefaultValidation::checkRelation(const osmobjects::OsmRelation &relation, const std::string &type) +{ + auto status = std::make_shared(relation); + status->timestamp = boost::posix_time::microsec_clock::universal_time(); + status->uid = relation.uid; + if (yamls.size() == 0) { + log_error("No config files!"); + return status; + } + yaml::Yaml tests = yamls[type]; + semantic::Semantic::checkRelation(relation, type, tests, status); + // geospatial::Geospatial::checkRelation(relation, type, tests, status); + // if (relation.linestring.size() > 2) { + // boost::geometry::centroid(way.linestring, status->center); + // } + status->source = type; + return status; +} + }; // namespace defaultvalidation #endif // EOF __DEFAULTVALIDATION_H__ diff --git a/src/validate/defaultvalidation.hh b/src/validate/defaultvalidation.hh index d3bf20f2..46958dea 100644 --- a/src/validate/defaultvalidation.hh +++ b/src/validate/defaultvalidation.hh @@ -55,6 +55,10 @@ public: /// is a building std::shared_ptr checkWay(const osmobjects::OsmWay &way, const std::string &type); + + /// This checks a relation. A relation should always have some tags. + std::shared_ptr checkRelation(const osmobjects::OsmRelation &relation, const std::string &type); + // Factory method static std::shared_ptr create(void) { return std::make_shared(); diff --git a/src/validate/semantic.cc b/src/validate/semantic.cc index ac5a955b..91f8581c 100644 --- a/src/validate/semantic.cc +++ b/src/validate/semantic.cc @@ -116,6 +116,11 @@ bool Semantic::isRequiredTag(const std::string &key, yaml::Node required_tags) { std::shared_ptr Semantic::checkNode(const osmobjects::OsmNode &node, const std::string &type, yaml::Yaml &tests, std::shared_ptr &status) { + + auto config = tests.get("config"); + bool check_badvalue = config.get_value("badvalue") == "yes"; + bool check_incomplete = config.get_value("incomplete") == "yes"; + if (node.tags.size() == 0) { status->status.insert(notags); return status; @@ -137,18 +142,24 @@ Semantic::checkNode(const osmobjects::OsmNode &node, const std::string &type, ya if (node.tags.count(type)) { for (auto vit = std::begin(node.tags); vit != std::end(node.tags); ++vit) { - if (!isValidTag(vit->first, vit->second, tags)) { - status->status.insert(badvalue); - status->values.insert(vit->first + "=" + vit->second); + if (check_badvalue) { + if (!isValidTag(vit->first, vit->second, tags)) { + status->status.insert(badvalue); + status->values.insert(vit->first + "=" + vit->second); + } } - if (isRequiredTag(vit->first, required_tags)) { - tagexists++; + if (check_incomplete) { + if (isRequiredTag(vit->first, required_tags)) { + tagexists++; + } } checkTag(vit->first, vit->second, status); } - if (tagexists != required_tags.children.size()) { - status->status.insert(incomplete); + if (check_incomplete) { + if (tagexists != required_tags.children.size()) { + status->status.insert(incomplete); + } } } return status; @@ -177,7 +188,7 @@ Semantic::checkWay(const osmobjects::OsmWay &way, const std::string &type, yaml: // List of valid tags to be validated yaml::Node tags; if (check_badvalue) { - tags = tests.get("tags"); + tags = tests.get("tags"); } if (check_badvalue && way.tags.size() == 0) { @@ -209,6 +220,60 @@ Semantic::checkWay(const osmobjects::OsmWay &way, const std::string &type, yaml: return status; } +// This checks a relation. +std::shared_ptr +Semantic::checkRelation(const osmobjects::OsmRelation &relation, const std::string &type, yaml::Yaml &tests, std::shared_ptr &status) +{ + if (relation.action == osmobjects::remove) { + return status; + } + + // These values are in the config section of the YAML file + auto config = tests.get("config"); + bool check_badvalue = config.get_value("badvalue") == "yes"; + bool check_incomplete = config.get_value("incomplete") == "yes"; + + // List of required tags to be validated + yaml::Node required_tags; + if (check_incomplete) { + required_tags = tests.get("required_tags"); + } + + // List of valid tags to be validated + yaml::Node tags; + if (check_badvalue) { + tags = tests.get("tags"); + } + + if (check_badvalue && relation.tags.size() == 0) { + status->status.insert(notags); + return status; + } + + int tagexists = 0; + if (relation.tags.count(type)) { + for (auto vit = std::begin(relation.tags); vit != std::end(relation.tags); ++vit) { + if (check_badvalue) { + if (tags.children.size() > 0 && !isValidTag(vit->first, vit->second, tags)) { + status->status.insert(badvalue); + status->values.insert(vit->first + "=" + vit->second); + } + checkTag(vit->first, vit->second, status); + } + if (check_incomplete) { + if (isRequiredTag(vit->first, required_tags)) { + tagexists++; + } + } + } + + if (check_incomplete && tagexists != required_tags.children.size()) { + status->status.insert(incomplete); + } + } + return status; +} + }; // namespace semantic #endif // EOF __SEMANTIC_H__ diff --git a/src/validate/semantic.hh b/src/validate/semantic.hh index 14f93e11..7dfe3a85 100644 --- a/src/validate/semantic.hh +++ b/src/validate/semantic.hh @@ -57,6 +57,7 @@ public: ~Semantic(void) { }; static std::shared_ptr checkNode(const osmobjects::OsmNode &node, const std::string &type, yaml::Yaml &tests, std::shared_ptr &status); static std::shared_ptr checkWay(const osmobjects::OsmWay &way, const std::string &type, yaml::Yaml &tests, std::shared_ptr &status); + static std::shared_ptr checkRelation(const osmobjects::OsmRelation &relation, const std::string &type, yaml::Yaml &tests, std::shared_ptr &status); private: static bool isValidTag(const std::string &key, const std::string &value, yaml::Node tags); static bool isRequiredTag(const std::string &key, yaml::Node required_tags); diff --git a/src/validate/validate.hh b/src/validate/validate.hh index e8d51e0e..d3ed80f2 100644 --- a/src/validate/validate.hh +++ b/src/validate/validate.hh @@ -118,6 +118,14 @@ class ValidateStatus { version = way.version; timestamp = way.timestamp; } + ValidateStatus(const osmobjects::OsmRelation &relation) { + osm_id = relation.id; + uid = relation.uid; + changeset = relation.changeset; + objtype = osmobjects::relation; + version = relation.version; + timestamp = relation.timestamp; + } /// Does this change have a particular status value bool hasStatus(const valerror_t &val) const { auto match = std::find(status.begin(), status.end(), val);