diff --git a/config/validate/building.yaml b/config/validate/building.yaml index 98b2e609..dce36bc6 100644 --- a/config/validate/building.yaml +++ b/config/validate/building.yaml @@ -7,8 +7,8 @@ config: - badgeom_maxangle: - 91 - badvalue: - - yes + - no - incomplete: - - yes + - no diff --git a/setup/bootstrap.sh b/setup/bootstrap.sh index 4c4a6b0e..db0cd30d 100644 --- a/setup/bootstrap.sh +++ b/setup/bootstrap.sh @@ -107,7 +107,7 @@ then PGPASSWORD=$PASS psql --host $HOST --user $USER --port $PORT $DB < db/indexes.sql echo "Configuring Underpass ..." - python3 poly2geojson.py $COUNTRY.poly + python3 ../utils/poly2geojson.py $COUNTRY.poly if "$use_docker"; then docker cp $COUNTRY.geojson underpass:/etc/underpass/priority.geojson diff --git a/setup/db/underpass.sql b/setup/db/underpass.sql index bae39f2f..0542e21d 100644 --- a/setup/db/underpass.sql +++ b/setup/db/underpass.sql @@ -94,11 +94,28 @@ CREATE TABLE IF NOT EXISTS public.nodes ( uid int8 ); +CREATE TABLE IF NOT EXISTS public.relations ( + osm_id int8, + changeset int8, + geom public.geometry(Geometry,4326), + tags JSONB, + refs int8[], + timestamp timestamp with time zone, + version int, + "user" text, + uid int8 +); + CREATE TABLE IF NOT EXISTS public.way_refs ( way_id int8, node_id int8 ); +CREATE TABLE IF NOT EXISTS public.rel_refs ( + rel_id int8, + way_id int8 +); + CREATE UNIQUE INDEX nodes_id_idx ON public.nodes (osm_id DESC); CREATE UNIQUE INDEX ways_poly_id_idx ON public.ways_poly (osm_id DESC); CREATE UNIQUE INDEX ways_line_id_idx ON public.ways_line(osm_id DESC); diff --git a/src/bootstrap/bootstrap.cc b/src/bootstrap/bootstrap.cc index 13b4c6b1..2f81f22d 100644 --- a/src/bootstrap/bootstrap.cc +++ b/src/bootstrap/bootstrap.cc @@ -43,12 +43,14 @@ using namespace queryraw; using namespace underpassconfig; using namespace logger; +typedef boost::geometry::model::d2::point_xy point_t; + namespace bootstrap { -const int PAGE_SIZE = 100; +Bootstrap::Bootstrap(void) {} std::string -allTasksQueries(std::shared_ptr> tasks) { +Bootstrap::allTasksQueries(std::shared_ptr> tasks) { std::string queries = ""; for (auto it = tasks->begin(); it != tasks->end(); ++it) { queries += it->query ; @@ -56,10 +58,10 @@ allTasksQueries(std::shared_ptr> tasks) { return queries; } -void startProcessingWays(const underpassconfig::UnderpassConfig &config) { - +void +Bootstrap::start(const underpassconfig::UnderpassConfig &config) { std::cout << "Connecting to the database ..." << std::endl; - auto db = std::make_shared(); + db = std::make_shared(); if (!db->connect(config.underpass_db_url)) { std::cout << "Could not connect to Underpass DB, aborting bootstrapping thread!" << std::endl; return; @@ -81,40 +83,49 @@ void startProcessingWays(const underpassconfig::UnderpassConfig &config) { log_debug("Couldn't load plugin! %1%", e.what()); exit(0); } - auto validator = creator(); - auto queryvalidate = std::make_shared(db); - auto queryraw = std::make_shared(db); + validator = creator(); + queryvalidate = std::make_shared(db); + queryraw = std::make_shared(db); + page_size = config.bootstrap_page_size; + concurrency = config.concurrency; + norefs = config.norefs; + + processWays(); + processNodes(); + processRelations(); + +} + +void +Bootstrap::processWays() { + std::vector tables = { QueryRaw::polyTable, QueryRaw::lineTable }; for (auto table_it = tables.begin(); table_it != tables.end(); ++table_it) { - std::cout << std::endl << "Counting geometries ... " << std::endl; - long int total = queryraw->getWaysCount(*table_it); + std::cout << std::endl << "Processing ways ... "; + long int total = queryraw->getCount(*table_it); long int count = 0; - int num_chunks = total / PAGE_SIZE; + int num_chunks = total / page_size; - std::cout << "Total: " << total << std::endl; - std::cout << "Threads: " << config.concurrency << std::endl; - std::cout << "Page size: " << PAGE_SIZE << std::endl; long lastid = 0; - int concurrentTasks = config.concurrency; + int concurrentTasks = concurrency; int taskIndex = 0; - std::chrono::steady_clock::time_point begin; - std::chrono::steady_clock::time_point end; + int percentage = 0; - for (int chunkIndex = 1; chunkIndex <= (num_chunks/concurrentTasks); chunkIndex++) { + for (int chunkIndex = 0; chunkIndex <= (num_chunks/concurrentTasks); chunkIndex++) { - int percentage = (count * 100) / total; + percentage = (count * 100) / total; auto ways = std::make_shared>(); - if (!config.norefs) { - ways = queryraw->getWaysFromDB(lastid, config.concurrency * PAGE_SIZE, *table_it); + if (!norefs) { + ways = queryraw->getWaysFromDB(lastid, concurrency * page_size, *table_it); } else { - ways = queryraw->getWaysFromDBWithoutRefs(lastid, config.concurrency * PAGE_SIZE, *table_it); + ways = queryraw->getWaysFromDBWithoutRefs(lastid, concurrency * page_size, *table_it); } auto tasks = std::make_shared>(concurrentTasks); @@ -122,15 +133,13 @@ void startProcessingWays(const underpassconfig::UnderpassConfig &config) { for (int taskIndex = 0; taskIndex < concurrentTasks; taskIndex++) { auto taskWays = std::make_shared>(); WayTask wayTask { - std::ref(validator), - std::ref(queryvalidate), - config, taskIndex, std::ref(tasks), std::ref(ways), }; std::cout << "\r" << "Processing " << *table_it << ": " << count << "/" << total << " (" << percentage << "%)"; - boost::asio::post(pool, boost::bind(threadBootstrapTask, wayTask)); + + boost::asio::post(pool, boost::bind(&Bootstrap::threadBootstrapWayTask, this, wayTask)); } pool.join(); @@ -141,21 +150,117 @@ void startProcessingWays(const underpassconfig::UnderpassConfig &config) { count += it->processed; } } + percentage = (count * 100) / total; + std::cout << "\r" << "Processing " << *table_it << ": " << count << "/" << total << " (" << percentage << "%)"; + } + std::cout << std::endl; + +} + +void +Bootstrap::processNodes() { + + std::cout << "Processing nodes ... "; + long int total = queryraw->getCount("nodes"); + long int count = 0; + int num_chunks = total / page_size; + + long lastid = 0; + + int concurrentTasks = concurrency; + int taskIndex = 0; + int percentage = 0; + + for (int chunkIndex = 0; chunkIndex <= (num_chunks/concurrentTasks); chunkIndex++) { + + percentage = (count * 100) / total; + + auto nodes = std::make_shared>(); + nodes = queryraw->getNodesFromDB(lastid, concurrency * page_size); + + auto tasks = std::make_shared>(concurrentTasks); + boost::asio::thread_pool pool(concurrentTasks); + for (int taskIndex = 0; taskIndex < concurrentTasks; taskIndex++) { + auto taskNodes = std::make_shared>(); + NodeTask nodeTask { + taskIndex, + std::ref(tasks), + std::ref(nodes), + }; + std::cout << "\r" << "Processing nodes: " << count << "/" << total << " (" << percentage << "%)"; + boost::asio::post(pool, boost::bind(&Bootstrap::threadBootstrapNodeTask, this, nodeTask)); + } + + pool.join(); + + db->query(allTasksQueries(tasks)); + lastid = nodes->back().id; + for (auto it = tasks->begin(); it != tasks->end(); ++it) { + count += it->processed; + } + } + percentage = (count * 100) / total; + std::cout << "\r" << "Processing nodes: " << count << "/" << total << " (" << percentage << "%)"; + std::cout << std::endl; + +} + +void +Bootstrap::processRelations() { + + std::cout << "Processing relations ... "; + long int total = queryraw->getCount("relations"); + long int count = 0; + int num_chunks = total / page_size; + + long lastid = 0; + + int concurrentTasks = concurrency; + int taskIndex = 0; + int percentage = 0; + + for (int chunkIndex = 0; chunkIndex <= (num_chunks/concurrentTasks); chunkIndex++) { + + percentage = (count * 100) / total; + + auto relations = std::make_shared>(); + relations = queryraw->getRelationsFromDB(lastid, concurrency * page_size); + + auto tasks = std::make_shared>(concurrentTasks); + boost::asio::thread_pool pool(concurrentTasks); + for (int taskIndex = 0; taskIndex < concurrentTasks; taskIndex++) { + auto taskRelations = std::make_shared>(); + RelationTask relationTask { + taskIndex, + std::ref(tasks), + std::ref(relations), + }; + std::cout << "\r" << "Processing relations: " << count << "/" << total << " (" << percentage << "%)"; + boost::asio::post(pool, boost::bind(&Bootstrap::threadBootstrapRelationTask, this, relationTask)); + } + + pool.join(); + + db->query(allTasksQueries(tasks)); + lastid = relations->back().id; + for (auto it = tasks->begin(); it != tasks->end(); ++it) { + count += it->processed; + } } + percentage = (count * 100) / total; + std::cout << "\r" << "Processing relations: " << count << "/" << total << " (" << percentage << "%)"; + std::cout << std::endl; } // This thread get started for every page of way void -threadBootstrapTask(WayTask wayTask) +Bootstrap::threadBootstrapWayTask(WayTask wayTask) { #ifdef TIMING_DEBUG - boost::timer::auto_cpu_timer timer("bootstrap::threadBootstrapTask(wayTask): took %w seconds\n"); + boost::timer::auto_cpu_timer timer("bootstrap::threadBootstrapWayTask(wayTask): took %w seconds\n"); #endif - auto plugin = wayTask.plugin; - auto queryvalidate = wayTask.queryvalidate; - auto config = wayTask.config; auto taskIndex = wayTask.taskIndex; auto tasks = wayTask.tasks; auto ways = wayTask.ways; @@ -163,16 +268,15 @@ threadBootstrapTask(WayTask wayTask) BootstrapTask task; int processed = 0; + auto wayval = std::make_shared>>(); + // Proccesing ways - for (int i = 0; i < PAGE_SIZE; ++i) { - if (i * taskIndex < ways->size()) { - auto way = ways->at(i * (taskIndex + 1)); - auto status = plugin->checkWay(way, "building"); - for (auto status_it = status->status.begin(); status_it != status->status.end(); ++status_it) { - task.query += queryvalidate->applyChange(*status, *status_it); - } + for (int i = taskIndex * page_size; i < (taskIndex + 1) * page_size; ++i) { + if (i < ways->size()) { + auto way = ways->at(i); + wayval->push_back(validator->checkWay(way, "building")); // Fill the way_refs table - if (!config.norefs) { + if (!norefs) { for (auto ref = way.refs.begin(); ref != way.refs.end(); ++ref) { task.query += "INSERT INTO way_refs (way_id, node_id) VALUES (" + std::to_string(way.id) + "," + std::to_string(*ref) + "); "; } @@ -180,6 +284,78 @@ threadBootstrapTask(WayTask wayTask) ++processed; } } + queryvalidate->ways(wayval, task.query); + task.processed = processed; + const std::lock_guard lock(tasks_change_mutex); + (*tasks)[taskIndex] = task; + +} + +// This thread get started for every page of node +void +Bootstrap::threadBootstrapNodeTask(NodeTask nodeTask) +{ +#ifdef TIMING_DEBUG + boost::timer::auto_cpu_timer timer("bootstrap::threadBootstrapNodeTask(nodeTask): took %w seconds\n"); +#endif + auto taskIndex = nodeTask.taskIndex; + auto tasks = nodeTask.tasks; + auto nodes = nodeTask.nodes; + + BootstrapTask task; + int processed = 0; + + auto nodeval = std::make_shared>>(); + + // Proccesing nodes + std::vector node_tests = {"building", "natural", "place", "waterway"}; + for (int i = taskIndex * page_size; i < (taskIndex + 1) * page_size; ++i) { + if (i < nodes->size()) { + auto node = nodes->at(i); + for (auto test_it = std::begin(node_tests); test_it != std::end(node_tests); ++test_it) { + if (node.containsKey(*test_it)) { + nodeval->push_back(validator->checkNode(node, *test_it)); + } + } + ++processed; + } + } + queryvalidate->nodes(nodeval, task.query); + task.processed = processed; + const std::lock_guard lock(tasks_change_mutex); + (*tasks)[taskIndex] = task; + +} + +// This thread get started for every page of relation +void +Bootstrap::threadBootstrapRelationTask(RelationTask relationTask) +{ +#ifdef TIMING_DEBUG + boost::timer::auto_cpu_timer timer("bootstrap::threadBootstrapRelationTask(relationTask): took %w seconds\n"); +#endif + auto taskIndex = relationTask.taskIndex; + auto tasks = relationTask.tasks; + auto relations = relationTask.relations; + + BootstrapTask task; + int processed = 0; + + auto relationval = std::make_shared>>(); + + // Proccesing relations + for (int i = taskIndex * page_size; i < (taskIndex + 1) * page_size; ++i) { + if (i < relations->size()) { + auto relation = relations->at(i); + // relationval->push_back(validator->checkRelation(way, "building")); + // Fill the rel_members table + // for (auto ref = relation.refs.begin(); ref != relation.refs.end(); ++ref) { + // task.query += "INSERT INTO rel_refs (rel_id, way_id) VALUES (" + std::to_string(rel.id) + "," + std::to_string(*ref) + "); "; + // } + ++processed; + } + } + // queryvalidate->relations(relval, task.query); task.processed = processed; const std::lock_guard lock(tasks_change_mutex); (*tasks)[taskIndex] = task; diff --git a/src/bootstrap/bootstrap.hh b/src/bootstrap/bootstrap.hh index 8210ad99..4142e400 100644 --- a/src/bootstrap/bootstrap.hh +++ b/src/bootstrap/bootstrap.hh @@ -21,6 +21,7 @@ #include "raw/queryraw.hh" #include "underpassconfig.hh" #include "validate/validate.hh" +#include using namespace queryvalidate; using namespace queryraw; @@ -37,18 +38,50 @@ struct BootstrapTask { }; struct WayTask { - std::shared_ptr plugin; - std::shared_ptr queryvalidate; - underpassconfig::UnderpassConfig config; int taskIndex; std::shared_ptr> tasks; std::shared_ptr> ways; }; -void startProcessingWays(const underpassconfig::UnderpassConfig &config); +struct NodeTask { + int taskIndex; + std::shared_ptr> tasks; + std::shared_ptr> nodes; +}; + +struct RelationTask { + int taskIndex; + std::shared_ptr> tasks; + std::shared_ptr> relations; +}; + +class Bootstrap { + public: + Bootstrap(void); + ~Bootstrap(void){}; -// This thread get started for every page of way -void threadBootstrapTask(WayTask wayTask); + static const std::string polyTable; + static const std::string lineTable; + + void start(const underpassconfig::UnderpassConfig &config); + void processWays(); + void processNodes(); + void processRelations(); + + // This thread get started for every page of way + void threadBootstrapWayTask(WayTask wayTask); + void threadBootstrapNodeTask(NodeTask nodeTask); + void threadBootstrapRelationTask(RelationTask relationTask); + std::string allTasksQueries(std::shared_ptr> tasks); + + std::shared_ptr validator; + std::shared_ptr queryvalidate; + std::shared_ptr queryraw; + std::shared_ptr db; + bool norefs; + unsigned int concurrency; + unsigned int page_size; +}; static std::mutex tasks_change_mutex; diff --git a/src/osm/osmobjects.cc b/src/osm/osmobjects.cc index 9f9c0ae3..d1f93fdd 100644 --- a/src/osm/osmobjects.cc +++ b/src/osm/osmobjects.cc @@ -27,7 +27,6 @@ #include #include #include -//#include #ifdef LIBXML #include #endif @@ -36,11 +35,6 @@ using namespace boost::posix_time; using namespace boost::gregorian; #define BOOST_BIND_GLOBAL_PLACEHOLDERS 1 -// #include -// typedef boost::geometry::model::d2::point_xy point_t; -// typedef boost::geometry::model::polygon polygon_t; -// typedef boost::geometry::model::multi_polygon multipolygon_t; -// typedef boost::geometry::model::linestring linestring_t; #include "osm/osmobjects.hh" @@ -90,20 +84,6 @@ OsmObject::dump(void) const } }; -// This represents an ODM node. A node has point coordinates, and may -// contain tags if it's a POI. -// void -// OsmWay::makeLinestring(point_t point) -// { -// // If the first and last ref are the same, it's a closed polygon, -// // like a building. -// if (refs.begin() == refs.end()) { -// boost::geometry::append(polygon, point); -// } else { -// boost::geometry::append(linestring, point); -// } -// }; - void OsmWay::dump(void) const { diff --git a/src/osm/osmobjects.hh b/src/osm/osmobjects.hh index e014cb80..b090c3e6 100644 --- a/src/osm/osmobjects.hh +++ b/src/osm/osmobjects.hh @@ -42,6 +42,7 @@ typedef boost::geometry::model::d2::point_xy point_t; typedef boost::geometry::model::polygon polygon_t; typedef boost::geometry::model::multi_polygon multipolygon_t; typedef boost::geometry::model::linestring linestring_t; +typedef boost::geometry::model::multi_linestring multilinestring_t; typedef boost::geometry::model::segment segment_t; typedef boost::geometry::model::point> sphere_t; @@ -174,9 +175,6 @@ class OsmWay : public OsmObject { /// Return the number of nodes in this way int numPoints(void) { return boost::geometry::num_points(linestring); }; - /// Add a point to the way's geometric data storage - // void makeLinestring(point_t point); - /// Calculate the length of the linestring in Kilometers double getLength(void) { @@ -220,6 +218,10 @@ class OsmRelation : public OsmObject { public: OsmRelation(void) { type = relation; }; + multilinestring_t multilinestring; ///< Store the members as a linestring + multipolygon_t multipolygon; ///< Store the members as a multipolygon + point_t center; ///< Store the centroid of the relation + /// Add a member to this relation void addMember(long ref, osmtype_t _type, const std::string role) { members.push_back({ref, _type, role}); }; diff --git a/src/raw/queryraw.cc b/src/raw/queryraw.cc index c75738ab..9decd2ec 100644 --- a/src/raw/queryraw.cc +++ b/src/raw/queryraw.cc @@ -24,8 +24,6 @@ /// includes querying existing data in the database, as well as /// updating the database. -// TODO: add support for relations/multipolygon - // This is generated by autoconf #ifdef HAVE_CONFIG_H #include "unconfig.h" @@ -233,6 +231,12 @@ QueryRaw::applyChange(const OsmWay &way) const return query; } +std::string +QueryRaw::applyChange(const OsmRelation &relation) const +{ + +} + std::vector arrayStrToVector(std::string &refs_str) { refs_str.erase(0, 1); refs_str.erase(refs_str.size() - 1); @@ -261,10 +265,8 @@ void QueryRaw::getNodeCache(std::shared_ptr osmchanges, const mul if (way->action != osmobjects::remove) { // Save referenced nodes for later use for (auto rit = std::begin(way->refs); rit != std::end(way->refs); ++rit) { - if (way->action != osmobjects::remove) { - if (!osmchanges->nodecache.count(*rit)) { - referencedNodeIds += std::to_string(*rit) + ","; - } + if (!osmchanges->nodecache.count(*rit)) { + referencedNodeIds += std::to_string(*rit) + ","; } } } else { @@ -337,6 +339,10 @@ void QueryRaw::getNodeCache(std::shared_ptr osmchanges, const mul } } + // Build relation multipolyon geometries + // TODO + + } void @@ -371,20 +377,42 @@ QueryRaw::getNodeCacheFromWays(std::shared_ptr> ways, std::m } } -std::map parseTagsString(std::string input) { - std::map tags; +std::map parseJSONObjectStr(std::string input) { + std::map obj; boost::property_tree::ptree pt; try { std::istringstream jsonStream(input); boost::property_tree::read_json(jsonStream, pt); } catch (const boost::property_tree::json_parser::json_parser_error& e) { std::cerr << "Error parsing JSON: " << e.what() << std::endl; - return tags; + return obj; } for (const auto& pair : pt) { - tags[pair.first] = pair.second.get_value(); + obj[pair.first] = pair.second.get_value(); } - return tags; + return obj; +} + +std::vector> parseJSONArrayStr(std::string input) { + std::vector> arr; + boost::property_tree::ptree pt; + try { + std::istringstream jsonStream(input); + boost::property_tree::read_json(jsonStream, pt); + } catch (const boost::property_tree::json_parser::json_parser_error& e) { + std::cerr << "Error parsing JSON: " << e.what() << std::endl; + return arr; + } + + for (const auto& item : pt) { + std::map obj; + for (const auto& pair : item.second) { + obj[pair.first] = pair.second.get_value(); + } + arr.push_back(obj); + } + + return arr; } std::list> @@ -411,7 +439,7 @@ QueryRaw::getWaysByNodesRefs(std::string &nodeIds) const way->version = (*way_it)[2].as(); auto tags = (*way_it)[3]; if (!tags.is_null()) { - auto tags = parseTagsString((*way_it)[3].as()); + auto tags = parseJSONObjectStr((*way_it)[3].as()); for (auto const& [key, val] : tags) { way->addTag(key, val); @@ -430,12 +458,49 @@ QueryRaw::getWaysByNodesRefs(std::string &nodeIds) const return ways; } -int QueryRaw::getWaysCount(const std::string &tableName) { +int QueryRaw::getCount(const std::string &tableName) { std::string query = "select count(osm_id) from " + tableName; auto result = dbconn->query(query); return result[0][0].as(); } +std::shared_ptr> +QueryRaw::getNodesFromDB(long lastid, int pageSize) { + std::string nodesQuery = "SELECT osm_id, ST_AsText(geom, 4326)"; + + if (lastid > 0) { + nodesQuery += ", version, tags FROM nodes where osm_id < " + std::to_string(lastid) + " order by osm_id desc limit " + std::to_string(pageSize) + ";"; + } else { + nodesQuery += ", version, tags FROM nodes order by osm_id desc limit " + std::to_string(pageSize) + ";"; + } + + auto nodes_result = dbconn->query(nodesQuery); + // Fill vector of OsmNode objects + auto nodes = std::make_shared>(); + for (auto node_it = nodes_result.begin(); node_it != nodes_result.end(); ++node_it) { + OsmNode node; + node.id = (*node_it)[0].as(); + + point_t point; + std::string point_str = (*node_it)[1].as(); + boost::geometry::read_wkt(point_str, point); + node.setPoint(boost::geometry::get<0>(point), boost::geometry::get<1>(point)); + node.version = (*node_it)[2].as(); + auto tags = (*node_it)[3]; + if (!tags.is_null()) { + auto tags = parseJSONObjectStr((*node_it)[3].as()); + for (auto const& [key, val] : tags) + { + node.addTag(key, val); + } + } + nodes->push_back(node); + } + + return nodes; +} + + std::shared_ptr> QueryRaw::getWaysFromDB(long lastid, int pageSize, const std::string &tableName) { std::string waysQuery; @@ -469,7 +534,7 @@ QueryRaw::getWaysFromDB(long lastid, int pageSize, const std::string &tableName) way.version = (*way_it)[3].as(); auto tags = (*way_it)[4]; if (!tags.is_null()) { - auto tags = parseTagsString((*way_it)[4].as()); + auto tags = parseJSONObjectStr((*way_it)[4].as()); for (auto const& [key, val] : tags) { way.addTag(key, val); @@ -511,7 +576,7 @@ QueryRaw::getWaysFromDBWithoutRefs(long lastid, int pageSize, const std::string } auto tags = (*way_it)[2]; if (!tags.is_null()) { - auto tags = parseTagsString((*way_it)[2].as()); + auto tags = parseJSONObjectStr((*way_it)[2].as()); for (auto const& [key, val] : tags) { way.addTag(key, val); @@ -524,6 +589,54 @@ QueryRaw::getWaysFromDBWithoutRefs(long lastid, int pageSize, const std::string return ways; } +std::shared_ptr> +QueryRaw::getRelationsFromDB(long lastid, int pageSize) { + std::string relationsQuery = "SELECT osm_id, refs, ST_AsText(geom, 4326)"; + if (lastid > 0) { + relationsQuery += ", version, tags FROM relations where osm_id < " + std::to_string(lastid) + " order by osm_id desc limit " + std::to_string(pageSize) + ";"; + } else { + relationsQuery += ", version, tags FROM relations order by osm_id desc limit " + std::to_string(pageSize) + ";"; + } + + auto relations_result = dbconn->query(relationsQuery); + // Fill vector of OsmRelation objects + auto relations = std::make_shared>(); + for (auto rel_it = relations_result.begin(); rel_it != relations_result.end(); ++rel_it) { + OsmRelation relation; + relation.id = (*rel_it)[0].as(); + auto refs = (*rel_it)[1]; + if (!refs.is_null()) { + auto refs = parseJSONArrayStr((*rel_it)[1].as()); + for (auto ref_it = refs.begin(); ref_it != refs.end(); ++ref_it) { + if (ref_it->at("type") == "w" && (ref_it->at("role") == "inner" || ref_it->at("role") == "outer")) { + relation.addMember( + std::stoi(ref_it->at("ref")), + osmobjects::osmtype_t::way, + ref_it->at("role") + ); + } + } + std::string geometry = (*rel_it)[2].as(); + if (geometry.substr(0, 12) == "MULTIPOLYGON") { + boost::geometry::read_wkt(geometry, relation.multipolygon); + } + relation.version = (*rel_it)[3].as(); + } + auto tags = (*rel_it)[4]; + if (!tags.is_null()) { + auto tags = parseJSONObjectStr((*rel_it)[4].as()); + for (auto const& [key, val] : tags) + { + relation.addTag(key, val); + } + } + relations->push_back(relation); + } + + return relations; +} + + } // namespace queryraw // local Variables: diff --git a/src/raw/queryraw.hh b/src/raw/queryraw.hh index 20c99eda..8cabf4cf 100644 --- a/src/raw/queryraw.hh +++ b/src/raw/queryraw.hh @@ -64,23 +64,28 @@ class QueryRaw { std::string applyChange(const OsmNode &node) const; /// Build query for processed Way std::string applyChange(const OsmWay &way) const; - /// Build query for updating modified nodes in ways geometries - // std::string applyChange(const std::shared_ptr>> nodes) const; + /// Build query for processed Relation + std::string applyChange(const OsmRelation &relation) const; /// Get nodes for filling Node cache void getNodeCache(std::shared_ptr osmchanges, const multipolygon_t &poly); /// Get nodes for filling Node cache from ways refs void getNodeCacheFromWays(std::shared_ptr> ways, std::map &nodecache) const; // Get ways by refs std::list> getWaysByNodesRefs(std::string &nodeIds) const; + // DB connection std::shared_ptr dbconn; // Get ways count - int getWaysCount(const std::string &tableName); + int getCount(const std::string &tableName); // Build tags query std::string buildTagsQuery(std::map tags) const; // Get ways by page std::shared_ptr> getWaysFromDB(long lastid, int pageSize, const std::string &tableName); std::shared_ptr> getWaysFromDBWithoutRefs(long lastid, int pageSize, const std::string &tableName); + // Get nodes by page + std::shared_ptr> getNodesFromDB(long lastid, int pageSize); + // Get relations by page + std::shared_ptr> getRelationsFromDB(long lastid, int pageSize); }; diff --git a/src/replicator/threads.cc b/src/replicator/threads.cc index 670d2565..9187ab98 100644 --- a/src/replicator/threads.cc +++ b/src/replicator/threads.cc @@ -479,6 +479,7 @@ threadOsmChange(OsmChangeTask osmChangeTask) auto removed_nodes = std::make_shared>(); auto removed_ways = std::make_shared>(); + auto removed_relations = std::make_shared>(); auto validation_removals = std::make_shared>(); // Raw data and validation @@ -524,6 +525,25 @@ threadOsmChange(OsmChangeTask osmChangeTask) } } + // Relations + // for (auto rit = std::begin(change->relations); rit != std::end(change->relations); ++rit) { + // osmobjects::OsmRelation *relation = rit->get(); + + // if (relation->action != osmobjects::remove && !relation->priority) { + // continue; + // } + + // // Remove deleted relations from validation table + // if (!config->disable_validation && relation->action == osmobjects::remove) { + // removed_relations->push_back(relation->id); + // } + + // // Update relations, ignore new ones outside priority area + // if (!config->disable_raw) { + // task.query += queryraw->applyChange(*relation); + // } + // } + } } @@ -532,30 +552,20 @@ threadOsmChange(OsmChangeTask osmChangeTask) // Validate ways auto wayval = osmchanges->validateWays(poly, plugin); - for (auto it = wayval->begin(); it != wayval->end(); ++it) { + queryvalidate->ways(wayval, task.query, validation_removals); - if (it->get()->status.size() > 0) { - for (auto status_it = it->get()->status.begin(); status_it != it->get()->status.end(); ++status_it) { - task.query += queryvalidate->applyChange(*it->get(), *status_it); - } - if (!it->get()->hasStatus(overlapping)) { - task.query += queryvalidate->updateValidation(it->get()->osm_id, overlapping, "building"); - } - if (!it->get()->hasStatus(duplicate)) { - task.query += queryvalidate->updateValidation(it->get()->osm_id, duplicate, "building"); - } - if (!it->get()->hasStatus(badgeom)) { - task.query += queryvalidate->updateValidation(it->get()->osm_id, badgeom, "building"); - } - } else { - validation_removals->push_back(it->get()->osm_id); - } - } + // Validate nodes + auto nodeval = osmchanges->validateNodes(poly, plugin); + queryvalidate->nodes(nodeval, task.query, validation_removals); + + // Validate relations + // task.query += queryvalidate->rels(wayval, task.query, validation_removals); // Remove validation entries for removed objects task.query += queryvalidate->updateValidation(validation_removals); task.query += queryvalidate->updateValidation(removed_nodes); task.query += queryvalidate->updateValidation(removed_ways); + // task.query += queryvalidate->updateValidation(removed_relations); } diff --git a/src/underpass.cc b/src/underpass.cc index b16bec4c..723db754 100644 --- a/src/underpass.cc +++ b/src/underpass.cc @@ -329,8 +329,9 @@ main(int argc, char *argv[]) if (vm.count("bootstrap")){ std::thread bootstrapThread; - std::cout << "Starting bootstrapping proccess ..." << std::endl; - bootstrapThread = std::thread(bootstrap::startProcessingWays, std::ref(config)); + std::cout << "Starting bootstrapping process ..." << std::endl; + auto boostrapper = bootstrap::Bootstrap(); + bootstrapThread = std::thread(&bootstrap::Bootstrap::start, &boostrapper, std::ref(config)); log_info("Waiting..."); if (bootstrapThread.joinable()) { bootstrapThread.join(); diff --git a/src/underpassconfig.hh b/src/underpassconfig.hh index 45446e14..85f7abf7 100644 --- a/src/underpassconfig.hh +++ b/src/underpassconfig.hh @@ -110,6 +110,9 @@ struct UnderpassConfig { if (yaml.contains_key("underpass_db_url")) { underpass_db_url = yamlConfig.get_value("underpass_db_url"); } + if (yaml.contains_key("bootstrap_page_size")) { + bootstrap_page_size = std::stoul(yamlConfig.get_value("bootstrap_page_size")); + } if (yaml.contains_key("planet_servers")) { std::vector planet_servers_config = yamlConfig.get_values("planet_servers"); for (auto it = planet_servers_config.begin(); it != planet_servers_config.end(); ++it) { @@ -155,6 +158,7 @@ struct UnderpassConfig { std::string datadir; std::vector planet_servers; unsigned int concurrency = 1; + unsigned int bootstrap_page_size = 100; frequency_t frequency = frequency_t::minutely; ptime start_time = not_a_date_time; ///< Starting time for changesets and OSM changes import diff --git a/src/validate/defaultvalidation.cc b/src/validate/defaultvalidation.cc index 5a95502c..b4520ad4 100644 --- a/src/validate/defaultvalidation.cc +++ b/src/validate/defaultvalidation.cc @@ -77,6 +77,27 @@ DefaultValidation::checkWay(const osmobjects::OsmWay &way, const std::string &ty return status; } +// This checks a relation. A relation should always have some tags. +std::shared_ptr +DefaultValidation::checkRelation(const osmobjects::OsmRelation &relation, const std::string &type) +{ + auto status = std::make_shared(relation); + status->timestamp = boost::posix_time::microsec_clock::universal_time(); + status->uid = relation.uid; + if (yamls.size() == 0) { + log_error("No config files!"); + return status; + } + yaml::Yaml tests = yamls[type]; + semantic::Semantic::checkRelation(relation, type, tests, status); + // geospatial::Geospatial::checkRelation(relation, type, tests, status); + // if (relation.linestring.size() > 2) { + // boost::geometry::centroid(way.linestring, status->center); + // } + status->source = type; + return status; +} + }; // namespace defaultvalidation #endif // EOF __DEFAULTVALIDATION_H__ diff --git a/src/validate/defaultvalidation.hh b/src/validate/defaultvalidation.hh index d3bf20f2..46958dea 100644 --- a/src/validate/defaultvalidation.hh +++ b/src/validate/defaultvalidation.hh @@ -55,6 +55,10 @@ public: /// is a building std::shared_ptr checkWay(const osmobjects::OsmWay &way, const std::string &type); + + /// This checks a relation. A relation should always have some tags. + std::shared_ptr checkRelation(const osmobjects::OsmRelation &relation, const std::string &type); + // Factory method static std::shared_ptr create(void) { return std::make_shared(); diff --git a/src/validate/queryvalidate.cc b/src/validate/queryvalidate.cc index 499beae6..afdd11db 100644 --- a/src/validate/queryvalidate.cc +++ b/src/validate/queryvalidate.cc @@ -117,6 +117,17 @@ QueryValidate::updateValidation(long osm_id, const valerror_t &status, const std return query; } +std::string +QueryValidate::updateValidation(long osm_id, const valerror_t &status) const +{ + std::string format = "DELETE FROM validation WHERE osm_id = %d and status = '%s';"; + boost::format fmt(format); + fmt % osm_id; + fmt % status_list[status]; + std::string query = fmt.str(); + return query; +} + std::string QueryValidate::applyChange(const ValidateStatus &validation, const valerror_t &status) const { @@ -171,6 +182,84 @@ QueryValidate::applyChange(const ValidateStatus &validation, const valerror_t &s return query; } + +void +QueryValidate::ways( + std::shared_ptr>> wayval, + std::string &task_query +) { + for (auto it = wayval->begin(); it != wayval->end(); ++it) { + if (it->get()->status.size() > 0) { + for (auto status_it = it->get()->status.begin(); status_it != it->get()->status.end(); ++status_it) { + task_query += applyChange(*it->get(), *status_it); + } + } + } +} + +void +QueryValidate::ways( + std::shared_ptr>> wayval, + std::string &task_query, + std::shared_ptr> validation_removals +) { + for (auto it = wayval->begin(); it != wayval->end(); ++it) { + if (it->get()->status.size() > 0) { + for (auto status_it = it->get()->status.begin(); status_it != it->get()->status.end(); ++status_it) { + task_query += applyChange(*it->get(), *status_it); + } + if (!it->get()->hasStatus(overlapping)) { + task_query += updateValidation(it->get()->osm_id, overlapping, "building"); + } + if (!it->get()->hasStatus(duplicate)) { + task_query += updateValidation(it->get()->osm_id, duplicate, "building"); + } + if (!it->get()->hasStatus(badgeom)) { + task_query += updateValidation(it->get()->osm_id, badgeom, "building"); + } + if (!it->get()->hasStatus(badvalue)) { + task_query += updateValidation(it->get()->osm_id, badvalue); + } + } else { + validation_removals->push_back(it->get()->osm_id); + } + } +} + +void +QueryValidate::nodes( + std::shared_ptr>> nodeval, + std::string &task_query +) { + for (auto it = nodeval->begin(); it != nodeval->end(); ++it) { + if (it->get()->status.size() > 0) { + for (auto status_it = it->get()->status.begin(); status_it != it->get()->status.end(); ++status_it) { + task_query += applyChange(*it->get(), *status_it); + } + } + } +} + +void +QueryValidate::nodes( + std::shared_ptr>> nodeval, + std::string &task_query, + std::shared_ptr> validation_removals +) { + for (auto it = nodeval->begin(); it != nodeval->end(); ++it) { + if (it->get()->status.size() > 0) { + for (auto status_it = it->get()->status.begin(); status_it != it->get()->status.end(); ++status_it) { + task_query += applyChange(*it->get(), *status_it); + } + if (!it->get()->hasStatus(badvalue)) { + task_query += updateValidation(it->get()->osm_id, badvalue); + } + } else { + validation_removals->push_back(it->get()->osm_id); + } + } +} + } // namespace queryvalidate // local Variables: diff --git a/src/validate/queryvalidate.hh b/src/validate/queryvalidate.hh index f3b16b5c..f3c3536f 100644 --- a/src/validate/queryvalidate.hh +++ b/src/validate/queryvalidate.hh @@ -78,6 +78,13 @@ class QueryValidate { /// Update the validation table, delete any feature that has been fixed. std::string updateValidation(std::shared_ptr> removals); std::string updateValidation(long osm_id, const valerror_t &status, const std::string &source) const; + std::string updateValidation(long osm_id, const valerror_t &status) const; + void ways(std::shared_ptr>> wayval, std::string &task_query); + void nodes(std::shared_ptr>> nodeval, std::string &task_query); + void rels(std::shared_ptr>> relval, std::string &task_query); + void ways(std::shared_ptr>> wayval, std::string &task_query, std::shared_ptr> validation_removals); + void nodes(std::shared_ptr>> nodeval, std::string &task_query, std::shared_ptr> validation_removals); + void rels(std::shared_ptr>> relval, std::string &task_query, std::shared_ptr> validation_removals); // Database connection, used for escape strings std::shared_ptr dbconn; }; diff --git a/src/validate/semantic.cc b/src/validate/semantic.cc index ac5a955b..91f8581c 100644 --- a/src/validate/semantic.cc +++ b/src/validate/semantic.cc @@ -116,6 +116,11 @@ bool Semantic::isRequiredTag(const std::string &key, yaml::Node required_tags) { std::shared_ptr Semantic::checkNode(const osmobjects::OsmNode &node, const std::string &type, yaml::Yaml &tests, std::shared_ptr &status) { + + auto config = tests.get("config"); + bool check_badvalue = config.get_value("badvalue") == "yes"; + bool check_incomplete = config.get_value("incomplete") == "yes"; + if (node.tags.size() == 0) { status->status.insert(notags); return status; @@ -137,18 +142,24 @@ Semantic::checkNode(const osmobjects::OsmNode &node, const std::string &type, ya if (node.tags.count(type)) { for (auto vit = std::begin(node.tags); vit != std::end(node.tags); ++vit) { - if (!isValidTag(vit->first, vit->second, tags)) { - status->status.insert(badvalue); - status->values.insert(vit->first + "=" + vit->second); + if (check_badvalue) { + if (!isValidTag(vit->first, vit->second, tags)) { + status->status.insert(badvalue); + status->values.insert(vit->first + "=" + vit->second); + } } - if (isRequiredTag(vit->first, required_tags)) { - tagexists++; + if (check_incomplete) { + if (isRequiredTag(vit->first, required_tags)) { + tagexists++; + } } checkTag(vit->first, vit->second, status); } - if (tagexists != required_tags.children.size()) { - status->status.insert(incomplete); + if (check_incomplete) { + if (tagexists != required_tags.children.size()) { + status->status.insert(incomplete); + } } } return status; @@ -177,7 +188,7 @@ Semantic::checkWay(const osmobjects::OsmWay &way, const std::string &type, yaml: // List of valid tags to be validated yaml::Node tags; if (check_badvalue) { - tags = tests.get("tags"); + tags = tests.get("tags"); } if (check_badvalue && way.tags.size() == 0) { @@ -209,6 +220,60 @@ Semantic::checkWay(const osmobjects::OsmWay &way, const std::string &type, yaml: return status; } +// This checks a relation. +std::shared_ptr +Semantic::checkRelation(const osmobjects::OsmRelation &relation, const std::string &type, yaml::Yaml &tests, std::shared_ptr &status) +{ + if (relation.action == osmobjects::remove) { + return status; + } + + // These values are in the config section of the YAML file + auto config = tests.get("config"); + bool check_badvalue = config.get_value("badvalue") == "yes"; + bool check_incomplete = config.get_value("incomplete") == "yes"; + + // List of required tags to be validated + yaml::Node required_tags; + if (check_incomplete) { + required_tags = tests.get("required_tags"); + } + + // List of valid tags to be validated + yaml::Node tags; + if (check_badvalue) { + tags = tests.get("tags"); + } + + if (check_badvalue && relation.tags.size() == 0) { + status->status.insert(notags); + return status; + } + + int tagexists = 0; + if (relation.tags.count(type)) { + for (auto vit = std::begin(relation.tags); vit != std::end(relation.tags); ++vit) { + if (check_badvalue) { + if (tags.children.size() > 0 && !isValidTag(vit->first, vit->second, tags)) { + status->status.insert(badvalue); + status->values.insert(vit->first + "=" + vit->second); + } + checkTag(vit->first, vit->second, status); + } + if (check_incomplete) { + if (isRequiredTag(vit->first, required_tags)) { + tagexists++; + } + } + } + + if (check_incomplete && tagexists != required_tags.children.size()) { + status->status.insert(incomplete); + } + } + return status; +} + }; // namespace semantic #endif // EOF __SEMANTIC_H__ diff --git a/src/validate/semantic.hh b/src/validate/semantic.hh index 14f93e11..7dfe3a85 100644 --- a/src/validate/semantic.hh +++ b/src/validate/semantic.hh @@ -57,6 +57,7 @@ public: ~Semantic(void) { }; static std::shared_ptr checkNode(const osmobjects::OsmNode &node, const std::string &type, yaml::Yaml &tests, std::shared_ptr &status); static std::shared_ptr checkWay(const osmobjects::OsmWay &way, const std::string &type, yaml::Yaml &tests, std::shared_ptr &status); + static std::shared_ptr checkRelation(const osmobjects::OsmRelation &relation, const std::string &type, yaml::Yaml &tests, std::shared_ptr &status); private: static bool isValidTag(const std::string &key, const std::string &value, yaml::Node tags); static bool isRequiredTag(const std::string &key, yaml::Node required_tags); diff --git a/src/validate/validate.hh b/src/validate/validate.hh index e8d51e0e..d3ed80f2 100644 --- a/src/validate/validate.hh +++ b/src/validate/validate.hh @@ -118,6 +118,14 @@ class ValidateStatus { version = way.version; timestamp = way.timestamp; } + ValidateStatus(const osmobjects::OsmRelation &relation) { + osm_id = relation.id; + uid = relation.uid; + changeset = relation.changeset; + objtype = osmobjects::relation; + version = relation.version; + timestamp = relation.timestamp; + } /// Does this change have a particular status value bool hasStatus(const valerror_t &val) const { auto match = std::find(status.begin(), status.end(), val);