From fb9e6afff642fe24e891606104307f2681da1475 Mon Sep 17 00:00:00 2001 From: Emillio Mariscal Date: Wed, 21 Feb 2024 18:28:54 -0300 Subject: [PATCH 1/5] Fixes for special characters, less verbosity --- src/bootstrap/bootstrap.cc | 2 -- src/data/pq.cc | 32 ++++++-------------------------- src/raw/queryraw.cc | 8 ++++---- src/stats/querystats.cc | 1 - src/underpass.cc | 1 - src/validate/semantic.cc | 6 ++++++ 6 files changed, 16 insertions(+), 34 deletions(-) diff --git a/src/bootstrap/bootstrap.cc b/src/bootstrap/bootstrap.cc index 4d1710c6..13b4c6b1 100644 --- a/src/bootstrap/bootstrap.cc +++ b/src/bootstrap/bootstrap.cc @@ -165,10 +165,8 @@ threadBootstrapTask(WayTask wayTask) // Proccesing ways for (int i = 0; i < PAGE_SIZE; ++i) { - // std::cout << "--> processing way " << i * taskIndex << std::endl; if (i * taskIndex < ways->size()) { auto way = ways->at(i * (taskIndex + 1)); - // std::cout << "[task " << taskIndex << "] way " << i * (taskIndex + 1) << std::endl; auto status = plugin->checkWay(way, "building"); for (auto status_it = status->status.begin(); status_it != status->status.end(); ++status_it) { task.query += queryvalidate->applyChange(*status, *status_it); diff --git a/src/data/pq.cc b/src/data/pq.cc index 5f5f9bf1..2125ac74 100644 --- a/src/data/pq.cc +++ b/src/data/pq.cc @@ -157,44 +157,24 @@ Pq::query(const std::string &query) return result; } -std::string Latin1ToUTF8(const std::string& latin1str) { - std::string utf8str; - for (char c : latin1str) { - if (static_cast(c) <= 0x7F) { - utf8str.push_back(c); - } else { - utf8str.push_back(0xC0 | static_cast(c) >> 6); - utf8str.push_back(0x80 | (static_cast(c) & 0x3F)); - } - } - return utf8str; -} - std::string Pq::escapedString(const std::string &s) { std::string newstr; int i = 0; while (i < s.size()) { + // Single quote (') if (s[i] == '\'') { - newstr += "'"; - } else if (s[i] == '\"') { - newstr += """; - } else if (s[i] == '\'') { - newstr += """; - } else if (s[i] == ')') { - newstr += ")"; - } else if (s[i] == '(') { - newstr += "("; - } else if ((s[i] == '\\') || (s[i] == '\n')) { - // drop this character + newstr += "''"; + // Slash (\) + } else if (s[i] == '\\') { + newstr += "\\\\"; } else { newstr += s[i]; } i++; } - - return sdb->esc(Latin1ToUTF8(newstr)); + return sdb->esc(newstr); } std::string diff --git a/src/raw/queryraw.cc b/src/raw/queryraw.cc index 397d303d..80b8f9d2 100644 --- a/src/raw/queryraw.cc +++ b/src/raw/queryraw.cc @@ -86,14 +86,14 @@ QueryRaw::applyChange(const OsmNode &node) const std::string tags = ""; if (node.tags.size() > 0) { for (auto it = std::begin(node.tags); it != std::end(node.tags); ++it) { - std::string tag_format = "\"%s\" : \"%s\","; + std::string tag_format = "'%s', '%s',"; boost::format tag_fmt(tag_format); tag_fmt % dbconn->escapedString(dbconn->escapedJSON(it->first)); tag_fmt % dbconn->escapedString(dbconn->escapedJSON(it->second)); tags += tag_fmt.str(); } tags.erase(tags.size() - 1); - tags = "'{" + tags + "}'"; + tags = "jsonb_build_object(" + tags + ")"; } else { tags = "null"; @@ -168,14 +168,14 @@ QueryRaw::applyChange(const OsmWay &way) const std::string tags = ""; if (way.tags.size() > 0) { for (auto it = std::begin(way.tags); it != std::end(way.tags); ++it) { - std::string tag_format = "\"%s\" : \"%s\","; + std::string tag_format = "'%s', '%s',"; boost::format tag_fmt(tag_format); tag_fmt % dbconn->escapedString(dbconn->escapedJSON(it->first)); tag_fmt % dbconn->escapedString(dbconn->escapedJSON(it->second)); tags += tag_fmt.str(); } tags.erase(tags.size() - 1); - tags = "'{" + tags + "}'"; + tags = "jsonb_build_object(" + tags + ")"; } else { tags = "null"; } diff --git a/src/stats/querystats.cc b/src/stats/querystats.cc index 21011723..30829923 100644 --- a/src/stats/querystats.cc +++ b/src/stats/querystats.cc @@ -71,7 +71,6 @@ QueryStats::applyChange(const osmchange::ChangeStats &change) const #ifdef TIMING_DEBUG_X boost::timer::auto_cpu_timer timer("applyChange(statistics): took %w seconds\n"); #endif - // std::cout << "Applying OsmChange data" << std::endl; if (change.closed_at != not_a_date_time) { diff --git a/src/underpass.cc b/src/underpass.cc index e4028ae8..9a5f4977 100644 --- a/src/underpass.cc +++ b/src/underpass.cc @@ -349,7 +349,6 @@ main(int argc, char *argv[]) std::cout << "the environment variable names and their current " "values (possibly defaults)." << std::endl; - // std::cout << config.dbConfigHelp() << std::endl; exit(0); } diff --git a/src/validate/semantic.cc b/src/validate/semantic.cc index 4ea26dae..ac5a955b 100644 --- a/src/validate/semantic.cc +++ b/src/validate/semantic.cc @@ -86,6 +86,12 @@ Semantic::checkTag(const std::string &key, const std::string &value, std::shared status->status.insert(badvalue); status->values.insert(key + "=" + value); } + // Check for a underscore at the beginning of the tag key + if(key.at(0) == '_') { + log_error("WARNING: underscore at the beginning of the tag key \"%1%\"", key); + status->status.insert(badvalue); + status->values.insert(key + "=" + value); + } } bool Semantic::isValidTag(const std::string &key, const std::string &value, yaml::Node tags) { From b27e7586acb1bb5f1c2b09bdd1988ae8ac781c1a Mon Sep 17 00:00:00 2001 From: Emillio Mariscal Date: Wed, 21 Feb 2024 18:37:09 -0300 Subject: [PATCH 2/5] 0755 mode for service scripts --- setup/service/restart.sh | 0 setup/service/start.sh | 0 2 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 setup/service/restart.sh mode change 100644 => 100755 setup/service/start.sh diff --git a/setup/service/restart.sh b/setup/service/restart.sh old mode 100644 new mode 100755 diff --git a/setup/service/start.sh b/setup/service/start.sh old mode 100644 new mode 100755 From 15cfd88bb293942a1d4b15aaba44c13651b18742 Mon Sep 17 00:00:00 2001 From: Emillio Mariscal Date: Wed, 21 Feb 2024 19:43:29 -0300 Subject: [PATCH 3/5] Fix for function 100 parameter limit in PostgreSQL --- src/raw/queryraw.cc | 58 +++++++++++++++++++++------------------------ src/raw/queryraw.hh | 3 +++ 2 files changed, 30 insertions(+), 31 deletions(-) diff --git a/src/raw/queryraw.cc b/src/raw/queryraw.cc index 80b8f9d2..e84df51e 100644 --- a/src/raw/queryraw.cc +++ b/src/raw/queryraw.cc @@ -62,6 +62,30 @@ QueryRaw::QueryRaw(std::shared_ptr db) { dbconn = db; } +std::string +QueryRaw::buildTagsQuery(std::map tags) const { + if (tags.size() > 0) { + std::string tagsStr = "jsonb_build_object("; + int count = 0; + for (auto it = std::begin(tags); it != std::end(tags); ++it) { + std::string tag_format = "'%s', '%s',"; + boost::format tag_fmt(tag_format); + tag_fmt % dbconn->escapedString(dbconn->escapedJSON(it->first)); + tag_fmt % dbconn->escapedString(dbconn->escapedJSON(it->second)); + tagsStr += tag_fmt.str(); + ++count; + if (count == 101) { + tagsStr.erase(tagsStr.size() - 1); + tagsStr += ") || jsonb_build_object("; + } + } + tagsStr.erase(tagsStr.size() - 1); + return tagsStr + ")"; + } else { + return "null"; + } +} + std::string QueryRaw::applyChange(const OsmNode &node) const { @@ -83,21 +107,7 @@ QueryRaw::applyChange(const OsmNode &node) const fmt % geometry; // tags - std::string tags = ""; - if (node.tags.size() > 0) { - for (auto it = std::begin(node.tags); it != std::end(node.tags); ++it) { - std::string tag_format = "'%s', '%s',"; - boost::format tag_fmt(tag_format); - tag_fmt % dbconn->escapedString(dbconn->escapedJSON(it->first)); - tag_fmt % dbconn->escapedString(dbconn->escapedJSON(it->second)); - tags += tag_fmt.str(); - } - tags.erase(tags.size() - 1); - tags = "jsonb_build_object(" + tags + ")"; - - } else { - tags = "null"; - } + auto tags = buildTagsQuery(node.tags); fmt % tags; // timestamp std::string timestamp = to_simple_string(boost::posix_time::microsec_clock::universal_time()); @@ -164,22 +174,8 @@ QueryRaw::applyChange(const OsmWay &way) const // osm_id fmt % way.id; - // tags - std::string tags = ""; - if (way.tags.size() > 0) { - for (auto it = std::begin(way.tags); it != std::end(way.tags); ++it) { - std::string tag_format = "'%s', '%s',"; - boost::format tag_fmt(tag_format); - tag_fmt % dbconn->escapedString(dbconn->escapedJSON(it->first)); - tag_fmt % dbconn->escapedString(dbconn->escapedJSON(it->second)); - tags += tag_fmt.str(); - } - tags.erase(tags.size() - 1); - tags = "jsonb_build_object(" + tags + ")"; - } else { - tags = "null"; - } - + //tags + auto tags = buildTagsQuery(way.tags); fmt % tags; // refs diff --git a/src/raw/queryraw.hh b/src/raw/queryraw.hh index 9a4820b3..20c99eda 100644 --- a/src/raw/queryraw.hh +++ b/src/raw/queryraw.hh @@ -33,6 +33,7 @@ #endif #include +#include #include "data/pq.hh" #include "osm/osmobjects.hh" #include "osm/osmchange.hh" @@ -75,6 +76,8 @@ class QueryRaw { std::shared_ptr dbconn; // Get ways count int getWaysCount(const std::string &tableName); + // Build tags query + std::string buildTagsQuery(std::map tags) const; // Get ways by page std::shared_ptr> getWaysFromDB(long lastid, int pageSize, const std::string &tableName); std::shared_ptr> getWaysFromDBWithoutRefs(long lastid, int pageSize, const std::string &tableName); From 08fb6edc57250894b5d76602aa2a539bb705b216 Mon Sep 17 00:00:00 2001 From: Emillio Mariscal Date: Wed, 21 Feb 2024 20:13:10 -0300 Subject: [PATCH 4/5] Fix for 100 parameter limit for PostgreSQL functions --- src/raw/queryraw.cc | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/raw/queryraw.cc b/src/raw/queryraw.cc index e84df51e..c75738ab 100644 --- a/src/raw/queryraw.cc +++ b/src/raw/queryraw.cc @@ -68,16 +68,18 @@ QueryRaw::buildTagsQuery(std::map tags) const { std::string tagsStr = "jsonb_build_object("; int count = 0; for (auto it = std::begin(tags); it != std::end(tags); ++it) { + ++count; + // PostgreSQL has an argument limit for functions + if (count == 50) { + tagsStr.erase(tagsStr.size() - 1); + tagsStr += ") || jsonb_build_object("; + count = 0; + } std::string tag_format = "'%s', '%s',"; boost::format tag_fmt(tag_format); tag_fmt % dbconn->escapedString(dbconn->escapedJSON(it->first)); tag_fmt % dbconn->escapedString(dbconn->escapedJSON(it->second)); tagsStr += tag_fmt.str(); - ++count; - if (count == 101) { - tagsStr.erase(tagsStr.size() - 1); - tagsStr += ") || jsonb_build_object("; - } } tagsStr.erase(tagsStr.size() - 1); return tagsStr + ")"; From 748286f60e740042dbe9eb52ef517452a1af5778 Mon Sep 17 00:00:00 2001 From: Emillio Mariscal Date: Wed, 21 Feb 2024 20:29:20 -0300 Subject: [PATCH 5/5] Fix for stats query --- python/dbapi/api/stats.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/dbapi/api/stats.py b/python/dbapi/api/stats.py index d8b88a84..8338f2d7 100644 --- a/python/dbapi/api/stats.py +++ b/python/dbapi/api/stats.py @@ -49,7 +49,7 @@ async def getCount( count_validated_features as ( \ select count(distinct(all_features.osm_id)) as count from all_features \ left join validation v on all_features.osm_id = v.osm_id \ - where v.status = 'badgeom' \ + where v.status = '{6}' \ ), count_features as (\ select count(distinct(all_features.osm_id)) as total from all_features \ ) \ @@ -60,7 +60,7 @@ async def getCount( "AND ST_Intersects(\"geom\", ST_GeomFromText('MULTIPOLYGON((({0})))', 4326) )".format(area) if area else "", "AND (" + tagsQueryFilter(tags, table) + ")" if tags else "", "AND " + hashtagQueryFilter(hashtag, table) if hashtag else "", - "AND status = '" + status + "'" if status else "", + status ) else: query = "select count(distinct {0}.osm_id) as count from {0} \