Merge branch 'master' of github.com:hotosm/underpass

python/dbapi/api/stats.py

@@ -49,7 +49,7 @@ async def getCount(
             count_validated_features as ( \
                 select count(distinct(all_features.osm_id)) as count from all_features \
                 left join validation v on all_features.osm_id = v.osm_id \
-                where v.status = 'badgeom' \
+                where v.status = '{6}' \
             ), count_features as (\
                 select count(distinct(all_features.osm_id)) as total from all_features \
             ) \
@@ -60,7 +60,7 @@ async def getCount(
                 "AND ST_Intersects(\"geom\", ST_GeomFromText('MULTIPOLYGON((({0})))', 4326) )".format(area) if area else "",
                 "AND (" + tagsQueryFilter(tags, table) + ")" if tags else "",
                 "AND " + hashtagQueryFilter(hashtag, table) if hashtag else "",
-                "AND status = '" + status + "'" if status else "",
+                status
             )
         else:
            query = "select count(distinct {0}.osm_id) as count from {0} \

src/bootstrap/bootstrap.cc

@@ -165,10 +165,8 @@ threadBootstrapTask(WayTask wayTask)
 
     // Proccesing ways
     for (int i = 0; i < PAGE_SIZE; ++i) {
-        // std::cout << "--> processing way " << i * taskIndex << std::endl;
         if (i * taskIndex < ways->size()) {
             auto way = ways->at(i * (taskIndex + 1));
-            // std::cout << "[task " << taskIndex << "] way " << i * (taskIndex + 1) << std::endl;
             auto status = plugin->checkWay(way, "building");
             for (auto status_it = status->status.begin(); status_it != status->status.end(); ++status_it) {
                 task.query += queryvalidate->applyChange(*status, *status_it);

src/data/pq.cc

@@ -157,44 +157,24 @@ Pq::query(const std::string &query)
     return result;
 }
 
-std::string Latin1ToUTF8(const std::string& latin1str) {
-    std::string utf8str;
-    for (char c : latin1str) {
-        if (static_cast<unsigned char>(c) <= 0x7F) {
-            utf8str.push_back(c);
-        } else {
-            utf8str.push_back(0xC0 | static_cast<unsigned char>(c) >> 6);
-            utf8str.push_back(0x80 | (static_cast<unsigned char>(c) & 0x3F));
-        }
-    }
-    return utf8str;
-}
-
 std::string
 Pq::escapedString(const std::string &s)
 {
     std::string newstr;
     int i = 0;
     while (i < s.size()) {
+        // Single quote (')
         if (s[i] == '\'') {
-            newstr += "&apos;";
-        } else if (s[i] == '\"') {
-            newstr += "&quot;";
-        } else if (s[i] == '\'') {
-            newstr += "&quot;";
-        } else if (s[i] == ')') {
-            newstr += "&#41;";
-        } else if (s[i] == '(') {
-            newstr += "&#40;";
-        } else if ((s[i] == '\\') || (s[i] == '\n')) {
-            // drop this character
+            newstr += "''";
+        // Slash (\)
+        } else if (s[i] == '\\') {
+            newstr += "\\\\";
         } else {
             newstr += s[i];
         }
         i++;
     }
-
-    return sdb->esc(Latin1ToUTF8(newstr));
+    return sdb->esc(newstr);
 }
 
 std::string 

src/raw/queryraw.cc

@@ -62,6 +62,32 @@ QueryRaw::QueryRaw(std::shared_ptr<Pq> db) {
     dbconn = db;
 }
 
+std::string
+QueryRaw::buildTagsQuery(std::map<std::string, std::string> tags) const {
+    if (tags.size() > 0) {
+        std::string tagsStr = "jsonb_build_object(";
+        int count = 0;
+        for (auto it = std::begin(tags); it != std::end(tags); ++it) {
+            ++count;
+            // PostgreSQL has an argument limit for functions
+            if (count == 50) {
+                tagsStr.erase(tagsStr.size() - 1);
+                tagsStr += ") || jsonb_build_object(";
+                count = 0;
+            }
+            std::string tag_format = "'%s', '%s',";
+            boost::format tag_fmt(tag_format);
+            tag_fmt % dbconn->escapedString(dbconn->escapedJSON(it->first));
+            tag_fmt % dbconn->escapedString(dbconn->escapedJSON(it->second));
+            tagsStr += tag_fmt.str();
+        }
+        tagsStr.erase(tagsStr.size() - 1);
+        return tagsStr + ")";
+    } else {
+        return "null";
+    }
+}
+
 std::string
 QueryRaw::applyChange(const OsmNode &node) const
 {
@@ -83,21 +109,7 @@ QueryRaw::applyChange(const OsmNode &node) const
         fmt % geometry;
 
         // tags
-        std::string tags = "";
-        if (node.tags.size() > 0) {
-            for (auto it = std::begin(node.tags); it != std::end(node.tags); ++it) {
-                std::string tag_format = "\"%s\" : \"%s\",";
-                boost::format tag_fmt(tag_format);
-                tag_fmt % dbconn->escapedString(dbconn->escapedJSON(it->first));
-                tag_fmt % dbconn->escapedString(dbconn->escapedJSON(it->second));
-                tags += tag_fmt.str();
-            }
-            tags.erase(tags.size() - 1);
-            tags = "'{" + tags + "}'";
-
-        } else {
-            tags = "null";
-        }
+        auto tags = buildTagsQuery(node.tags);
         fmt % tags;
         // timestamp
         std::string timestamp = to_simple_string(boost::posix_time::microsec_clock::universal_time());
@@ -164,22 +176,8 @@ QueryRaw::applyChange(const OsmWay &way) const
             // osm_id
             fmt % way.id;
 
-            // tags
-            std::string tags = "";
-            if (way.tags.size() > 0) {
-                for (auto it = std::begin(way.tags); it != std::end(way.tags); ++it) {
-                    std::string tag_format = "\"%s\" : \"%s\",";
-                    boost::format tag_fmt(tag_format);
-                    tag_fmt % dbconn->escapedString(dbconn->escapedJSON(it->first));
-                    tag_fmt % dbconn->escapedString(dbconn->escapedJSON(it->second));
-                    tags += tag_fmt.str();
-                }
-                tags.erase(tags.size() - 1);
-                tags = "'{" + tags + "}'";
-            } else {
-                tags = "null";
-            }
-
+            //tags
+            auto tags = buildTagsQuery(way.tags);
             fmt % tags;
 
             // refs

src/raw/queryraw.hh

@@ -33,6 +33,7 @@
 #endif
 
 #include <iostream>
+#include <map>
 #include "data/pq.hh"
 #include "osm/osmobjects.hh"
 #include "osm/osmchange.hh"
@@ -75,6 +76,8 @@ class QueryRaw {
     std::shared_ptr<Pq> dbconn;
     // Get ways count
     int getWaysCount(const std::string &tableName);
+    // Build tags query
+    std::string buildTagsQuery(std::map<std::string, std::string> tags) const;
     // Get ways by page
     std::shared_ptr<std::vector<OsmWay>> getWaysFromDB(long lastid, int pageSize, const std::string &tableName);
     std::shared_ptr<std::vector<OsmWay>> getWaysFromDBWithoutRefs(long lastid, int pageSize, const std::string &tableName);

src/stats/querystats.cc

@@ -71,7 +71,6 @@ QueryStats::applyChange(const osmchange::ChangeStats &change) const
 #ifdef TIMING_DEBUG_X
     boost::timer::auto_cpu_timer timer("applyChange(statistics): took %w seconds\n");
 #endif
-    // std::cout << "Applying OsmChange data" << std::endl;
 
     if (change.closed_at != not_a_date_time) {
 

src/underpass.cc

@@ -349,7 +349,6 @@ main(int argc, char *argv[])
     std::cout << "the environment variable names and their current "
                     "values (possibly defaults)."
                 << std::endl;
-    // std::cout << config.dbConfigHelp() << std::endl;
     exit(0);
 
 }

src/validate/semantic.cc

@@ -86,6 +86,12 @@ Semantic::checkTag(const std::string &key, const std::string &value, std::shared
         status->status.insert(badvalue);
         status->values.insert(key + "=" + value);
     }
+    // Check for a underscore at the beginning of the tag key
+    if(key.at(0) == '_') {
+        log_error("WARNING: underscore at the beginning of the tag key \"%1%\"", key);
+        status->status.insert(badvalue);
+        status->values.insert(key + "=" + value);
+    }
 }
 
 bool Semantic::isValidTag(const std::string &key, const std::string &value, yaml::Node tags) {