From b8c69afd2f05e3f50ff0eb280e2aab926a87352d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Rodrigo?= Date: Sat, 31 Aug 2019 21:40:59 +0200 Subject: [PATCH 01/11] Avoid duplicate issues in analyser_osmosis_powerline #560 --- analysers/analyser_osmosis_powerline.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/analysers/analyser_osmosis_powerline.py b/analysers/analyser_osmosis_powerline.py index e76051cbb..9184d418e 100644 --- a/analysers/analyser_osmosis_powerline.py +++ b/analysers/analyser_osmosis_powerline.py @@ -295,7 +295,7 @@ """ sql60 = """ -SELECT +SELECT DISTINCT ON (line_ends1.wid) line_ends1.wid, line_terminators.type_id, ST_AsText(line_ends1.geom) @@ -307,6 +307,8 @@ line_terminators.power = 'substation' AND (line_terminators.substation IS NULL OR line_terminators.substation != 'minor_distribution') AND NOT line_ends1.voltage <@ line_terminators.voltage +ORDER BY + line_ends1.wid """ From 8c095b19cfc95e28b4b0ad8cce18999c4e58502c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Rodrigo?= Date: Sat, 31 Aug 2019 22:33:18 +0200 Subject: [PATCH 02/11] Avoid duplicate error from analyser_osmosis_boundary_intersect #560 --- .../analyser_osmosis_boundary_intersect.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/analysers/analyser_osmosis_boundary_intersect.py b/analysers/analyser_osmosis_boundary_intersect.py index 15444e0d6..46294b33b 100644 --- a/analysers/analyser_osmosis_boundary_intersect.py +++ b/analysers/analyser_osmosis_boundary_intersect.py @@ -24,6 +24,11 @@ sql10 = """ CREATE TEMP TABLE boundary AS +SELECT + id, + linestring, + array_agg(boundary) AS boundaries +FROM ( ( SELECT ways.id, @@ -41,7 +46,7 @@ relations.tags->'type' = 'boundary' AND relations.tags?'boundary' ) -UNION +UNION ALL ( SELECT ways.id, @@ -55,10 +60,10 @@ tags->'type' = 'boundary' AND tags?'boundary' ) -""" - -sql11 = """ -CREATE INDEX boundary_boundary ON boundary(boundary) +) AS t +GROUP BY + id, + linestring """ sql12 = """ @@ -73,7 +78,7 @@ FROM boundary AS b1 JOIN boundary AS b2 ON - b1.boundary = b2.boundary AND + b1.boundaries && b2.boundaries AND b1.linestring && b2.linestring AND b1.id < b2.id AND -- Ways not linked @@ -91,6 +96,5 @@ def __init__(self, config, logger = None): def analyser_osmosis_common(self): self.run(sql10) - self.run(sql11) self.run(sql12) self.run(sql20, self.callback20) From c9b0478d10ffc8b4815115cc2daac4d514c7267d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Rodrigo?= Date: Sat, 31 Aug 2019 23:16:44 +0200 Subject: [PATCH 03/11] Base subclass on node index #560 --- analysers/analyser_osmosis_way_approximate.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/analysers/analyser_osmosis_way_approximate.py b/analysers/analyser_osmosis_way_approximate.py index dc84416f3..76b45d801 100644 --- a/analysers/analyser_osmosis_way_approximate.py +++ b/analysers/analyser_osmosis_way_approximate.py @@ -80,7 +80,8 @@ ) ) AS d, type, - {3} + {3}, + index FROM ( SELECT id, @@ -123,7 +124,7 @@ def __init__(self, config, logger = None): ) for t in self.tags: self.classs_change[t[0]] = {"item":"1190", "level": 3, "tag": ["geom", "highway", "railway", "fix:imagery"], "desc": T_f(u"Approximate {0}", t[1]) } - self.callback10 = lambda res: {"class":res[4], "subclass":stablehash(res[1]), "data":[self.way_full, self.positionAsText], "text": T_f(u"{0} deviation of {1}m", res[3], res[2])} + self.callback10 = lambda res: {"class":res[4], "subclass":res[5], "data":[self.way_full, self.positionAsText], "text": T_f(u"{0} deviation of {1}m", res[3], res[2])} def analyser_osmosis_full(self): self.run(sql10) From 4614d780b0d3c1d447c95410377e23cd407bd804 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Rodrigo?= Date: Sun, 1 Sep 2019 13:38:37 +0200 Subject: [PATCH 04/11] Avoid duplicate issues in analyser_osmosis_relation_public_transport #560 --- .../analyser_osmosis_relation_public_transport.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/analysers/analyser_osmosis_relation_public_transport.py b/analysers/analyser_osmosis_relation_public_transport.py index f92f21357..33d467f86 100644 --- a/analysers/analyser_osmosis_relation_public_transport.py +++ b/analysers/analyser_osmosis_relation_public_transport.py @@ -113,7 +113,7 @@ sql20 = """ CREATE TEMP TABLE stop_platform AS ( -SELECT +SELECT DISTINCT ON (relations.id, ways.id) relations.id, relation_members.member_type, ways.id AS mid, @@ -130,8 +130,11 @@ relations.tags->'type' = 'route' AND relations.tags->'route' IN ('train', 'subway', 'monorail', 'tram', 'bus', 'trolleybus', 'aerialway', 'ferry', 'coach', 'funicular', 'share_taxi', 'light_rail', 'school_bus') AND (NOT relations.tags?(relations.tags->'route') OR relations.tags->(relations.tags->'route') != 'on_demand') +ORDER BY + relations.id, + ways.id ) UNION ALL ( -SELECT +SELECT DISTINCT ON (relations.id, nodes.id) relations.id, relation_members.member_type, nodes.id AS mid, @@ -147,7 +150,10 @@ WHERE relations.tags->'type' = 'route' AND relations.tags->'route' IN ('train', 'subway', 'monorail', 'tram', 'bus', 'trolleybus', 'aerialway', 'ferry', 'coach', 'funicular', 'share_taxi', 'light_rail', 'school_bus') AND - (NOT relations.tags?(relations.tags->'route') OR relations.tags->(relations.tags->'route') != 'on_demand') + (NOT relations.tags?(relations.tags->'route') OR relations.tags->(relations.tags->'route') != 'on_demand') +ORDER BY + relations.id, + nodes.id ) """ From bb720f376e02a31fc3fe30f5e0f54e6d53fe25a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Rodrigo?= Date: Sun, 1 Sep 2019 14:08:53 +0200 Subject: [PATCH 05/11] Avoid duplicated issues in analyser_osmosis_relation_associatedStreet #560 --- analysers/analyser_osmosis_relation_associatedStreet.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/analysers/analyser_osmosis_relation_associatedStreet.py b/analysers/analyser_osmosis_relation_associatedStreet.py index 7638b72bc..eb1af9a7f 100644 --- a/analysers/analyser_osmosis_relation_associatedStreet.py +++ b/analysers/analyser_osmosis_relation_associatedStreet.py @@ -82,7 +82,7 @@ # ways with addr:housenumber or addr:housename and without addr:street and not member of a associatedStreet sql10 = """ -SELECT +SELECT DISTINCT ON (id) id, ST_AsText(ST_Transform(way_locate(linestring_proj), 4326)) FROM @@ -90,11 +90,13 @@ WHERE name IS NULL AND rid IS NULL +ORDER BY + id """ # same for nodes sql11 = """ -SELECT +SELECT DISTINCT ON (id) id, ST_AsText(ST_Transform(geom_proj, 4326)) FROM @@ -102,6 +104,8 @@ WHERE name IS NULL AND rid IS NULL +ORDER BY + id """ # No role street in relation From 4c4a29651b64c3309a1ce9629a7a7b2531e825ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Rodrigo?= Date: Sun, 1 Sep 2019 15:08:55 +0200 Subject: [PATCH 06/11] Avoid duplicate issues in Highway_Lanes #360 --- plugins/Highway_Lanes.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/plugins/Highway_Lanes.py b/plugins/Highway_Lanes.py index af4d9fb46..a7d1c946e 100644 --- a/plugins/Highway_Lanes.py +++ b/plugins/Highway_Lanes.py @@ -69,12 +69,12 @@ def way(self, data, tags, nds): unknown = False i = 0 for tt in ttt: - for t in tt.split(";"): + for t in set(tt.split(";")): if t not in ["left", "slight_left", "sharp_left", "through", "right", "slight_right", "sharp_right", "reverse", "merge_to_left", "merge_to_right", "none", ""]: unknown = True - err.append({"class": 31606, "subclass": 0 + stablehash(tags_lanes[tl]), "text": T_f(u"Unknown turn lanes value \"{0}\"", t)}) + err.append({"class": 31606, "subclass": 0 + stablehash(tl + '|' + tt + '|' + str(i)), "text": T_f(u"Unknown turn lanes value \"{0}\"", t)}) if (t == "merge_to_left" and i == 0) or (t == "merge_to_right" and i == len(ttt) - 1): - err.append({"class": 31600, "subclass": 1 + stablehash(tags_lanes[tl])}) + err.append({"class": 31600, "subclass": 1 + stablehash(tl + '|' + tt + '|' + str(i))}) i += 1 if not unknown: # merge_to_left is a on the right and vice versa @@ -97,7 +97,7 @@ def way(self, data, tags, nds): (first_space == None or last_space == None or first_space <= last_space) and (last_space == None or first_right == None or last_space < first_right) and (last_left == None or first_right == None or last_left < first_right)): - err.append({"class": 31607, "subclass": 1 + stablehash(tags_lanes[tl])}) + err.append({"class": 31607, "subclass": 1 + stablehash(tl)}) # Check acces lanes values From 8565940e8155b7c7a3874bf34dd8001338a92717 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Rodrigo?= Date: Sun, 1 Sep 2019 19:27:47 +0200 Subject: [PATCH 07/11] Avoid duplicate issues in Analyser_Merge #560 --- analysers/Analyser_Merge.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/analysers/Analyser_Merge.py b/analysers/Analyser_Merge.py index 73501cbb7..ea4ef726f 100644 --- a/analysers/Analyser_Merge.py +++ b/analysers/Analyser_Merge.py @@ -193,7 +193,8 @@ ELSE ST_AsText(any_locate(osm_item.type, osm_item.id)) END, osm_item.tags, - osm_item.geom + osm_item.geom, + osm_item.ref FROM osm_item LEFT JOIN %(official)s AS official ON @@ -944,6 +945,7 @@ def analyser_osmosis_common(self): # Invalid OSM self.run(sql23 % {"official": table, "joinClause": joinClause}, lambda res: { "class": self.missing_osm["class"], + "subclass": str(stablehash(res[4])) if self.mapping.osmRef != "NULL" else None, "data": [self.typeMapping[res[1]], None, self.positionAsText] } ) From 9772045b7118ab8f4459403431a2b76420902e1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Rodrigo?= Date: Sun, 1 Sep 2019 22:50:17 +0200 Subject: [PATCH 08/11] Avoid duplicate issues in analyser_osmosis_building_overlaps #560 --- analysers/analyser_osmosis_building_overlaps.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/analysers/analyser_osmosis_building_overlaps.py b/analysers/analyser_osmosis_building_overlaps.py index d33f696d4..5f2ba5b43 100644 --- a/analysers/analyser_osmosis_building_overlaps.py +++ b/analysers/analyser_osmosis_building_overlaps.py @@ -22,6 +22,7 @@ ########################################################################### from .Analyser_Osmosis import Analyser_Osmosis +from modules.Stablehash import stablehash sql20 = """ CREATE TEMP TABLE bnodes AS @@ -111,7 +112,7 @@ """ sql60 = """ -SELECT DISTINCT ON(ST_Centroid(geom)) +SELECT ST_AsText(ST_Transform(ST_Centroid(geom), 4326)), ST_Area(geom) FROM @@ -167,7 +168,7 @@ def __init__(self, config, logger = None): self.callback30 = lambda res: {"class":2 if res[3]>res[4] else 1, "data":[self.way, self.way, self.positionAsText]} self.callback40 = lambda res: {"class":3, "data":[self.way, self.positionAsText]} self.callback50 = lambda res: {"class":4, "data":[self.way, self.way, self.positionAsText]} - self.callback60 = lambda res: {"class":5, "data":[self.positionAsText]} + self.callback60 = lambda res: {"class":5, "subclass": stablehash(res[0]), "data":[self.positionAsText]} if self.FR: self.callback70 = lambda res: {"class":6, "data":[self.way, self.positionAsText]} From e4040a41eab1412abdd11009a9c77e5913b47645 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Rodrigo?= Date: Mon, 2 Sep 2019 23:00:13 +0200 Subject: [PATCH 09/11] Avoid duplicate in analyser_merge_geodesie #560 --- analysers/analyser_merge_geodesie.py | 1 + 1 file changed, 1 insertion(+) diff --git a/analysers/analyser_merge_geodesie.py b/analysers/analyser_merge_geodesie.py index 9d03e01c4..8026419cb 100644 --- a/analysers/analyser_merge_geodesie.py +++ b/analysers/analyser_merge_geodesie.py @@ -34,6 +34,7 @@ def __init__(self, config, logger = None): file = "geodesie.csv.bz2"), header = False), Load("lon", "lat", + where = lambda res: not('Point constaté détruit' in res['description'] or 'Point non retrouvé' in res['description']), # No more uniq ref create = """ id VARCHAR(254) PRIMARY KEY, lat VARCHAR(254), From e5a0c02e6456c5e30d4b2f6d20e8519a61557573 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Rodrigo?= Date: Mon, 2 Sep 2019 23:06:33 +0200 Subject: [PATCH 10/11] Contains in place of Intersects in analyser_osmosis_double_tagging #598 --- analysers/analyser_osmosis_double_tagging.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/analysers/analyser_osmosis_double_tagging.py b/analysers/analyser_osmosis_double_tagging.py index cc1a16b3b..3d7ea30e9 100644 --- a/analysers/analyser_osmosis_double_tagging.py +++ b/analysers/analyser_osmosis_double_tagging.py @@ -80,7 +80,7 @@ FROM {0}{2} AS {2} JOIN {1}{3} AS {3} ON - ST_Intersects({2}.geom, {3}.geom) AND + ST_Contains({2}.geom, {3}.geom) AND ( ({2}.name IS NULL AND {3}.name IS NULL) OR {2}.name = {3}.name @@ -129,5 +129,5 @@ def f(o1, o2, ret1, ret2, class_): def apply(self, callback): ret = {"nodes_alb": self.node_full, "ways_alb": self.way_full, "relations_alb": self.relation_full} - for c in [["ways_alb", "nodes_alb", 1], ["ways_alb", "relations_alb", 2], ["relations_alb", "nodes_alb", 3]]: + for c in [["ways_alb", "nodes_alb", 1], ["relations_alb", "ways_alb", 2], ["relations_alb", "nodes_alb", 3]]: callback(c[0], c[1], ret[c[0]], ret[c[1]], c[2]) From 5ee34310129d3cb73a25610c7cb07c934b5bb133 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Rodrigo?= Date: Mon, 2 Sep 2019 23:54:55 +0200 Subject: [PATCH 11/11] Fix fallback coord on geocoding fails #591 --- analysers/analyser_merge_heritage_FR_merimee.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/analysers/analyser_merge_heritage_FR_merimee.py b/analysers/analyser_merge_heritage_FR_merimee.py index 41e7f249e..2b847231d 100644 --- a/analysers/analyser_merge_heritage_FR_merimee.py +++ b/analysers/analyser_merge_heritage_FR_merimee.py @@ -96,5 +96,6 @@ def parseDPRO(dpro): def coord_fallback(self, fields): if not fields['longitude'] and fields['Coordonnées INSEE']: - fields['latitude'], fields['longitude'] = list(map(lambda s: float(s.strip()), fields['Coordonnées INSEE'].split(','))) + # It is too late to write to fields['longitude'] and fields['latitude'] + fields['_y'], fields['_x'] = list(map(lambda s: float(s.strip()), fields['Coordonnées INSEE'].split(','))) return fields