diff --git a/analysers/Analyser_Merge.py b/analysers/Analyser_Merge.py index cd555c0ac..793e05e87 100644 --- a/analysers/Analyser_Merge.py +++ b/analysers/Analyser_Merge.py @@ -22,6 +22,7 @@ import io import bz2 +import gzip from backports import csv # In python3 only just "import csv" import hashlib import inspect @@ -278,7 +279,7 @@ """ class Source: - def __init__(self, attribution = None, millesime = None, url = None, name = None, encoding = "utf-8", file = None, fileUrl = None, fileUrlCache = 30, zip = None, filter = None, logger = None): + def __init__(self, attribution = None, millesime = None, encoding = "utf-8", file = None, fileUrl = None, fileUrlCache = 30, zip = None, gzip = False, filter = None, logger = None): """ Describe the source file. @param encoding: file charset encoding @@ -286,6 +287,7 @@ def __init__(self, attribution = None, millesime = None, url = None, name = None @param urlFile: remote URL of source file @param fileUrlCache: days for file in cache @param zip: extract file from zip + @param gzip: uncompress as gzip @param filter: lambda expression applied on text file before loading @param logger: a logger """ @@ -296,6 +298,7 @@ def __init__(self, attribution = None, millesime = None, url = None, name = None self.fileUrl = fileUrl self.fileUrlCache = fileUrlCache self.zip = zip + self.gzip = gzip self.filter = filter self.logger = logger @@ -333,6 +336,10 @@ def open(self): z = zipfile.ZipFile(f, 'r').open(self.zip) f = io.BytesIO(z.read()) f.seek(0) + elif self.gzip: + d = gzip.open(downloader.path(self.fileUrl, self.fileUrlCache), mode='r') + f = io.BytesIO(d.read()) + f.seek(0) f = io.StringIO(f.read().decode(self.encoding, 'ignore')) f.seek(0) if self.filter: diff --git a/analysers/analyser_merge_power_plant_FR.py b/analysers/analyser_merge_power_plant_FR.py index fd199fdea..6056b0dcb 100644 --- a/analysers/analyser_merge_power_plant_FR.py +++ b/analysers/analyser_merge_power_plant_FR.py @@ -36,7 +36,8 @@ def __init__(self, config, logger = None): CSV(Power_Plant_FR_Source(attribution = u"data.gouv.fr:RTE", millesime = "2017", fileUrl = u"https://opendata.reseaux-energies.fr/explore/dataset/registre-national-installation-production-stockage-electricite-agrege-311217/download/?format=csv&timezone=Europe/Berlin&use_labels_for_header=true", logger=logger), separator = u";"), - Load("longitude", "latitude"), + Load("longitude", "latitude", + where = lambda res: res.get('max_puissance') and float(res["max_puissance"]) > 1000), Mapping( select = Select( types = ["ways", "relations"], @@ -50,7 +51,7 @@ def __init__(self, config, logger = None): # No voltage tga on power=plant #"voltage": lambda fields: (int(fields["Tension raccordement"].split(' ')[0]) * 1000) if fields.get("Tension raccordement") and fields["Tension raccordement"] not in ["< 45 kV", "BT", "HTA"] else None, "plant:source": lambda fields: self.filiere[fields["Filière"]][fields["Combustible"]], - "plant:output:electricity": lambda fields: (int(float(fields["max_puissance"]) * 1000)) if fields.get("max_puissance") and float(fields["max_puissance"]) >= 1000 else None}, + "plant:output:electricity": lambda fields: int(float(fields["max_puissance"]) * 1000)}, mapping2 = { "start_date": lambda fields: None if not fields.get(u"dateMiseEnService") else fields[u"dateMiseEnService"][0:4] if fields[u"dateMiseEnService"].endswith('-01-01') or fields[u"dateMiseEnService"].endswith('-12-31') else fields[u"dateMiseEnService"]}, tag_keep_multiple_values = ["voltage"], diff --git a/analysers/analyser_merge_shop_FR.py b/analysers/analyser_merge_shop_FR.py index 5a144ce4d..fc9151ede 100644 --- a/analysers/analyser_merge_shop_FR.py +++ b/analysers/analyser_merge_shop_FR.py @@ -42,7 +42,7 @@ def __init__(self, config, logger = None): level = int(level) osmTags = filter(lambda a: a, map(lambda t: (t.split('=') + [None])[0:2] if t else None, row[4:])) if len(osmTags) > 0: - self.classFactory(SubAnalyser_Merge_Shop_FR, classs, items, classs, level, title, dict(osmTags), dict(filter(lambda a: a[1], osmTags))) + self.classFactory(SubAnalyser_Merge_Shop_FR, classs.replace('.', ''), items, classs, level, title, dict(osmTags), dict(filter(lambda a: a[1], osmTags))) class SubAnalyser_Merge_Shop_FR(SubAnalyser_Merge_Dynamic): @@ -56,10 +56,11 @@ def __init__(self, config, error_file, logger, items, classs, level, title, sele SubAnalyser_Merge_Dynamic.__init__(self, config, error_file, logger, u"http://www.sirene.fr/sirene/public/static/open-data", u"Sirene", - CSV(Source(attribution = u"INSEE", millesime = "07/2017", file = "shop_FR-light.csv.bz2")), + CSV(Source(attribution = u"INSEE", millesime = "06/2018", gzip = True, + fileUrl = u"http://data.cquest.org/geo_sirene/v2019/last/dep/geo_siret_84.csv.gz")), Load("longitude", "latitude", - select = {"APET700": classs, "NJ": True}, - uniq = ["SIREN", "NIC"]), + select = {"activitePrincipaleEtablissement": classs}, + uniq = ["siren", "nic"]), Mapping( select = Select( types = ['nodes', 'ways'], @@ -69,12 +70,12 @@ def __init__(self, config, error_file, logger, items, classs, level, title, sele static1 = generateTags, static2 = {"source": self.source}, mapping1 = { - # "ref:FR:SIRET": lambda fields: fields["SIREN"] + fields["NIC"], + # "ref:FR:SIRET": lambda fields: fields["siren"] + fields["nic"], # "ref:FR:RNA": "RNA", - "name": lambda fields: fields["ENSEIGNE"] or (fields["NOMEN_LONG"] if fields["NJ"] else None), - "short_name": "SIGLE"}, + "name": lambda fields: fields["enseigne1Etablissement"] or fields["denominationUsuelleEtablissement"] or None}, + #"short_name": "SIGLE"}, #"start_date": lambda fields: # "-".join([fields["DDEBACT"][0:4], fields["DDEBACT"][4:6], fields["DDEBACT"][6:8]]) if fields["DDEBACT"] != "19000101" else # "-".join([fields["DCRET"][0:4], fields["DCRET"][4:6], fields["DCRET"][6:8]]) if fields["DCRET"] != "19000101" else # None}, - text = lambda tags, fields: {"en": ', '.join(filter(lambda f: f, [fields["ENSEIGNE"] or (fields["NOMEN_LONG"] if fields["NJ"] else None)] + map(lambda k: fields[k], ["L1_DECLAREE", "L2_DECLAREE" ,"L3_DECLAREE", "L4_DECLAREE", "L5_DECLAREE", "L6_DECLAREE", "L7_DECLAREE"])))} ))) + text = lambda tags, fields: {"en": ', '.join(filter(lambda f: f, [fields["enseigne1Etablissement"] or fields["denominationUsuelleEtablissement"]] + map(lambda k: fields[k], ["numeroVoieEtablissement", "indiceRepetitionEtablissement" ,"typeVoieEtablissement", "libelleVoieEtablissement", "complementAdresseEtablissement", "codePostalEtablissement", "libelleCommuneEtablissement"])))} ))) diff --git a/merge_data/hydrant_point_CH_lausanne.csv.bz2 b/merge_data/hydrant_point_CH_lausanne.csv.bz2 deleted file mode 100644 index 7fe8c6994..000000000 Binary files a/merge_data/hydrant_point_CH_lausanne.csv.bz2 and /dev/null differ diff --git a/merge_data/pharmacy_FR.csv.bz2 b/merge_data/pharmacy_FR.csv.bz2 deleted file mode 100644 index 82dd00fe8..000000000 Binary files a/merge_data/pharmacy_FR.csv.bz2 and /dev/null differ diff --git a/merge_data/shop_FR-light.csv.bz2 b/merge_data/shop_FR-light.csv.bz2 deleted file mode 100644 index 0f0a1be54..000000000 Binary files a/merge_data/shop_FR-light.csv.bz2 and /dev/null differ diff --git a/merge_data/wikipedia_insee_FR.csv.bz2 b/merge_data/wikipedia_insee_FR.csv.bz2 deleted file mode 100644 index 0670e5251..000000000 Binary files a/merge_data/wikipedia_insee_FR.csv.bz2 and /dev/null differ diff --git a/plugins/TagRemove_Fixme.py b/plugins/TagRemove_Fixme.py index 1cb7c4490..fcd28bc25 100644 --- a/plugins/TagRemove_Fixme.py +++ b/plugins/TagRemove_Fixme.py @@ -30,8 +30,8 @@ def init(self, logger): self.errors[40611] = { "item": 4061, "level": 2, "tag": ["fixme", "fix:chair", "highway"], "desc": T_(u"Highway classification need review") } def node(self, data, tags): - if "fixme" in tags: - return [{"class": 40610, "subclass": 1, "text": {"en": tags['fixme']}}] + if "fixme" in tags or "FIXME" in tags: + return [{"class": 40610, "subclass": 1, "text": {"en": tags.get('fixme') or tags['FIXME']}}] else: return [] @@ -56,3 +56,4 @@ def test(self): assert not a.way(None, {"highway": "trunk"}, None) self.check_err(a.way(None, {"highway": "road"}, None)) self.check_err(a.way(None, {"fixme": "plop"}, None)) + self.check_err(a.way(None, {"FIXME": "plop"}, None))