Merge remote-tracking branch 'frodrigo/master'

* frodrigo/master:
  Remove not used data files
  Data source of analyser_merge_shop_FR from data.cquest.org #338
  Support gzip source in Analyser_Merge
  Set correct max_puissance filter in analyser_merge_power_plant_FR #510
  Allow check for tag 'FIXME' in TagRemove_Fixme #520

analysers/Analyser_Merge.py

@@ -22,6 +22,7 @@
 
 import io
 import bz2
+import gzip
 from backports import csv # In python3 only just "import csv"
 import hashlib
 import inspect
@@ -278,14 +279,15 @@
 """
 
 class Source:
-    def __init__(self, attribution = None, millesime = None, url = None, name = None, encoding = "utf-8", file = None, fileUrl = None, fileUrlCache = 30, zip = None, filter = None, logger = None):
+    def __init__(self, attribution = None, millesime = None, encoding = "utf-8", file = None, fileUrl = None, fileUrlCache = 30, zip = None, gzip = False, filter = None, logger = None):
         """
         Describe the source file.
         @param encoding: file charset encoding
         @param file: file name in storage
         @param urlFile: remote URL of source file
         @param fileUrlCache: days for file in cache
         @param zip: extract file from zip
+        @param gzip: uncompress as gzip
         @param filter: lambda expression applied on text file before loading
         @param logger: a logger
         """
@@ -296,6 +298,7 @@ def __init__(self, attribution = None, millesime = None, url = None, name = None
         self.fileUrl = fileUrl
         self.fileUrlCache = fileUrlCache
         self.zip = zip
+        self.gzip = gzip
         self.filter = filter
         self.logger = logger
 
@@ -333,6 +336,10 @@ def open(self):
                 z = zipfile.ZipFile(f, 'r').open(self.zip)
                 f = io.BytesIO(z.read())
                 f.seek(0)
+            elif self.gzip:
+                d = gzip.open(downloader.path(self.fileUrl, self.fileUrlCache), mode='r')
+                f = io.BytesIO(d.read())
+                f.seek(0)
         f = io.StringIO(f.read().decode(self.encoding, 'ignore'))
         f.seek(0)
         if self.filter:

analysers/analyser_merge_power_plant_FR.py

@@ -36,7 +36,8 @@ def __init__(self, config, logger = None):
             CSV(Power_Plant_FR_Source(attribution = u"data.gouv.fr:RTE", millesime = "2017",
                     fileUrl = u"https://opendata.reseaux-energies.fr/explore/dataset/registre-national-installation-production-stockage-electricite-agrege-311217/download/?format=csv&timezone=Europe/Berlin&use_labels_for_header=true", logger=logger),
                 separator = u";"),
-            Load("longitude", "latitude"),
+            Load("longitude", "latitude",
+                where = lambda res: res.get('max_puissance') and float(res["max_puissance"]) > 1000),
             Mapping(
                 select = Select(
                     types = ["ways", "relations"],
@@ -50,7 +51,7 @@ def __init__(self, config, logger = None):
                         # No voltage tga on power=plant
                         #"voltage": lambda fields: (int(fields["Tension raccordement"].split(' ')[0]) * 1000) if fields.get("Tension raccordement") and fields["Tension raccordement"] not in ["< 45 kV", "BT", "HTA"] else None,
                         "plant:source": lambda fields: self.filiere[fields["Filière"]][fields["Combustible"]],
-                        "plant:output:electricity": lambda fields: (int(float(fields["max_puissance"]) * 1000)) if fields.get("max_puissance") and float(fields["max_puissance"]) >= 1000 else None},
+                        "plant:output:electricity": lambda fields: int(float(fields["max_puissance"]) * 1000)},
                     mapping2 = {
                         "start_date": lambda fields: None if not fields.get(u"dateMiseEnService") else fields[u"dateMiseEnService"][0:4] if fields[u"dateMiseEnService"].endswith('-01-01') or fields[u"dateMiseEnService"].endswith('-12-31') else fields[u"dateMiseEnService"]},
                    tag_keep_multiple_values = ["voltage"],

analysers/analyser_merge_shop_FR.py

@@ -42,7 +42,7 @@ def __init__(self, config, logger = None):
                 level = int(level)
                 osmTags = filter(lambda a: a, map(lambda t: (t.split('=') + [None])[0:2] if t else None, row[4:]))
                 if len(osmTags) > 0:
-                    self.classFactory(SubAnalyser_Merge_Shop_FR, classs, items, classs, level, title, dict(osmTags), dict(filter(lambda a: a[1], osmTags)))
+                    self.classFactory(SubAnalyser_Merge_Shop_FR, classs.replace('.', ''), items, classs, level, title, dict(osmTags), dict(filter(lambda a: a[1], osmTags)))
 
 
 class SubAnalyser_Merge_Shop_FR(SubAnalyser_Merge_Dynamic):
@@ -56,10 +56,11 @@ def __init__(self, config, error_file, logger, items, classs, level, title, sele
         SubAnalyser_Merge_Dynamic.__init__(self, config, error_file, logger,
             u"http://www.sirene.fr/sirene/public/static/open-data",
             u"Sirene",
-            CSV(Source(attribution = u"INSEE", millesime = "07/2017", file = "shop_FR-light.csv.bz2")),
+            CSV(Source(attribution = u"INSEE", millesime = "06/2018", gzip = True,
+                fileUrl = u"http://data.cquest.org/geo_sirene/v2019/last/dep/geo_siret_84.csv.gz")),
             Load("longitude", "latitude",
-                select = {"APET700": classs, "NJ": True},
-                uniq = ["SIREN", "NIC"]),
+                select = {"activitePrincipaleEtablissement": classs},
+                uniq = ["siren", "nic"]),
             Mapping(
                 select = Select(
                     types = ['nodes', 'ways'],
@@ -69,12 +70,12 @@ def __init__(self, config, error_file, logger, items, classs, level, title, sele
                     static1 = generateTags,
                     static2 = {"source": self.source},
                     mapping1 = {
-                        # "ref:FR:SIRET": lambda fields: fields["SIREN"] + fields["NIC"],
+                        # "ref:FR:SIRET": lambda fields: fields["siren"] + fields["nic"],
                         # "ref:FR:RNA": "RNA",
-                        "name": lambda fields: fields["ENSEIGNE"] or (fields["NOMEN_LONG"] if fields["NJ"] else None),
-                        "short_name": "SIGLE"},
+                        "name": lambda fields: fields["enseigne1Etablissement"] or fields["denominationUsuelleEtablissement"] or None},
+                        #"short_name": "SIGLE"},
                         #"start_date": lambda fields:
                         #    "-".join([fields["DDEBACT"][0:4], fields["DDEBACT"][4:6], fields["DDEBACT"][6:8]]) if fields["DDEBACT"] != "19000101" else
                         #    "-".join([fields["DCRET"][0:4], fields["DCRET"][4:6], fields["DCRET"][6:8]]) if fields["DCRET"] != "19000101" else
                         #    None},
-                text = lambda tags, fields: {"en": ', '.join(filter(lambda f: f, [fields["ENSEIGNE"] or (fields["NOMEN_LONG"] if fields["NJ"] else None)] + map(lambda k: fields[k], ["L1_DECLAREE", "L2_DECLAREE" ,"L3_DECLAREE", "L4_DECLAREE", "L5_DECLAREE", "L6_DECLAREE", "L7_DECLAREE"])))} )))
+                text = lambda tags, fields: {"en": ', '.join(filter(lambda f: f, [fields["enseigne1Etablissement"] or fields["denominationUsuelleEtablissement"]] + map(lambda k: fields[k], ["numeroVoieEtablissement", "indiceRepetitionEtablissement" ,"typeVoieEtablissement", "libelleVoieEtablissement", "complementAdresseEtablissement", "codePostalEtablissement", "libelleCommuneEtablissement"])))} )))

plugins/TagRemove_Fixme.py

@@ -30,8 +30,8 @@ def init(self, logger):
         self.errors[40611] = { "item": 4061, "level": 2, "tag": ["fixme", "fix:chair", "highway"], "desc": T_(u"Highway classification need review") }
 
     def node(self, data, tags):
-        if "fixme" in tags:
-            return [{"class": 40610, "subclass": 1, "text": {"en": tags['fixme']}}]
+        if "fixme" in tags or "FIXME" in tags:
+            return [{"class": 40610, "subclass": 1, "text": {"en": tags.get('fixme') or tags['FIXME']}}]
         else:
             return []
 
@@ -56,3 +56,4 @@ def test(self):
         assert not a.way(None, {"highway": "trunk"}, None)
         self.check_err(a.way(None, {"highway": "road"}, None))
         self.check_err(a.way(None, {"fixme": "plop"}, None))
+        self.check_err(a.way(None, {"FIXME": "plop"}, None))