Better select in analyser_merge_shop_FR & move mapping to json osm-fr…

…#338
frodrigo · Jun 17, 2019 · fcbb5f6 · fcbb5f6
1 parent 098e9d9
commit fcbb5f6
Show file tree

Hide file tree

Showing 3 changed files with 965 additions and 79 deletions.
diff --git a/analysers/analyser_merge_shop_FR.py b/analysers/analyser_merge_shop_FR.py
@@ -22,6 +22,7 @@
 
 from io import open # In python3 only, this import is not required
 from backports import csv # In python3 only just "import csv"
+import json
 from .Analyser_Merge_Dynamic import Analyser_Merge_Dynamic, SubAnalyser_Merge_Dynamic
 from .Analyser_Merge import Source, CSV, Load, Mapping, Select, Generate
 from time import gmtime, strftime
@@ -32,33 +33,26 @@ class Analyser_Merge_Shop_FR(Analyser_Merge_Dynamic):
     def __init__(self, config, logger = None):
         Analyser_Merge_Dynamic.__init__(self, config, logger)
 
-        with open("merge_data/shop_FR.mapping.csv") as mappingfile:
-            spamreader = csv.reader(mappingfile)
-            for row in spamreader:
-                if row[0][0] == '#':
-                    continue
-                items, classs, level, title = row[0:4]
-                items = items.split('|')
-                level = int(level)
-                osmTags = filter(lambda a: a, map(lambda t: (t.split('=') + [None])[0:2] if t else None, row[4:]))
-                if len(osmTags) > 0:
-                    self.classFactory(SubAnalyser_Merge_Shop_FR, classs.replace('.', ''), items, classs, level, title, dict(osmTags), dict(filter(lambda a: a[1], osmTags)))
+        mapping = 'merge_data/shop_FR.mapping.json'
+        mapingfile = json.loads(open(mapping).read())
+        for r in mapingfile:
+            self.classFactory(SubAnalyser_Merge_Shop_FR, r['class'].replace('.', ''), r['missing_official'], r['missing_osm'], r['class'], r['level'], r['title'], r['match'], r['generate'])
 
 
 class SubAnalyser_Merge_Shop_FR(SubAnalyser_Merge_Dynamic):
-    def __init__(self, config, error_file, logger, items, classs, level, title, selectTags, generateTags):
+    def __init__(self, config, error_file, logger, missing_official, missing_osm, classs, level, title, selectTags, generateTags):
         classss = int(classs.replace('.', '0')[:-1]) * 100 + ord(classs[-1]) - 65
-        self.missing_official = {"item": items[0], "class": classss+1, "level": level, "tag": ["merge"], "desc": T_(u"%s not integrated", title) }
-        #self.missing_osm      = {"item": items[1], "class": classss+2, "level": level, "tag": ["merge"], "desc": T_f(u"{0} without tag \"{1}\" or invalid", title, 'ref:FR:SIRET') }
-        #self.possible_merge   = {"item": items[0][0:-1]+"1", "class": classss+3, "level": level, "tag": ["merge"], "desc": T_(u"%s, integration suggestion", title) }
-        #self.update_official  = {"item": items[0][0:-1]+"2", "class": classss+4, "level": level, "tag": ["merge"], "desc": T_(u"%s update", title) }
+        self.missing_official = {"item": missing_official, "class": classss+1, "level": level, "tag": ["merge"], "desc": T_(u"%s not integrated", title) }
+        #self.missing_osm      = {"item": missing_osm, "class": classss+2, "level": level, "tag": ["merge"], "desc": T_f(u"{0} without tag \"{1}\" or invalid", title, 'ref:FR:SIRET') }
+        #self.possible_merge   = {"item": missing_official[0:-1]+"1", "class": classss+3, "level": level, "tag": ["merge"], "desc": T_(u"%s, integration suggestion", title) }
+        #self.update_official  = {"item": missing_official[0:-1]+"2", "class": classss+4, "level": level, "tag": ["merge"], "desc": T_(u"%s update", title) }
 
         dep_code = config.options.get('country').split('-')[1]
 
         SubAnalyser_Merge_Dynamic.__init__(self, config, error_file, logger,
             u"http://www.sirene.fr/sirene/public/static/open-data",
             u"Sirene",
-            CSV(Source(attribution = u"INSEE", millesime = "06/2018", gzip = True,
+            CSV(Source(attribution = u"INSEE", millesime = "06/2019", gzip = True,
                 fileUrl = u"http://data.cquest.org/geo_sirene/v2019/last/dep/geo_siret_{0}.csv.gz".format(dep_code))),
             Load("longitude", "latitude",
                 select = {"activitePrincipaleEtablissement": classs, "geo_type": "housenumber", "etatAdministratifEtablissement": "A"},
@@ -81,4 +75,4 @@ def __init__(self, config, error_file, logger, items, classs, level, title, sele
                         #    "-".join([fields["DDEBACT"][0:4], fields["DDEBACT"][4:6], fields["DDEBACT"][6:8]]) if fields["DDEBACT"] != "19000101" else
                         #    "-".join([fields["DCRET"][0:4], fields["DCRET"][4:6], fields["DCRET"][6:8]]) if fields["DCRET"] != "19000101" else
                         #    None},
-                text = lambda tags, fields: {"en": ', '.join(filter(lambda f: f and f != 'None', [fields["enseigne1Etablissement"] or fields["denominationUsuelleEtablissement"]] + map(lambda k: fields[k], ["numeroVoieEtablissement", "indiceRepetitionEtablissement" ,"typeVoieEtablissement", "libelleVoieEtablissement", "complementAdresseEtablissement", "codePostalEtablissement", "libelleCommuneEtablissement"])))} )))
+                text = lambda tags, fields: {"en": ', '.join(filter(lambda f: f and f != 'None', [fields["enseigne1Etablissement"] or fields["denominationUsuelleEtablissement"]] + list(map(lambda k: fields[k], ["numeroVoieEtablissement", "indiceRepetitionEtablissement" ,"typeVoieEtablissement", "libelleVoieEtablissement", "complementAdresseEtablissement", "codePostalEtablissement", "libelleCommuneEtablissement"]))))} )))
diff --git a/merge_data/shop_FR.mapping.csv b/merge_data/shop_FR.mapping.csv