Merge remote-tracking branch 'frodrigo/master'

* frodrigo/master: Rename test features to be case independant #246 Add way in result of analyser_osmosis_highway_traffic_signals #360 Use ST_Intersect in place of bbxox intersect in analyser_osmosis_double_tagging #379 Remove duplicate police FR merge analyser Fix name of analyser_merge_police_FR_pn Docker: postgres tmpfs Mapcss code style Support driving side in Bicycle.mapcss mapcss: support :righthandtraffic Docker build better cache usage How to debug with docker Docker: change Docker default cmd
osm-fr · Nov 5, 2018 · 63d5bae · 63d5bae
2 parents d038de5 + 0729e71
commit 63d5bae
Show file tree

Hide file tree

Showing 24 changed files with 280 additions and 163 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -0,0 +1 @@
+.git
diff --git a/analysers/analyser_merge_police_FR_pn.py b/analysers/analyser_merge_police_FR_pn.py
@@ -42,7 +42,7 @@ def __init__(self, config, logger = None):
                 generate = Generate(
                     static1 = {
                         "amenity": "police",
-                        "name": "Commissariat",
+                        "name": "Police nationale",
                         "operator": "Police nationale"},
                     static2 = {"source": self.source},
                     mapping2 = {

diff --git a/analysers/analyser_merge_service_public_FR.py b/analysers/analyser_merge_service_public_FR.py
@@ -99,13 +99,13 @@ class Analyser_Merge_ServicePublic_FR_Mairie(_Analyser_Merge_ServicePublic_FR):
     def __init__(self, config, logger = None):
         _Analyser_Merge_ServicePublic_FR.__init__(self, config, logger, "8110", 1, 3, ["mairie", "paris_mairie_arrondissement"], {"amenity": "townhall"}, {"amenity": "townhall"})
 
-class Analyser_Merge_ServicePublic_FR_Gendarmerie(_Analyser_Merge_ServicePublic_Name_FR):
-    def __init__(self, config, logger = None):
-        _Analyser_Merge_ServicePublic_Name_FR.__init__(self, config, logger, "8110", 2, 3, "gendarmerie", {"amenity": "police"}, {"amenity": "police", "operator": "Gendarmerie nationale"})
+#class Analyser_Merge_ServicePublic_FR_Gendarmerie(_Analyser_Merge_ServicePublic_Name_FR):
+#    def __init__(self, config, logger = None):
+#        _Analyser_Merge_ServicePublic_Name_FR.__init__(self, config, logger, "8110", 2, 3, "gendarmerie", {"amenity": "police"}, {"amenity": "police", "operator": "Gendarmerie nationale"})
 
-class Analyser_Merge_ServicePublic_FR_Police(_Analyser_Merge_ServicePublic_Name_FR):
-    def __init__(self, config, logger = None):
-        _Analyser_Merge_ServicePublic_Name_FR.__init__(self, config, logger, "8110", 3, 3, "commissariat_police", {"amenity": "police"}, {"amenity": "police", "operator": "Police nationale"})
+#class Analyser_Merge_ServicePublic_FR_Police(_Analyser_Merge_ServicePublic_Name_FR):
+#    def __init__(self, config, logger = None):
+#        _Analyser_Merge_ServicePublic_Name_FR.__init__(self, config, logger, "8110", 3, 3, "commissariat_police", {"amenity": "police"}, {"amenity": "police", "operator": "Police nationale"})
 
 #class Analyser_Merge_ServicePublic_FR_EPIC(_Analyser_Merge_ServicePublic_Name_FR):
 #    def __init__(self, config, logger = None):

diff --git a/analysers/analyser_osmosis_double_tagging.py b/analysers/analyser_osmosis_double_tagging.py
@@ -80,7 +80,7 @@
 FROM
     {0}{2} AS {2}
     JOIN {1}{3} AS {3} ON
-        {2}.geom && {3}.geom AND
+        ST_Intersects({2}.geom, {3}.geom) AND
         (
             ({2}.name IS NULL AND {3}.name IS NULL) OR
             {2}.name = {3}.name

diff --git a/analysers/analyser_osmosis_highway_traffic_signals.py b/analysers/analyser_osmosis_highway_traffic_signals.py
@@ -93,6 +93,7 @@
 sql30 = """
 SELECT
   nodes.id,
+  max(ways.id),
   ST_AsText(nodes.geom)
 FROM
   {0}traffic_signals AS nodes
@@ -130,6 +131,7 @@
 sql41 = """
 SELECT
   nodes.id,
+  max(ways.id),
   ST_AsText(nodes.geom)
 FROM
   {0}stops AS nodes
@@ -164,11 +166,11 @@ def __init__(self, config, logger = None):
             [{"+":{"crossing":"traffic_signals"}}, {"-":["crossing"]}]
         ] }
         self.callback20 = lambda res: {"class":2, "data":[self.node_full, self.positionAsText]}
-        self.callback30 = lambda res: {"class":3, "data":[self.node_full, self.positionAsText], "fix":[
+        self.callback30 = lambda res: {"class":3, "data":[self.node_full, self.way, self.positionAsText], "fix":[
             [{"+":{"traffic_signals:direction":"forward"}}],
             [{"+":{"traffic_signals:direction":"backward"}}],
         ] }
-        self.callback40 = lambda res: {"class":4, "data":[self.node_full, self.positionAsText], "fix":[
+        self.callback40 = lambda res: {"class":4, "data":[self.node_full, self.way, self.positionAsText], "fix":[
             [{"+":{"direction":"forward"}}],
             [{"+":{"direction":"backward"}}],
         ] }

diff --git a/analysers/analyser_sax.py b/analysers/analyser_sax.py
@@ -616,25 +616,25 @@ def test_FR(self):
         self.check_num_err(min=53)
 
     def test_fr(self):
-        self.xml_res_file = os.path.join(self.dirname, "sax.test.fr.xml")
+        self.xml_res_file = os.path.join(self.dirname, "sax.test.Lang_fr.xml")
         self.config.dst = self.xml_res_file
         self.config.options = {"language": "fr", "project": "openstreetmap"}
         with Analyser_Sax(self.config) as analyser_obj:
             analyser_obj.analyser()
 
-        self.compare_results("tests/results/sax.test.fr.xml")
+        self.compare_results("tests/results/sax.test.Lang_fr.xml")
 
         self.root_err = self.load_errors()
         self.check_num_err(min=41)
 
     def test_fr_nl(self):
-        self.xml_res_file = os.path.join(self.dirname, "sax.test.fr_nl.xml")
+        self.xml_res_file = os.path.join(self.dirname, "sax.test.Lang_fr_nl.xml")
         self.config.dst = self.xml_res_file
         self.config.options = {"language": ["fr", "nl"], "project": "openstreetmap"}
         with Analyser_Sax(self.config) as analyser_obj:
             analyser_obj.analyser()
 
-        self.compare_results("tests/results/sax.test.fr_nl.xml")
+        self.compare_results("tests/results/sax.test.Lang_fr_nl.xml")
 
         self.root_err = self.load_errors()
         self.check_num_err(min=37)

diff --git a/docker/Dockerfile b/docker/Dockerfile
@@ -5,31 +5,34 @@ MAINTAINER Daniele Venzano <[email protected]>
 RUN apt-get update && \
     apt-get -y dist-upgrade && \
     DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
-                                build-essential \
-                                git \
-                                libprotobuf-dev \
-                                openjdk-8-jre-headless \
-                                postgresql \
-                                postgresql-contrib \
-                                postgresql-10-postgis-2.4 \
-                                postgresql-10-postgis-2.4-scripts \
-                                protobuf-compiler \
-                                python \
-                                python-dev \
-                                python-pip \
-                                python-setuptools \
-                                python-wheel \
-                                sudo && \
+        build-essential \
+        git \
+        libprotobuf-dev \
+        openjdk-8-jre-headless \
+        postgresql \
+        postgresql-contrib \
+        postgresql-10-postgis-2.4 \
+        postgresql-10-postgis-2.4-scripts \
+        protobuf-compiler \
+        python \
+        python-dev \
+        python-pip \
+        python-setuptools \
+        python-wheel \
+        sudo && \
     apt-get clean
 
-RUN mkdir -p /data/work/osmose
-RUN useradd -s /bin/bash -d /data/work/osmose osmose
-RUN chown osmose /data/work/osmose
-
-ADD . /opt/osmose-backend/
+RUN mkdir -p /data/work/osmose && \
+    useradd -s /bin/bash -d /data/work/osmose osmose && \
+    chown osmose /data/work/osmose
 
+ADD requirements.txt /opt/osmose-backend/requirements.txt
 WORKDIR /opt/osmose-backend
 RUN pip install -r requirements.txt
 ENV PYTHONPATH /opt/osmose-backend
 
+ADD . /opt/osmose-backend/
+
+ENV LANG en_US.UTF-8
 ENTRYPOINT ["/opt/osmose-backend/tools/docker-entrypoint.sh"]
+CMD bash
diff --git a/docker/README.md b/docker/README.md
@@ -23,29 +23,64 @@ Running a country alone
 Taking the comoros (a quick one) as an example, once you have the image, you can
 run osmose checks like this:
 ```
-docker run -it --rm osm-fr/osmose_backend --country=comoros
+docker run -it --rm osm-fr/osmose_backend ./osmose_run.py --country=comoros
 ```
 This will run interactively and you will see the output scrolling on your screen. The
-container will be deleted at the end and no data will be saved on disk.
+container will be deleted at the end of the process and all data will wiped.
 
+To run with the password file and enable result upload to the frontend
+you can use the following command line:
+```
+docker run -it --rm -v $PWD/osmose_config_password.py:/opt/osmose-backend/osmose_config_password.py osm-fr/osmose_backend ./osmose_run.py --country=comoros
+```
+
+
+Speed improvment
+----------------
+
+The postgres database inside the container can be put in memory inplace
+of file system. Resulting in better speed and SSD lifetime.
+```
+docker run -it --rm --tmpfs /var/lib/postgres osm-fr/osmose_backend ./osmose_run.py --country=comoros
+```
+
+
+Develop analyser with docker
+============================
 
 Running and keep results
 ------------------------
 
-If, instead, you want to keep the output files locally you can add a volume in the right
+If you want to keep the output files locally you can add a volume in the right
 location, like this:
 ```
-docker run -it --rm -v $PWD/work:/data/work/osmose osm-fr/osmose_backend --country=comoros
+docker run -it --rm --tmpfs /var/lib/postgres -v $PWD/work:/data/work/osmose osm-fr/osmose_backend ./osmose_run.py --country=comoros
 ```
-The directory on your host, `work` in this case, needs to be writable by everyone, as the
+The directory on your host, `work` in this case, needs to be writable by anyone, as the
 `osmose` user in the container will have some random UID (probably 1000).
 
-Finally, to run with the password file and enable result upload to the frontend you can
-use the following command line:
+
+Enter the container to test and debug
+-------------------------------------
+
+Override the Osmose source code in the contrainer with the working
+directory. Use the local source directory as volume to override source in
+the container.
+```
+docker run -it --rm --tmpfs /var/lib/postgres -v $PWD/work:/data/work/osmose -v $PWD/..:/opt/osmose-backend osm-fr/osmose_backend bash
+```
+
+On docker container you can run analyser:
 ```
-docker run -it --rm -v $PWD/osmose_config_password.py:/opt/osmose-backend/osmose_config_password.py osm-fr/osmose_backend --country=comoros
+./osmose_run.py --no-clean --country=comoros
 ```
 
+Open psql shell on database (after `--no-clean` run of `osmose_run.py`):
+```
+psql
+```
+
+
 Docker-Compose - Run and show the result on a map
 =================================================
 

diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml
@@ -12,7 +12,7 @@ services:
       - ./work:/data/work/osmose
     environment:
       - URL_FRONTEND_UPDATE=http://frontend:20009/control/send-update
-    command: --country=comoros
+    command: ./osmose_run.py --country=comoros
 
   frontend:
     image: osm-fr/osmose_frontend

diff --git a/mapcss/mapcss2osmose.py b/mapcss/mapcss2osmose.py
@@ -176,6 +176,18 @@ def declaration_value_function_param_regex(t, c):
             t['params'][0]['value'] = {'type': 'regexExpression', 'value': t['params'][0]['value']['value']}
     return t
 
+def pseudo_class_righthandtraffic(t, c):
+    """
+    type = pseudo_class
+    Replace pseudo class :righthandtraffic by call to setting()
+    """
+    if t['pseudo_class'] == 'righthandtraffic':
+        t = {'type': 'booleanExpression', 'operator': '=' if t['not_class'] else '!=', 'operands': [
+            {'type': 'functionExpression', 'name': 'setting', 'params': [{'type': 'primaryExpression', 'derefered': False, 'value': {'type': 'quoted', 'value': 'driving_side'}}]},
+            {'type': 'primaryExpression', 'derefered': False, 'value': {'type': 'quoted', 'value': 'left'}}
+        ]}
+    return t
+
 rule_declarations_order_map = {
     # subclass
     'group': 1,
@@ -305,6 +317,7 @@ def rule_after_set(t, c):
     ('booleanExpression', booleanExpression_negated_operator),
     ('booleanExpression', booleanExpression_operator_to_function),
     ('declaration_value_function', declaration_value_function_param_regex),
+    ('pseudo_class', pseudo_class_righthandtraffic),
     # Safty
     ('rule', rule_declarations_order),
     # Rule flag
@@ -364,7 +377,7 @@ def segregate_selectors_by_complexity(t):
             for selector in rule['selectors']:
                 if selector['operator']:
                     selector_complex.append(selector)
-                elif next(filter(lambda a: not a in('closed', 'closed2', 'tagged'), selector['simple_selectors'][0]['pseudo_class']), False):
+                elif any(map(lambda a: not a['pseudo_class'] in('closed', 'closed2', 'tagged', 'righthandtraffic'), selector['simple_selectors'][0]['pseudo_class'])):
                     selector_complex.append(selector)
                 else:
                     selector_simple.append(selector)
@@ -645,7 +658,7 @@ def to_p(t):
                 ("mapcss." + t['name'])
             ) + "(" + (
                 ("tags, " if t['name'] == 'tag' else "") +
-                ("self.father.config.options, " if t['name'] in ('inside', 'outside') else "") +
+                ("self.father.config.options, " if t['name'] in ('inside', 'outside', 'setting') else "") +
                 (("capture_tags, " + str(predicate_capture_index) + ", tags, ") if t['name'] == '_tag_capture' else "") +
                 (("capture_tags, " + str(predicate_capture_index) + ", ") if t['name'] == '_value_capture' else "")
             ) + ", ".join(map(to_p, t['params'])) + ")"

diff --git a/mapcss/mapcss_lib.py b/mapcss/mapcss_lib.py
@@ -480,8 +480,8 @@ def JOSM_pref(key, default):
 
 #setting()
 #    to use a style setting [since 7450] 
-def setting():
-    raise NotImplementedError
+def setting(options, key):
+    return options.get(key)
 
 #degree_to_radians()
 #    returns a in degree given direction in radians [since 8260] 

diff --git a/plugins/Bicycle.py b/plugins/Bicycle.py
@@ -105,13 +105,19 @@ def way(self, data, tags, *args):
                 # assertMatch:"way cycleway=opposite"
                 err.append({'class': 20301, 'subclass': 0, 'text': mapcss.tr(u'Opposite cycleway without oneway', capture_tags)})
 
-        # way["cycleway:right"=~/opposite|opposite_lane/][oneway=yes]
-        # way["cycleway:left"=~/opposite|opposite_lane/][oneway="-1"]
+        # way:righthandtraffic["cycleway:right"=~/opposite|opposite_lane/][oneway=yes]
+        # way:righthandtraffic["cycleway:left"=~/opposite|opposite_lane/][oneway="-1"]
+        # way!:righthandtraffic["cycleway:left"=~/opposite|opposite_lane/][oneway=yes]
+        # way!:righthandtraffic["cycleway:right"=~/opposite|opposite_lane/][oneway="-1"]
         if u'cycleway:left' in keys or u'cycleway:right' in keys:
             match = False
-            try: match = match or ((mapcss.regexp_test_(mapcss._value_capture(capture_tags, 0, self.re_67b51e41), mapcss._tag_capture(capture_tags, 0, tags, u'cycleway:right')) and mapcss._tag_capture(capture_tags, 1, tags, u'oneway') == mapcss._value_capture(capture_tags, 1, u'yes')))
+            try: match = match or ((mapcss.regexp_test_(mapcss._value_capture(capture_tags, 0, self.re_67b51e41), mapcss._tag_capture(capture_tags, 0, tags, u'cycleway:right')) and mapcss._tag_capture(capture_tags, 1, tags, u'oneway') == mapcss._value_capture(capture_tags, 1, u'yes') and mapcss.setting(self.father.config.options, u'driving_side') != u'left'))
             except mapcss.RuleAbort: pass
-            try: match = match or ((mapcss.regexp_test_(mapcss._value_capture(capture_tags, 0, self.re_67b51e41), mapcss._tag_capture(capture_tags, 0, tags, u'cycleway:left')) and mapcss._tag_capture(capture_tags, 1, tags, u'oneway') == mapcss._value_capture(capture_tags, 1, u'-1')))
+            try: match = match or ((mapcss.regexp_test_(mapcss._value_capture(capture_tags, 0, self.re_67b51e41), mapcss._tag_capture(capture_tags, 0, tags, u'cycleway:left')) and mapcss._tag_capture(capture_tags, 1, tags, u'oneway') == mapcss._value_capture(capture_tags, 1, u'-1') and mapcss.setting(self.father.config.options, u'driving_side') != u'left'))
+            except mapcss.RuleAbort: pass
+            try: match = match or ((mapcss.regexp_test_(mapcss._value_capture(capture_tags, 0, self.re_67b51e41), mapcss._tag_capture(capture_tags, 0, tags, u'cycleway:left')) and mapcss._tag_capture(capture_tags, 1, tags, u'oneway') == mapcss._value_capture(capture_tags, 1, u'yes') and mapcss.setting(self.father.config.options, u'driving_side') == u'left'))
+            except mapcss.RuleAbort: pass
+            try: match = match or ((mapcss.regexp_test_(mapcss._value_capture(capture_tags, 0, self.re_67b51e41), mapcss._tag_capture(capture_tags, 0, tags, u'cycleway:right')) and mapcss._tag_capture(capture_tags, 1, tags, u'oneway') == mapcss._value_capture(capture_tags, 1, u'-1') and mapcss.setting(self.father.config.options, u'driving_side') == u'left'))
             except mapcss.RuleAbort: pass
             if match:
                 # osmoseTags:list("tag","highway","cycleway","fix:survey")

diff --git a/plugins/Bicycle.mapcss → plugins/Bicycle.validator.mapcss b/plugins/Bicycle.mapcss → plugins/Bicycle.validator.mapcss
@@ -1,8 +1,26 @@
+/*#########################################################################
+##                                                                       ##
+## Copyrights Géovélo                                                    ##
+##                                                                       ##
+## This program is free software: you can redistribute it and/or modify  ##
+## it under the terms of the GNU General Public License as published by  ##
+## the Free Software Foundation, either version 3 of the License, or     ##
+## (at your option) any later version.                                   ##
+##                                                                       ##
+## This program is distributed in the hope that it will be useful,       ##
+## but WITHOUT ANY WARRANTY; without even the implied warranty of        ##
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the         ##
+## GNU General Public License for more details.                          ##
+##                                                                       ##
+## You should have received a copy of the GNU General Public License     ##
+## along with this program.  If not, see <http://www.gnu.org/licenses/>. ##
+##                                                                       ##
+#########################################################################*/
+
 meta {
-    title: "Bike validation ruleset";
+    title: "Osmose-QA – Bike validation ruleset";
     description: "Some validation for bike facilities";
-    author: "geovelo";
-    baselanguage: "en";
+    author: "Géovélo";
 }
 meta[lang=en] { /* lang=en, unused, only to use tr() to catch string for translation */
     description: tr("Some validation for bike facilities");
@@ -64,8 +82,11 @@ way[cycleway=~/opposite|opposite_lane/][oneway=no] {
     assertNoMatch: "way cycleway=lane oneway=yes";
 }
 
-way["cycleway:right"=~/opposite|opposite_lane/][oneway=yes],
-way["cycleway:left"=~/opposite|opposite_lane/][oneway="-1"] {
+
+way:righthandtraffic["cycleway:right"=~/opposite|opposite_lane/][oneway=yes],
+way:righthandtraffic["cycleway:left"=~/opposite|opposite_lane/][oneway="-1"],
+way!:righthandtraffic["cycleway:left"=~/opposite|opposite_lane/][oneway=yes],
+way!:righthandtraffic["cycleway:right"=~/opposite|opposite_lane/][oneway="-1"] {
     throwError: tr("Opposite or opposite lane in the same way of the oneway");
     osmoseItemClassLevel: "2030/20302/1";
     osmoseTags: list("tag", "highway", "cycleway", "fix:survey");