Merge pull request #4225 from hove-io/add_excluded_zones_manager

[Jormungandr] Add excluded zones manager

source/jormungandr/jormungandr/default_settings.py

@@ -157,6 +157,7 @@
     'FETCH_S3_DATA_TIMEOUT': 24 * 60,
     # TIMEOUT_TRANSFER_PATH = 24Hours
     "TIMEOUT_TRANSFER_PATH": 24 * 60 * 60,
+    'ASGARD_S3_DATA_TIMEOUT': 2 * 60,
 }
 
 
@@ -168,6 +169,7 @@
     'TIMEOUT_AUTHENTICATION': 30,
     'TIMEOUT_PARAMS': 30,
     "FETCH_S3_DATA_TIMEOUT": 2 * 60,
+    'ASGARD_S3_DATA_TIMEOUT': 1 * 60,
 }
 
 MEMORY_CACHE_CONFIGURATION = (

source/jormungandr/jormungandr/excluded_zones_manager.py

@@ -0,0 +1,94 @@
+# Copyright (c) 2001-2024, Hove and/or its affiliates. All rights reserved.
+#
+# This file is part of Navitia,
+#     the software to build cool stuff with public transport.
+#
+# Hope you'll enjoy and contribute to this project,
+#     powered by Hove (www.hove.com).
+# Help us simplify mobility and open public transport:
+#     a non ending quest to the responsive locomotion way of traveling!
+#
+# LICENCE: This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+# Stay tuned using
+# twitter @navitia
+# channel `#navitia` on riot https://riot.im/app/#/room/#navitia:matrix.org
+# https://groups.google.com/d/forum/navitia
+# www.navitia.io
+
+import boto3
+from botocore.client import Config
+import logging
+import json
+
+from jormungandr import app, memory_cache, cache
+from jormungandr.resource_s3_object import ResourceS3Object
+
+
+class ExcludedZonesManager:
+    @staticmethod
+    @cache.memoize(app.config[str('CACHE_CONFIGURATION')].get(str('ASGARD_S3_DATA_TIMEOUT'), 24 * 60))
+    def get_object(resource_s3_object):
+        logger = logging.getLogger(__name__)
+        try:
+            file_content = resource_s3_object.s3_object.get()['Body'].read().decode('utf-8')
+            return json.loads(file_content)
+        except Exception:
+            logger.exception('Error while loading file: {}'.format(resource_s3_object.s3_object.key))
+            return {}
+
+    @staticmethod
+    @memory_cache.memoize(
+        app.config[str('MEMORY_CACHE_CONFIGURATION')].get(str('ASGARD_S3_DATA_TIMEOUT'), 5 * 60)
+    )
+    def get_excluded_zones(instance_name=None, mode=None):
+        bucket_name = app.config.get(str("ASGARD_S3_BUCKET"))
+        folder = "excluded_zones"
+
+        logger = logging.getLogger(__name__)
+        args = {"connect_timeout": 2, "read_timeout": 2, "retries": {'max_attempts': 0}}
+        s3_resource = boto3.resource('s3', config=Config(**args))
+        excluded_zones = []
+        try:
+            my_bucket = s3_resource.Bucket(bucket_name)
+            for obj in my_bucket.objects.filter(Prefix="{}/".format(folder)):
+                if not obj.key.endswith('.json'):
+                    continue
+                try:
+                    json_content = ExcludedZonesManager.get_object(ResourceS3Object(obj, None))
+
+                    if instance_name is not None and json_content.get('instance') != instance_name:
+                        continue
+                    if mode is not None and mode not in json_content.get("modes", []):
+                        continue
+
+                    excluded_zones.append(json_content)
+                except Exception:
+                    logger.exception(
+                        "Error on fetching excluded zones: bucket: {}, instance: {}, mode ={}",
+                        bucket_name,
+                        instance_name,
+                        mode,
+                    )
+                    continue
+
+        except Exception:
+            logger.exception(
+                "Error on fetching excluded zones: bucket: {}, instance: {}, mode ={}",
+                bucket_name,
+                instance_name,
+                mode,
+            )
+
+        return excluded_zones

source/jormungandr/jormungandr/interfaces/v1/opg_excluded_zones.py

@@ -35,6 +35,7 @@
 from jormungandr.interfaces.v1.StatedResource import StatedResource
 from jormungandr.interfaces.v1.make_links import create_external_link
 from jormungandr import i_manager, app
+from jormungandr.excluded_zones_manager import ExcludedZonesManager
 
 
 class OpgExcludedZones(StatedResource):
@@ -74,27 +75,10 @@ def fetch_and_get_data(self, instance, bucket_name, folder, mode=None):
             return {}
         places = []
         logger = logging.getLogger(__name__)
-        args = {"connect_timeout": 2, "read_timeout": 2, "retries": {'max_attempts': 0}}
-        s3_resource = boto3.resource('s3', config=Config(**args))
         try:
-            my_bucket = s3_resource.Bucket(bucket_name)
-            for obj in my_bucket.objects.filter(Prefix="{}/".format(folder)):
-                if not obj.key.endswith('.json'):
-                    continue
-                try:
-                    file_content = obj.get()['Body'].read().decode('utf-8')
-                    json_content = json.loads(file_content)
-                    if json_content.get('instance') != instance.name:
-                        continue
-                    if mode is not None:
-                        if mode not in json_content.get("modes"):
-                            continue
-                    place = self.get_poi_place(instance, json_content)
-                    places.append(place)
-
-                except Exception:
-                    logger.exception("Error on OpgExcludedZones")
-                    continue
+            for json_content in ExcludedZonesManager.get_excluded_zones(instance.name, mode):
+                place = self.get_poi_place(instance, json_content)
+                places.append(place)
         except Exception:
             logger.exception("Error on OpgExcludedZones")
             return {}

source/jormungandr/jormungandr/olympic_site_params_manager.py

@@ -30,6 +30,8 @@
 import logging
 import json
 from collections import namedtuple
+
+from jormungandr.resource_s3_object import ResourceS3Object
 from jormungandr.utils import (
     is_olympic_poi,
     local_str_date_to_utc,
@@ -38,7 +40,6 @@
     str_datetime_utc_to_local,
 )
 import boto3
-from jormungandr import app
 from navitiacommon import type_pb2
 from botocore.client import Config
 from jormungandr import app, memory_cache, cache
@@ -47,15 +48,6 @@
 AttractivityVirtualFallback = namedtuple("AttractivityVirtualFallback", "attractivity, virtual_duration")
 
 
-class ResourceS3Object:
-    def __init__(self, s3_object, instance_name):
-        self.s3_object = s3_object
-        self.instance_name = instance_name
-
-    def __repr__(self):
-        return "{}-{}-{}".format(self.instance_name, self.s3_object.key, self.s3_object.e_tag)
-
-
 def has_applicable_scenario(api_request):
     return bool(api_request.get("olympic_site_params"))
 

source/jormungandr/jormungandr/resource_s3_object.py

@@ -0,0 +1,37 @@
+# Copyright (c) 2001-2024, Hove and/or its affiliates. All rights reserved.
+#
+# This file is part of Navitia,
+#     the software to build cool stuff with public transport.
+#
+# Hope you'll enjoy and contribute to this project,
+#     powered by Hove (www.hove.com).
+# Help us simplify mobility and open public transport:
+#     a non ending quest to the responsive locomotion way of traveling!
+#
+# LICENCE: This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+# Stay tuned using
+# twitter @navitia
+# channel `#navitia` on riot https://riot.im/app/#/room/#navitia:matrix.org
+# https://groups.google.com/d/forum/navitia
+# www.navitia.io
+
+
+class ResourceS3Object:
+    def __init__(self, s3_object, instance_name):
+        self.s3_object = s3_object
+        self.instance_name = instance_name
+
+    def __repr__(self):
+        return "{}-{}-{}".format(self.instance_name, self.s3_object.key, self.s3_object.e_tag)