feat: allow local constraints files (#1105)

CS-SI · May 23, 2024 · 59ccd8b · 59ccd8b
1 parent f343411
commit 59ccd8b
Show file tree

Hide file tree

Showing 9 changed files with 191 additions and 25 deletions.
diff --git a/eodag/config.py b/eodag/config.py
@@ -319,6 +319,7 @@ class OrderStatus(TypedDict):
     discover_queryables: Dict[str, Any]
     metadata_mapping: Dict[str, Union[str, List[str]]]
     free_params: Dict[Any, Any]
+    constraints_file_url: str
     free_text_search_operations: Dict[str, Any]  # ODataV4Search
     metadata_pre_mapping: Dict[str, Any]  # ODataV4Search
     data_request_url: str  # DataRequestSearch

diff --git a/eodag/plugins/search/base.py b/eodag/plugins/search/base.py
@@ -369,17 +369,30 @@ def list_queryables(
             ):
                 del metadata_mapping[param]
 
-        eoadag_queryables = copy_deepcopy(
+        eodag_queryables = copy_deepcopy(
             model_fields_to_annotated(Queryables.model_fields)
         )
-        for k, v in eoadag_queryables.items():
-            field_info = get_args(v)[1] if len(get_args(v)) > 1 else None
-            if not isinstance(field_info, FieldInfo):
+        for k, v in eodag_queryables.items():
+            eodag_queryable_field_info = (
+                get_args(v)[1] if len(get_args(v)) > 1 else None
+            )
+            if not isinstance(eodag_queryable_field_info, FieldInfo):
+                continue
+            # keep default field info of eodag queryables
+            if k in filters and k in queryables:
+                queryable_field_info = (
+                    get_args(queryables[k])[1]
+                    if len(get_args(queryables[k])) > 1
+                    else None
+                )
+                if not isinstance(queryable_field_info, FieldInfo):
+                    continue
+                queryable_field_info.default = filters[k]
+                continue
+            if k in queryables:
                 continue
-            if k in filters:
-                field_info.default = filters[k]
-            if field_info.is_required() or (
-                (field_info.alias or k) in metadata_mapping
+            if eodag_queryable_field_info.is_required() or (
+                (eodag_queryable_field_info.alias or k) in metadata_mapping
             ):
                 queryables[k] = v
 

diff --git a/eodag/resources/constraints/climate-dt.json b/eodag/resources/constraints/climate-dt.json
diff --git a/eodag/resources/constraints/extremes-dt.json b/eodag/resources/constraints/extremes-dt.json
@@ -0,0 +1,8 @@
+[
+    {"class": ["d1"], "dataset": ["extremes-dt"], "expver": ["0001"], "type": ["fc"], "date": ["20240404", "20240405", "20240406", "20240407", "20240408", "20240409", "20240410", "20240411", "20240412"], "time": ["0000"], "stream": ["oper"], "levtype": ["hl"], "levelist": ["100"], "step": ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61", "62", "63", "64", "65", "66", "67", "68", "69", "70", "71", "72", "73", "74", "75", "76", "77", "78", "79", "80", "81", "82", "83", "84", "85", "86", "87", "88", "89", "90", "91", "92", "93", "94", "95", "96"], "param": ["228246", "228247"]},
+    {"class": ["d1"], "dataset": ["extremes-dt"], "expver": ["0001"], "type": ["fc"], "date": ["20240404", "20240405", "20240406", "20240407", "20240408", "20240409", "20240410", "20240411", "20240412"], "time": ["0000"], "stream": ["oper"], "levtype": ["pl"], "levelist": ["1000", "925", "850", "700", "600", "500", "400", "300", "250", "200", "150", "100", "70", "50", "30", "20", "10", "5", "1"], "step": ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61", "62", "63", "64", "65", "66", "67", "68", "69", "70", "71", "72", "73", "74", "75", "76", "77", "78", "79", "80", "81", "82", "83", "84", "85", "86", "87", "88", "89", "90", "91", "92", "93", "94", "95", "96"], "param": ["129", "130", "131", "132", "133", "157"]},
+    {"class": ["d1"], "dataset": ["extremes-dt"], "expver": ["0001"], "type": ["fc"], "date": ["20240404", "20240405", "20240406", "20240407", "20240408", "20240409", "20240410", "20240411", "20240412"], "time": ["0000"], "stream": ["oper"], "levtype": ["sfc"], "step": ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61", "62", "63", "64", "65", "66", "67", "68", "69", "70", "71", "72", "73", "74", "75", "76", "77", "78", "79", "80", "81", "82", "83", "84", "85", "86", "87", "88", "89", "90", "91", "92", "93", "94", "95", "96"], "param": ["31", "34", "78", "134", "136", "137", "151", "165", "166", "167", "168", "3020", "228029", "228050", "228218", "228219", "228221", "228235", "260015"]},
+    {"class": ["d1"], "dataset": ["extremes-dt"], "expver": ["0001"], "type": ["fc"], "date": ["20240404", "20240405", "20240406", "20240407", "20240408", "20240409", "20240410", "20240411", "20240412"], "time": ["0000"], "stream": ["oper"], "levtype": ["sfc"], "step": ["0-1", "1-2", "2-3", "3-4", "4-5", "5-6", "6-7", "7-8", "8-9", "9-10", "10-11", "11-12", "12-13", "13-14", "14-15", "15-16", "16-17", "17-18", "18-19", "19-20", "20-21", "21-22", "22-23", "23-24", "24-25", "25-26", "26-27", "27-28", "28-29", "29-30", "30-31", "31-32", "32-33", "33-34", "34-35", "35-36", "36-37", "37-38", "38-39", "39-40", "40-41", "41-42", "42-43", "43-44", "44-45", "45-46", "46-47", "47-48", "48-49", "49-50", "50-51", "51-52", "52-53", "53-54", "54-55", "55-56", "56-57", "57-58", "58-59", "59-60", "60-61", "61-62", "62-63", "63-64", "64-65", "65-66", "66-67", "67-68", "68-69", "69-70", "70-71", "71-72", "72-73", "73-74", "74-75", "75-76", "76-77", "77-78", "78-79", "79-80", "80-81", "81-82", "82-83", "83-84", "84-85", "85-86", "86-87", "87-88", "88-89", "89-90", "90-91", "91-92", "92-93", "93-94", "94-95", "95-96"], "param": ["142", "144", "169", "175", "176", "177", "178", "179", "180", "181", "205", "228", "228216"]},
+    {"class": ["d1"], "dataset": ["extremes-dt"], "expver": ["0001"], "type": ["fc"], "date": ["20240404", "20240405", "20240406", "20240407", "20240408", "20240409", "20240410", "20240411", "20240412"], "time": ["0000"], "stream": ["oper"], "levtype": ["sfc"], "step": ["0-6", "6-12", "12-18", "18-24", "24-30", "30-36", "36-42", "42-48", "48-54", "54-60", "60-66", "66-72", "72-78", "78-84", "84-90", "90-96"], "param": ["228058"]},
+    {"class": ["d1"], "dataset": ["extremes-dt"], "expver": ["0001"], "type": ["fc"], "date": ["20240404", "20240405", "20240406", "20240407", "20240408", "20240409", "20240410", "20240411", "20240412"], "time": ["0000"], "stream": ["wave"], "levtype": ["sfc"], "step": ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61", "62", "63", "64", "65", "66", "67", "68", "69", "70", "71", "72", "73", "74", "75", "76", "77", "78", "79", "80", "81", "82", "83", "84", "85", "86", "87", "88", "89", "90", "91", "92", "93", "94", "95", "96"], "param": ["140221", "140229", "140230", "140231", "140232"]}
+]
diff --git a/eodag/resources/providers.yml b/eodag/resources/providers.yml
@@ -6274,7 +6274,7 @@
     discover_queryables:
       fetch_url: null
       product_type_fetch_url: null
-    constraints_file_url:
+    constraints_file_url: eodag/resources/constraints/{dataset}.json
     metadata_mapping:
       productType: destination-earth
       storageStatus: OFFLINE
@@ -6342,14 +6342,15 @@
         - $.type
   products:
     DT_EXTREMES:
-      class: rd
-      expver: i7yv
+      class: d1
+      dataset: extremes-dt
+      expver: "0001"
       stream: oper
       type: fc
       levtype: sfc
-      step: 0
-      param: 34
-      time: 0000
+      step: "0"
+      param: "31"
+      time: "0000"
     DT_CLIMATE_ADAPTATION:
       activity: ScenarioMIP
       class: d1
@@ -6364,7 +6365,7 @@
       stream: clte
       type: fc
       levtype: sfc
-      time: 0000
+      time: "0000"
   download: !plugin
     type: HTTPDownload
     base_uri: https://polytope.lumi.apps.dte.destination-earth.eu/api/v1/

diff --git a/eodag/utils/constraints.py b/eodag/utils/constraints.py
@@ -17,15 +17,17 @@
 # limitations under the License.
 import copy
 import logging
+import os
 from typing import TYPE_CHECKING, Any, Dict, List, Set, Union
 
 import requests
 
 from eodag.api.product.metadata_mapping import get_provider_queryable_key
 from eodag.plugins.apis.base import Api
 from eodag.plugins.search.base import Search
-from eodag.utils import HTTP_REQ_TIMEOUT, USER_AGENT, deepcopy
+from eodag.utils import HTTP_REQ_TIMEOUT, USER_AGENT, deepcopy, path_to_uri
 from eodag.utils.exceptions import TimeOutError, ValidationError
+from eodag.utils.requests import LocalFileAdapter
 
 if TYPE_CHECKING:
     from requests.auth import AuthBase
@@ -78,6 +80,13 @@ def get_constraint_queryables_with_additional_params(
                 params_available[param] = True
                 if value in constraint[provider_key]:
                     params_matched[param] = True
+                elif isinstance(value, str):
+                    # for Copernicus providers, values can be multiple and represented with a string
+                    # separated by slashes (example: time = "0000/0100/0200")
+                    values = value.split("/")
+                    params_matched[param] = all(
+                        [v in constraint[provider_key] for v in values]
+                    )
                 values_available[param].update(constraint[provider_key])
         # match with default values of params
         for default_param, default_value in defaults.items():
@@ -171,18 +180,22 @@ def fetch_constraints(
     :rtype: List[Dict[Any, Any]]
     """
     try:
+        session = requests.Session()
+        if not constraints_url.lower().startswith("http"):
+            constraints_url = path_to_uri(os.path.abspath(constraints_url))
+            session.mount("file://", LocalFileAdapter())
         headers = USER_AGENT
         logger.debug("fetching constraints from %s", constraints_url)
         if hasattr(plugin, "auth"):
             auth = plugin.auth if isinstance(plugin.auth, AuthBase) else None
-            res = requests.get(
+            res = session.get(
                 constraints_url,
                 headers=headers,
                 auth=auth,
                 timeout=HTTP_REQ_TIMEOUT,
             )
         else:
-            res = requests.get(
+            res = session.get(
                 constraints_url, headers=headers, timeout=HTTP_REQ_TIMEOUT
             )
         res.raise_for_status()

diff --git a/eodag/utils/requests.py b/eodag/utils/requests.py
@@ -0,0 +1,89 @@
+# -*- coding: utf-8 -*-
+# Copyright 2018, CS GROUP - France, https://www.csgroup.eu/
+#
+# This file is part of EODAG project
+#     https://www.github.com/CS-SI/EODAG
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import annotations
+
+import os
+from typing import Any, Tuple
+
+import requests
+
+from eodag.utils import uri_to_path
+from eodag.utils.exceptions import RequestError
+
+
+class LocalFileAdapter(requests.adapters.BaseAdapter):
+    """Protocol Adapter to allow Requests to GET file:// URLs inspired
+    by https://stackoverflow.com/questions/10123929/fetch-a-file-from-a-local-url-with-python-requests/27786580
+    `LocalFileAdapter` class available for the moment (on the 2024-04-22)
+    """
+
+    @staticmethod
+    def _chkpath(method: str, path: str) -> Tuple[int, str]:
+        """Return an HTTP status for the given filesystem path.
+
+        :param method: method of the request
+        :type method: str
+        :param path: path of the given file
+        :type path: str
+        :returns: HTTP status and its associated message
+        :rtype: Tuple[int, str]
+        """
+        if method.lower() in ("put", "delete"):
+            return 501, "Not Implemented"  # TODO
+        elif method.lower() not in ("get", "head"):
+            return 405, "Method Not Allowed"
+        elif os.path.isdir(path):
+            return 400, "Path Not A File"
+        elif not os.path.isfile(path):
+            return 404, "File Not Found"
+        elif not os.access(path, os.R_OK):
+            return 403, "Access Denied"
+        else:
+            return 200, "OK"
+
+    def send(self, req: requests.PreparedRequest, **kwargs: Any) -> requests.Response:
+        """Wraps a file, described in request, in a Response object.
+
+        :param req: The PreparedRequest being "sent".
+        :type req: :class:`~requests.PreparedRequest`
+        :param kwargs: (not used) additionnal arguments of the request
+        :type kwargs: Any
+        :returns: a Response object containing the file
+        :rtype: :class:`~requests.Response`
+        """
+        response = requests.Response()
+
+        path_url = uri_to_path(req.url)
+
+        if req.method is None or req.url is None:
+            raise RequestError("Method or url of the request is missing")
+        response.status_code, response.reason = self._chkpath(req.method, path_url)
+        if response.status_code == 200 and req.method.lower() != "head":
+            try:
+                response.raw = open(path_url, "rb")
+            except (OSError, IOError) as err:
+                response.status_code = 500
+                response.reason = str(err)
+        response.url = req.url
+        response.request = req
+
+        return response
+
+    def close(self):
+        """Closes without cleaning up adapter specific items."""
+        pass
diff --git a/tests/units/test_http_server.py b/tests/units/test_http_server.py
@@ -1426,22 +1426,23 @@ def test_stac_queryables_type(self):
         self.assertListEqual(["string", "null"], processing_level["type"])
         self.assertIsNone(processing_level["min"])
 
-    @mock.patch("eodag.utils.constraints.requests.get", autospec=True)
+    @mock.patch("eodag.utils.constraints.requests.Session.get", autospec=True)
     def test_product_type_queryables_from_constraints(
-        self, mock_requests_constraints: Mock
+        self, mock_requests_session_constraints: Mock
     ):
         constraints_path = os.path.join(TEST_RESOURCES_PATH, "constraints.json")
         with open(constraints_path) as f:
             constraints = json.load(f)
-        mock_requests_constraints.return_value = MockResponse(
+        mock_requests_session_constraints.return_value = MockResponse(
             constraints, status_code=200
         )
         res = self._request_valid(
             "collections/ERA5_SL/queryables?provider=cop_cds",
             check_links=False,
         )
 
-        mock_requests_constraints.assert_called_once_with(
+        mock_requests_session_constraints.assert_called_once_with(
+            mock.ANY,
             "http://datastore.copernicus-climate.eu/c3s/published-forms/c3sprod/"
             "reanalysis-era5-single-levels/constraints.json",
             headers=USER_AGENT,

diff --git a/tests/units/test_search_plugins.py b/tests/units/test_search_plugins.py
@@ -1935,14 +1935,14 @@ def test_plugins_search_buildsearchresult_with_custom_producttype(self):
             except Exception:
                 assert eoproduct.properties[param] == self.custom_query_params[param]
 
-    @mock.patch("eodag.utils.constraints.requests.get", autospec=True)
+    @mock.patch("eodag.utils.constraints.requests.Session.get", autospec=True)
     def test_plugins_search_buildsearchresult_discover_queryables(
-        self, mock_requests_constraints
+        self, mock_requests_session_constraints
     ):
         constraints_path = os.path.join(TEST_RESOURCES_PATH, "constraints.json")
         with open(constraints_path) as f:
             constraints = json.load(f)
-        mock_requests_constraints.return_value = MockResponse(
+        mock_requests_session_constraints.return_value = MockResponse(
             constraints, status_code=200
         )
         queryables = self.search_plugin.discover_queryables(
@@ -1961,3 +1961,30 @@ def test_plugins_search_buildsearchresult_discover_queryables(
         self.assertEqual("a", queryable.__metadata__[0].get_default())
         queryable = queryables.get("month")
         self.assertTrue(queryable.__metadata__[0].is_required())
+
+    def test_plugins_search_buildsearchresult_discover_queryables_with_local_constraints_file(
+        self,
+    ):
+        constraints_path = os.path.join(TEST_RESOURCES_PATH, "constraints.json")
+        tmp_search_constraints_file_url = self.search_plugin.config.constraints_file_url
+        self.search_plugin.config.constraints_file_url = constraints_path
+
+        queryables = self.search_plugin.discover_queryables(
+            productType="CAMS_EU_AIR_QUALITY_RE"
+        )
+        self.assertEqual(11, len(queryables))
+        self.assertIn("variable", queryables)
+        self.assertNotIn("metadata_mapping", queryables)
+        # with additional param
+        queryables = self.search_plugin.discover_queryables(
+            productType="CAMS_EU_AIR_QUALITY_RE",
+            variable="a",
+        )
+        self.assertEqual(11, len(queryables))
+        queryable = queryables.get("variable")
+        self.assertEqual("a", queryable.__metadata__[0].get_default())
+        queryable = queryables.get("month")
+        self.assertTrue(queryable.__metadata__[0].is_required())
+
+        # restore configuration
+        self.search_plugin.config.constraints_file_url = tmp_search_constraints_file_url