test: search count parameter

eodag/plugins/authentication/openid_connect.py

@@ -124,7 +124,7 @@ def _request_new_token(self) -> Dict[str, str]:
 
     def _request_new_token_error(self, e: requests.RequestException) -> Dict[str, str]:
         """Handle RequestException raised by `self._request_new_token()`"""
-        if self.token_info["access_token"]:
+        if self.token_info.get("access_token"):
             # try using already retrieved token if authenticate() fails (OTP use-case)
             if "access_token_expiration" in self.token_info:
                 return {

tests/integration/test_core_search.py

@@ -246,7 +246,7 @@ def test_core_search_fallback_find_nothing(
             ],
         )
 
-        search_result = self.dag.search(productType="S1_SAR_SLC")
+        search_result = self.dag.search(productType="S1_SAR_SLC", count=True)
         self.assertEqual(len(search_result), 0)
         self.assertEqual(search_result.estimated_total_number, 0)
         self.assertEqual(
@@ -325,7 +325,7 @@ def test_core_search_fallback_find_on_first(self, mock_get, mock_request):
 
         mock_get.return_value.json.return_value = peps_resp_search_file_content
 
-        search_result = self.dag.search(productType="S1_SAR_SLC")
+        search_result = self.dag.search(productType="S1_SAR_SLC", count=True)
         self.assertEqual(len(search_result), peps_resp_search_results_count)
         self.assertEqual(
             mock_get.call_count + mock_request.call_count,
@@ -420,7 +420,7 @@ def test_core_search_fallback_find_on_fourth_empty_results(self, mock_query):
             ([EOProduct("onda", dict(geometry="POINT (0 0)", id="a"))], 1),
         ]
 
-        search_result = self.dag.search(productType="S1_SAR_SLC")
+        search_result = self.dag.search(productType="S1_SAR_SLC", count=True)
         self.assertEqual(len(search_result), 1)
         self.assertEqual(search_result.estimated_total_number, 1)
         self.assertEqual(
@@ -449,7 +449,9 @@ def test_core_search_fallback_given_provider(self, mock_query):
 
         mock_query.return_value = ([], 0)
 
-        search_result = self.dag.search(productType="S1_SAR_SLC", provider="onda")
+        search_result = self.dag.search(
+            productType="S1_SAR_SLC", provider="onda", count=True
+        )
         self.assertEqual(len(search_result), 0)
         self.assertEqual(search_result.estimated_total_number, 0)
         self.assertEqual(

tests/integration/test_search_stac_static.py

@@ -157,7 +157,7 @@ def test_search_stac_static_load_root_recursive(self):
     )
     def test_search_stac_static(self, mock_fetch_product_types_list):
         """Use StaticStacSearch plugin to search all items"""
-        search_result = self.dag.search()
+        search_result = self.dag.search(count=True)
         self.assertEqual(len(search_result), self.root_cat_len)
         self.assertEqual(search_result.estimated_total_number, self.root_cat_len)
         for item in search_result:
@@ -281,7 +281,7 @@ def test_search_stac_static_crunch_filter_date(self):
     )
     def test_search_stac_static_by_date(self, mock_fetch_product_types_list):
         """Use StaticStacSearch plugin to search by date"""
-        filtered_sr = self.dag.search(start="2018-01-01", end="2019-01-01")
+        filtered_sr = self.dag.search(start="2018-01-01", end="2019-01-01", count=True)
         self.assertEqual(len(filtered_sr), self.child_cat_len)
         self.assertEqual(filtered_sr.estimated_total_number, self.child_cat_len)
         for item in filtered_sr:
@@ -352,9 +352,7 @@ def test_search_stac_static_crunch_filter_overlap(self):
     )
     def test_search_stac_static_by_geom(self, mock_fetch_product_types_list):
         """Use StaticStacSearch plugin to search by geometry"""
-        search_result = self.dag.search(
-            geom=self.extent_big,
-        )
+        search_result = self.dag.search(geom=self.extent_big, count=True)
         self.assertEqual(len(search_result), 3)
         self.assertEqual(search_result.estimated_total_number, 3)
 
@@ -390,7 +388,7 @@ def test_search_stac_static_crunch_filter_property(self):
     )
     def test_search_stac_static_by_property(self, mock_fetch_product_types_list):
         """Use StaticStacSearch plugin to search by property"""
-        search_result = self.dag.search(orbitNumber=110)
+        search_result = self.dag.search(orbitNumber=110, count=True)
         self.assertEqual(len(search_result), 3)
         self.assertEqual(search_result.estimated_total_number, 3)
 
@@ -399,7 +397,7 @@ def test_search_stac_static_by_property(self, mock_fetch_product_types_list):
     )
     def test_search_stac_static_by_cloudcover(self, mock_fetch_product_types_list):
         """Use StaticStacSearch plugin to search by cloud cover"""
-        search_result = self.dag.search(cloudCover=10)
+        search_result = self.dag.search(cloudCover=10, count=True)
         self.assertEqual(len(search_result), 1)
         self.assertEqual(search_result.estimated_total_number, 1)
 

tests/test_cli.py

@@ -208,6 +208,7 @@ def test_eodag_search_bbox_valid(self, dag):
                 productType=product_type,
                 id=None,
                 locations=None,
+                count=False,
             )
 
     def test_eodag_search_geom_wkt_invalid(self):
@@ -254,6 +255,7 @@ def test_eodag_search_geom_wkt_valid(self, dag):
                 productType=product_type,
                 id=None,
                 locations=None,
+                count=False,
             )
 
     def test_eodag_search_bbox_geom_mutually_exclusive(self):
@@ -337,7 +339,7 @@ def test_eodag_search_with_cruncher(self, dag):
                 user_conf_file_path=conf_file, locations_conf_path=None
             )
             api_obj.search.assert_called_once_with(
-                items_per_page=DEFAULT_ITEMS_PER_PAGE, page=1, **criteria
+                count=False, items_per_page=DEFAULT_ITEMS_PER_PAGE, page=1, **criteria
             )
             api_obj.crunch.assert_called_once_with(
                 search_results, search_criteria=criteria, **{cruncher: {}}
@@ -440,6 +442,7 @@ def test_eodag_search_query(self, dag):
                 foo="1",
                 bar=["2", "3"],
                 locations=None,
+                count=False,
             )
 
     @mock.patch("eodag.cli.EODataAccessGateway", autospec=True)
@@ -475,6 +478,7 @@ def test_eodag_search_locations(self, dag):
                 sensorType=None,
                 id=None,
                 locations={"country": "FRA", "continent": "Africa"},
+                count=False,
             )
 
     @mock.patch("eodag.cli.EODataAccessGateway", autospec=True)
@@ -514,6 +518,7 @@ def test_eodag_search_start_date(self, dag):
                 sensorType=None,
                 id=None,
                 locations=None,
+                count=False,
             )
 
     @mock.patch("eodag.cli.EODataAccessGateway", autospec=True)
@@ -535,6 +540,7 @@ def test_eodag_search_stop_date(self, dag):
                     product_type,
                     "--end",
                     stop_date_str,
+                    "--count",
                 ],
             )
             api_obj = dag.return_value
@@ -553,6 +559,7 @@ def test_eodag_search_stop_date(self, dag):
                 sensorType=None,
                 id=None,
                 locations=None,
+                count=True,
             )
 
     @mock.patch("eodag.cli.EODataAccessGateway", autospec=True)
@@ -576,7 +583,8 @@ def test_eodag_search_locs(self, dag):
             )
 
             dag.assert_called_once_with(
-                user_conf_file_path=conf_file, locations_conf_path=locs_file
+                user_conf_file_path=conf_file,
+                locations_conf_path=locs_file,
             )
 
     def test_eodag_list_product_type_ok(self):

tests/test_end_to_end.py

@@ -252,8 +252,7 @@ def execute_search(
             search_criteria["items_per_page"] = items_per_page
         if page:
             search_criteria["page"] = page
-        self.eodag.set_preferred_provider(provider)
-        results, nb_results = self.eodag.search(**search_criteria)
+        results = self.eodag.search(provider=provider, **search_criteria)
         if offline:
             results = [
                 prod
@@ -582,7 +581,7 @@ def test_search_by_tile(self):
         tile_id = "31TCJ"
 
         for provider, product_type in supported_providers_product_types:
-            products, _ = self.eodag.search(
+            products = self.eodag.search(
                 productType=product_type,
                 start="2021-06-01",
                 end="2021-06-30",
@@ -795,7 +794,7 @@ def test_end_to_end_complete_peps(self):
         # Search for products that are ONLINE and as small as possible
         today = datetime.date.today()
         month_span = datetime.timedelta(weeks=4)
-        search_results, _ = self.eodag.search(
+        search_results = self.eodag.search(
             productType="S2_MSI_L1C",
             start=(today - month_span).isoformat(),
             end=today.isoformat(),
@@ -981,7 +980,7 @@ def test_end_to_end_wrong_credentials_peps(self):
     def test_end_to_end_wrong_apikey_search_aws_eos(self):
         self.eodag.set_preferred_provider(AWSEOS_SEARCH_ARGS[0])
         with self.assertRaises(AuthenticationError):
-            results, _ = self.eodag.search(
+            self.eodag.search(
                 raise_errors=True,
                 **dict(
                     zip(["productType", "start", "end", "geom"], AWSEOS_SEARCH_ARGS[1:])
@@ -1009,7 +1008,7 @@ def test_end_to_end_good_apikey_wrong_credentials_aws_eos(self):
                 user_conf_file_path=os.path.join(TEST_RESOURCES_PATH, "user_conf.yml")
             )
             eodag.set_preferred_provider(AWSEOS_SEARCH_ARGS[0])
-            results, nb_results = eodag.search(
+            results = eodag.search(
                 raise_errors=True,
                 **dict(
                     zip(["productType", "start", "end", "geom"], AWSEOS_SEARCH_ARGS[1:])
@@ -1045,7 +1044,7 @@ def test_end_to_end_wrong_credentials_search_usgs(self):
         # It should already fail while searching for the products.
         self.eodag.set_preferred_provider(USGS_RECENT_SEARCH_ARGS[0])
         with self.assertRaises(AuthenticationError):
-            results, _ = self.eodag.search(
+            self.eodag.search(
                 raise_errors=True,
                 **dict(
                     zip(
@@ -1059,7 +1058,7 @@ def test_end_to_end_wrong_credentials_search_meteoblue(self):
         # It should already fail while searching for the products.
         self.eodag.set_preferred_provider(METEOBLUE_SEARCH_ARGS[0])
         with self.assertRaises(AuthenticationError):
-            results, _ = self.eodag.search(
+            self.eodag.search(
                 raise_errors=True,
                 **dict(
                     zip(
@@ -1073,7 +1072,7 @@ def test_end_to_end_wrong_credentials_search_hydroweb_next(self):
         # It should already fail while searching for the products.
         self.eodag.set_preferred_provider(HYDROWBEB_NEXT_SEARCH_ARGS[0])
         with self.assertRaises(AuthenticationError):
-            results, _ = self.eodag.search(
+            self.eodag.search(
                 raise_errors=True,
                 **dict(
                     zip(
@@ -1087,7 +1086,7 @@ def test_end_to_end_wrong_credentials_search_wekeo(self):
         # It should already fail while searching for the products.
         self.eodag.set_preferred_provider(WEKEO_SEARCH_ARGS[0])
         with self.assertRaises(AuthenticationError):
-            results, _ = self.eodag.search(
+            self.eodag.search(
                 raise_errors=True,
                 **dict(
                     zip(

tests/units/test_core.py

@@ -2305,7 +2305,7 @@ def test__search_by_id(
 
     @mock.patch("eodag.plugins.search.qssearch.QueryStringSearch", autospec=True)
     def test__do_search_support_itemsperpage_higher_than_maximum(self, search_plugin):
-        """_do_search must create a count query by default"""
+        """_do_search must support itemsperpage higher than maximum"""
         search_plugin.provider = "peps"
         search_plugin.query.return_value = (
             self.search_results.data,  # a list must be returned by .query
@@ -2316,17 +2316,23 @@ class DummyConfig:
             pagination = {"max_items_per_page": 1}
 
         search_plugin.config = DummyConfig()
-        sr = self.dag._do_search(
-            search_plugin=search_plugin,
-            items_per_page=2,
-        )
-        self.assertIsInstance(sr, SearchResult)
-        self.assertEqual(len(sr), self.search_results_size)
-        self.assertEqual(sr.estimated_total_number, self.search_results_size)
+        with self.assertLogs(level="WARNING") as cm:
+            sr = self.dag._do_search(
+                count=True,
+                search_plugin=search_plugin,
+                items_per_page=2,
+            )
+            self.assertIsInstance(sr, SearchResult)
+            self.assertEqual(len(sr), self.search_results_size)
+            self.assertEqual(sr.estimated_total_number, self.search_results_size)
+            self.assertIn(
+                "Try to lower the value of 'items_per_page'",
+                str(cm.output),
+            )
 
     @mock.patch("eodag.plugins.search.qssearch.QueryStringSearch", autospec=True)
-    def test__do_search_counts_by_default(self, search_plugin):
-        """_do_search must create a count query by default"""
+    def test__do_search_counts(self, search_plugin):
+        """_do_search must create a count query if specified"""
         search_plugin.provider = "peps"
         search_plugin.query.return_value = (
             self.search_results.data,  # a list must be returned by .query
@@ -2337,7 +2343,7 @@ class DummyConfig:
             pagination = {}
 
         search_plugin.config = DummyConfig()
-        sr = self.dag._do_search(search_plugin=search_plugin)
+        sr = self.dag._do_search(search_plugin=search_plugin, count=True)
         self.assertIsInstance(sr, SearchResult)
         self.assertEqual(len(sr), self.search_results_size)
         self.assertEqual(sr.estimated_total_number, self.search_results_size)
@@ -2362,10 +2368,9 @@ class DummyConfig:
 
     @mock.patch("eodag.plugins.search.qssearch.QueryStringSearch", autospec=True)
     def test__do_search_paginated_handle_no_count_returned(self, search_plugin):
-        """_do_search must provide a best estimate when a provider doesn't return a count"""
+        """_do_search must return None as count if provider does not return the count"""
         search_plugin.provider = "peps"
-        # If the provider doesn't return a count, .query returns 0
-        search_plugin.query.return_value = (self.search_results.data, 0)
+        search_plugin.query.return_value = (self.search_results.data, None)
 
         class DummyConfig:
             pagination = {}
@@ -2374,51 +2379,17 @@ class DummyConfig:
 
         page = 4
         sr = self.dag._do_search(
+            count=True,
             search_plugin=search_plugin,
             page=page,
             items_per_page=2,
         )
         self.assertEqual(len(sr), self.search_results_size)
-        # The count guess is: page * number_of_products_returned
-        self.assertEqual(sr.estimated_total_number, page * self.search_results_size)
-
-    @mock.patch("eodag.plugins.search.qssearch.QueryStringSearch", autospec=True)
-    def test__do_search_paginated_handle_fuzzy_count(self, search_plugin):
-        """_do_search must provide a best estimate when a provider returns a fuzzy count"""
-        search_plugin.provider = "peps"
-        search_plugin.query.return_value = (
-            self.search_results.data * 4,  # 8 products returned
-            22,  # fuzzy number, less than the real total count
-        )
-
-        class DummyConfig:
-            pagination = {}
-
-        search_plugin.config = DummyConfig()
-
-        page = 4
-        items_per_page = 10
-        sr = self.dag._do_search(
-            search_plugin=search_plugin,
-            page=page,
-            items_per_page=items_per_page,
-        )
-        # At page 4 with 10 items_per_page we should have a count of at least 30
-        # products available. However the provider returned 22. We know it's wrong.
-        # So we update the count with our current knowledge: 30 + 8
-        # Note that this estimate could still be largely inferior to the real total
-        # count.
-        expected_estimate = items_per_page * (page - 1) + len(sr)
-        self.assertEqual(len(sr), 8)
-        self.assertEqual(sr.estimated_total_number, expected_estimate)
+        self.assertEqual(sr.estimated_total_number, None)
 
     @mock.patch("eodag.plugins.search.qssearch.QueryStringSearch", autospec=True)
     def test__do_search_paginated_handle_null_count(self, search_plugin):
-        """_do_search must provide a best estimate when a provider returns a null count"""
-        # TODO: check the underlying implementation, it doesn't make so much sense since
-        # this case is already covered with nb_res = len(res) * page. This one uses
-        # nb_res = items_per_page * (page - 1) whick actually makes more sense. Choose
-        # one of them.
+        """_do_search must return provider response even if provider returns a null count"""
         search_plugin.provider = "peps"
         search_plugin.query.return_value = ([], 0)
 
@@ -2430,13 +2401,13 @@ class DummyConfig:
         page = 4
         items_per_page = 10
         sr = self.dag._do_search(
+            count=True,
             search_plugin=search_plugin,
             page=page,
             items_per_page=items_per_page,
         )
-        expected_estimate = items_per_page * (page - 1)
         self.assertEqual(len(sr), 0)
-        self.assertEqual(sr.estimated_total_number, expected_estimate)
+        self.assertEqual(sr.estimated_total_number, 0)
 
     def test__do_search_does_not_raise_by_default(self):
         """_do_search must not raise any error by default"""
@@ -2449,7 +2420,7 @@ class DummySearchPlugin:
             provider = "peps"
             config = DummyConfig()
 
-        sr = self.dag._do_search(search_plugin=DummySearchPlugin())
+        sr = self.dag._do_search(search_plugin=DummySearchPlugin(), count=True)
         self.assertIsInstance(sr, SearchResult)
         self.assertEqual(len(sr), 0)
         self.assertEqual(sr.estimated_total_number, 0)