Skip to content

Commit

Permalink
refactor: estimated_total_number to number_matched
Browse files Browse the repository at this point in the history
  • Loading branch information
sbrunato committed Jun 19, 2024
1 parent 0106109 commit e5ade76
Show file tree
Hide file tree
Showing 15 changed files with 39 additions and 41 deletions.
4 changes: 2 additions & 2 deletions docs/notebooks/api_user_guide/1_overview.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -314,7 +314,7 @@
"cell_type": "markdown",
"metadata": {},
"source": [
"The estimated total number of products matching the search criteria is stored in the `estimated_total_number` property."
"The estimated total number of products matching the search criteria is stored in the `number_matched` property."
]
},
{
Expand All @@ -331,7 +331,7 @@
}
],
"source": [
"print(f\"Got {len(products_first_page)} products and an estimated total number of {products_first_page.estimated_total_number} products.\")"
"print(f\"Got {len(products_first_page)} products and an estimated total number of {products_first_page.number_matched} products.\")"
]
},
{
Expand Down
12 changes: 6 additions & 6 deletions docs/notebooks/api_user_guide/4_search.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -181,7 +181,7 @@
}
],
"source": [
"print(f\"Got a hand on {len(products_first_page)} products and an estimated total number of {products_first_page.estimated_total_number} products available.\")"
"print(f\"Got a hand on {len(products_first_page)} products and an estimated total number of {products_first_page.number_matched} products available.\")"
]
},
{
Expand Down Expand Up @@ -217,7 +217,7 @@
}
],
"source": [
"print(f\"Got a hand on {len(products_another_second_page)} products and an estimated total number of {products_another_second_page.estimated_total_number} products available.\")"
"print(f\"Got a hand on {len(products_another_second_page)} products and an estimated total number of {products_another_second_page.number_matched} products available.\")"
]
},
{
Expand Down Expand Up @@ -325,7 +325,7 @@
}
],
"source": [
"print(f\"Got a hand on {len(products_first_page)} products and an estimated total number of {products_first_page.estimated_total_number} products available.\")"
"print(f\"Got a hand on {len(products_first_page)} products and an estimated total number of {products_first_page.number_matched} products available.\")"
]
},
{
Expand Down Expand Up @@ -369,7 +369,7 @@
"\u001b[0;31mHTTPError\u001b[0m: 400 Client Error: Bad Request for url: https://peps.cnes.fr/resto/api/collections/S2ST/search.json?startDate=malformed_start_date&completionDate=2021-03-31&geometry=POLYGON%20((1.0000%2043.0000,%201.0000%2044.0000,%202.0000%2044.0000,%202.0000%2043.0000,%201.0000%2043.0000))&productType=S2MSI1C&maxRecords=20&page=1",
"\nDuring handling of the above exception, another exception occurred:\n",
"\u001b[0;31mRequestError\u001b[0m Traceback (most recent call last)",
"Cell \u001b[0;32mIn[28], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m products_first_page, estimated_total_number \u001b[38;5;241m=\u001b[39m \u001b[43mdag\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msearch\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mbad_search_criteria\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mraise_errors\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m)\u001b[49m\n",
"Cell \u001b[0;32mIn[28], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m products_first_page, number_matched \u001b[38;5;241m=\u001b[39m \u001b[43mdag\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msearch\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mbad_search_criteria\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mraise_errors\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m)\u001b[49m\n",
"File \u001b[0;32m~/cs/eodag/.venv/lib/python3.12/site-packages/eodag/api/core.py:1089\u001b[0m, in \u001b[0;36mEODataAccessGateway.search\u001b[0;34m(self, page, items_per_page, raise_errors, start, end, geom, locations, provider, **kwargs)\u001b[0m\n\u001b[1;32m 1087\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m i, search_plugin \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28menumerate\u001b[39m(search_plugins):\n\u001b[1;32m 1088\u001b[0m search_plugin\u001b[38;5;241m.\u001b[39mclear()\n\u001b[0;32m-> 1089\u001b[0m search_results, total_results \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_do_search\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 1090\u001b[0m \u001b[43m \u001b[49m\u001b[43msearch_plugin\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1091\u001b[0m \u001b[43m \u001b[49m\u001b[43mcount\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m 1092\u001b[0m \u001b[43m \u001b[49m\u001b[43mraise_errors\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mraise_errors\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1093\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43msearch_kwargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1094\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1095\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(search_results) \u001b[38;5;241m==\u001b[39m \u001b[38;5;241m0\u001b[39m \u001b[38;5;129;01mand\u001b[39;00m i \u001b[38;5;241m<\u001b[39m \u001b[38;5;28mlen\u001b[39m(search_plugins) \u001b[38;5;241m-\u001b[39m \u001b[38;5;241m1\u001b[39m:\n\u001b[1;32m 1096\u001b[0m logger\u001b[38;5;241m.\u001b[39mwarning(\n\u001b[1;32m 1097\u001b[0m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mNo result could be obtained from provider \u001b[39m\u001b[38;5;132;01m{\u001b[39;00msearch_plugin\u001b[38;5;241m.\u001b[39mprovider\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m, \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 1098\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mwe will try to get the data from another provider\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m 1099\u001b[0m )\n",
"File \u001b[0;32m~/cs/eodag/.venv/lib/python3.12/site-packages/eodag/api/core.py:1717\u001b[0m, in \u001b[0;36mEODataAccessGateway._do_search\u001b[0;34m(self, search_plugin, count, raise_errors, **kwargs)\u001b[0m\n\u001b[1;32m 1714\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m need_auth \u001b[38;5;129;01mand\u001b[39;00m auth_plugin \u001b[38;5;129;01mand\u001b[39;00m can_authenticate:\n\u001b[1;32m 1715\u001b[0m search_plugin\u001b[38;5;241m.\u001b[39mauth \u001b[38;5;241m=\u001b[39m auth_plugin\u001b[38;5;241m.\u001b[39mauthenticate()\n\u001b[0;32m-> 1717\u001b[0m res, nb_res \u001b[38;5;241m=\u001b[39m \u001b[43msearch_plugin\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mquery\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcount\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcount\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mauth\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mauth_plugin\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1719\u001b[0m \u001b[38;5;66;03m# Only do the pagination computations when it makes sense. For example,\u001b[39;00m\n\u001b[1;32m 1720\u001b[0m \u001b[38;5;66;03m# for a search by id, we can reasonably guess that the provider will return\u001b[39;00m\n\u001b[1;32m 1721\u001b[0m \u001b[38;5;66;03m# At most 1 product, so we don't need such a thing as pagination\u001b[39;00m\n\u001b[1;32m 1722\u001b[0m page \u001b[38;5;241m=\u001b[39m kwargs\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mpage\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n",
"File \u001b[0;32m~/cs/eodag/.venv/lib/python3.12/site-packages/eodag/plugins/search/qssearch.py:545\u001b[0m, in \u001b[0;36mQueryStringSearch.query\u001b[0;34m(self, product_type, items_per_page, page, count, **kwargs)\u001b[0m\n\u001b[1;32m 542\u001b[0m \u001b[38;5;28;01mdel\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtotal_items_nb\n\u001b[1;32m 543\u001b[0m \u001b[38;5;28;01mdel\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mneed_count\n\u001b[0;32m--> 545\u001b[0m provider_results \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdo_search\u001b[49m\u001b[43m(\u001b[49m\u001b[43mitems_per_page\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mitems_per_page\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 546\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m count \u001b[38;5;129;01mand\u001b[39;00m total_items \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;28mhasattr\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtotal_items_nb\u001b[39m\u001b[38;5;124m\"\u001b[39m):\n\u001b[1;32m 547\u001b[0m total_items \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtotal_items_nb\n",
Expand Down Expand Up @@ -398,7 +398,7 @@
"source": [
"[search_all()](../../api_reference/core.rst#eodag.api.core.EODataAccessGateway.search_all) takes the pain away from thinking about pagination. It returns a [SearchResult](../../api_reference/searchresult.rst#eodag.api.search_result.SearchResult) that contains **all** the products matching the search criteria. It does so by iterating over the pages of a search result (with [search_iter_page()](../../api_reference/core.rst#eodag.api.core.EODataAccessGateway.search_iter_page)) and gathering products. Compared to [search()](../../api_reference/core.rst#eodag.api.core.EODataAccessGateway.search):\n",
"\n",
"* The property [SearchResult.estimated_total_number](../../api_reference/searchresult.rst#eodag.api.search_result.SearchResult) is `None`. The estimate of total number of products available isn't required here, since they all get collected anyway. This also spares some requests to be sent, since the estimate is usually obtained by sending an additional request.\n",
"* The property [SearchResult.number_matched](../../api_reference/searchresult.rst#eodag.api.search_result.SearchResult) is `None`. The estimate of total number of products available isn't required here, since they all get collected anyway. This also spares some requests to be sent, since the estimate is usually obtained by sending an additional request.\n",
"\n",
"* It tries to optimize the number of items/products requested per page. The limit of most providers has been configured in `eodag`, it is used if available (e.g. 500 products per page). If not available, a default value of 50 is used. An arbitrary value can also be used.\n",
"\n",
Expand Down Expand Up @@ -2088,7 +2088,7 @@
],
"source": [
"one_product_cop_dataspace = dag.search(id=product_id, productType=\"S2_MSI_L1C\", provider=\"cop_dataspace\")\n",
"one_product_cop_dataspace, one_product_cop_dataspace.estimated_total_number"
"one_product_cop_dataspace, one_product_cop_dataspace.number_matched"
]
},
{
Expand Down
4 changes: 2 additions & 2 deletions docs/notebooks/tutos/tuto_cds.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,7 @@
"print(\n",
" \"%s product built %s,\\n having variable = %s\\nand model_level=%s\\n\"\n",
" % (\n",
" products_from_product_type.estimated_total_number,\n",
" products_from_product_type.number_matched,\n",
" products_from_product_type[0],\n",
" products_from_product_type[0].properties.get(\"variable\"),\n",
" products_from_product_type[0].properties.get(\"model_level\"),\n",
Expand All @@ -121,7 +121,7 @@
"print(\n",
" \"%s product built %s,\\n having variable = %s\\nand model_level=%s\\n\"\n",
" % (\n",
" products_from_product_type.estimated_total_number,\n",
" products_from_product_type.number_matched,\n",
" products_from_product_type[0],\n",
" products_from_product_type[0].properties.get(\"variable\"),\n",
" products_from_product_type[0].properties.get(\"model_level\"),\n",
Expand Down
4 changes: 2 additions & 2 deletions docs/notebooks/tutos/tuto_ecmwf.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@
"print(\n",
" \"%s product built %s, \\n having param=%s\\n\"\n",
" % (\n",
" products_from_product_type.estimated_total_number,\n",
" products_from_product_type.number_matched,\n",
" products_from_product_type[0],\n",
" products_from_product_type[0].properties[\"param\"],\n",
" )\n",
Expand All @@ -83,7 +83,7 @@
"print(\n",
" \"%s product built %s, \\n having param=%s\"\n",
" % (\n",
" products_from_product_type.estimated_total_number,\n",
" products_from_product_type.number_matched,\n",
" products_from_product_type[0],\n",
" products_from_product_type[0].properties[\"param\"],\n",
" )\n",
Expand Down
2 changes: 1 addition & 1 deletion docs/notebooks/tutos/tuto_meteoblue.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@
"print(\n",
" \"%s product built %s,\\n using queries=%s\\n\"\n",
" % (\n",
" products_from_product_type.estimated_total_number,\n",
" products_from_product_type.number_matched,\n",
" products_from_product_type[0],\n",
" products_from_product_type[0].properties[\"queries\"],\n",
" )\n",
Expand Down
2 changes: 1 addition & 1 deletion docs/notebooks/tutos/tuto_stac_client.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -409,7 +409,7 @@
"query_args = {\"start\": \"2007-05-01\", \"end\": \"2007-05-06\" , \"geom\": search_polygon}\n",
"\n",
"products = dag.search(**query_args)\n",
"print(\"%s product(s) found\" % products.estimated_total_number)"
"print(\"%s product(s) found\" % products.number_matched)"
]
},
{
Expand Down
2 changes: 1 addition & 1 deletion eodag/api/core.py
Original file line number Diff line number Diff line change
Expand Up @@ -1550,7 +1550,7 @@ def _search_by_id(
results[0].product_type = guesses[0]
# reset driver
results[0].driver = results[0].get_driver()
results.estimated_total_number = 1
results.number_matched = 1
return results
elif len(results) > 1:
logger.info(
Expand Down
4 changes: 2 additions & 2 deletions eodag/api/search_result.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,10 +45,10 @@ class SearchResult(UserList):
data: List[EOProduct]

def __init__(
self, products: List[EOProduct], estimated_total_number: Optional[int] = None
self, products: List[EOProduct], number_matched: Optional[int] = None
) -> None:
super(SearchResult, self).__init__(products)
self.estimated_total_number = estimated_total_number
self.number_matched = number_matched

def crunch(self, cruncher: Crunch, **search_params: Any) -> SearchResult:
"""Do some crunching with the underlying EO products.
Expand Down
6 changes: 2 additions & 4 deletions eodag/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -370,11 +370,9 @@ def search_crunch(ctx: Context, **kwargs: Any) -> None:
results = gateway.search(
count=count, page=page, items_per_page=items_per_page, **criteria
)
if results.estimated_total_number is not None:
if results.number_matched is not None:
click.echo(
"Found a total number of {} products".format(
results.estimated_total_number
)
"Found a total number of {} products".format(results.number_matched)
)
click.echo("Returned {} products".format(len(results)))

Expand Down
4 changes: 2 additions & 2 deletions eodag/rest/core.py
Original file line number Diff line number Diff line change
Expand Up @@ -205,7 +205,7 @@ def search_stac_items(
provider=eodag_args.provider,
)
search_results.extend(sr)
search_results.estimated_total_number = len(search_results)
search_results.number_matched = len(search_results)
total = len(search_results)

elif time_interval_overlap(eodag_args, catalog):
Expand All @@ -215,7 +215,7 @@ def search_stac_items(
}

search_results = eodag_api.search(count=True, **criteria)
total = search_results.estimated_total_number
total = search_results.number_matched
if search_request.crunch:
search_results = crunch_products(
search_results, search_request.crunch, **criteria
Expand Down
6 changes: 3 additions & 3 deletions tests/integration/test_core_search.py
Original file line number Diff line number Diff line change
Expand Up @@ -248,7 +248,7 @@ def test_core_search_fallback_find_nothing(

search_result = self.dag.search(productType="S1_SAR_SLC", count=True)
self.assertEqual(len(search_result), 0)
self.assertEqual(search_result.estimated_total_number, 0)
self.assertEqual(search_result.number_matched, 0)
self.assertEqual(
mock_get.call_count + mock_post.call_count + mock_request.call_count,
len(available_providers),
Expand Down Expand Up @@ -422,7 +422,7 @@ def test_core_search_fallback_find_on_fourth_empty_results(self, mock_query):

search_result = self.dag.search(productType="S1_SAR_SLC", count=True)
self.assertEqual(len(search_result), 1)
self.assertEqual(search_result.estimated_total_number, 1)
self.assertEqual(search_result.number_matched, 1)
self.assertEqual(
mock_query.call_count, 4, "4 provider out of 6 must have been requested"
)
Expand Down Expand Up @@ -453,7 +453,7 @@ def test_core_search_fallback_given_provider(self, mock_query):
productType="S1_SAR_SLC", provider="onda", count=True
)
self.assertEqual(len(search_result), 0)
self.assertEqual(search_result.estimated_total_number, 0)
self.assertEqual(search_result.number_matched, 0)
self.assertEqual(
mock_query.call_count,
1,
Expand Down
4 changes: 2 additions & 2 deletions tests/integration/test_core_search_results.py
Original file line number Diff line number Diff line change
Expand Up @@ -293,7 +293,7 @@ def test_core_search_with_count(self, mock_urlopen):
self.dag.providers_config["creodias"].search.pagination["count_tpl"],
mock_urlopen.call_args_list[-1][0][0].full_url,
)
self.assertIsNone(search_results.estimated_total_number)
self.assertIsNone(search_results.number_matched)

# count enabled
search_results = self.dag.search(
Expand All @@ -303,4 +303,4 @@ def test_core_search_with_count(self, mock_urlopen):
self.dag.providers_config["creodias"].search.pagination["count_tpl"],
mock_urlopen.call_args_list[-1][0][0].full_url,
)
self.assertIsNotNone(search_results.estimated_total_number)
self.assertIsNotNone(search_results.number_matched)
Loading

0 comments on commit e5ade76

Please sign in to comment.