From 185cd340b05afe4d7a0ed94b612c7d21944db2b7 Mon Sep 17 00:00:00 2001 From: Julia Lahovnik Date: Thu, 1 Jun 2023 15:10:20 +0200 Subject: [PATCH 01/11] feat: enable parallel download of assets new incoming download requests can now be processed before a previous one is finished --- eodag/rest/server.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/eodag/rest/server.py b/eodag/rest/server.py index 026635be9..976217cde 100755 --- a/eodag/rest/server.py +++ b/eodag/rest/server.py @@ -397,12 +397,10 @@ async def stac_collections_item(collection_id, item_id, request: Request): @router.get("/collections/{collection_id}/items/{item_id}/download", tags=["Data"]) -async def stac_collections_item_download(collection_id, item_id, request: Request): +def stac_collections_item_download(collection_id, item_id, request: Request): """STAC collection item local download""" - try: - body = await request.json() - except JSONDecodeError: - body = {} + + body = {} arguments = dict(request.query_params, **body) provider = arguments.pop("provider", None) From 752d44c83f34dcd1a2328b3a7a79c34ea97e2d18 Mon Sep 17 00:00:00 2001 From: Julia Lahovnik Date: Tue, 6 Jun 2023 13:47:41 +0200 Subject: [PATCH 02/11] feat: enable parallel request for several endpoints requests to the search, collections and items endpoints are now not blocking the thread anymore --- eodag/rest/server.py | 52 ++++++++++++++++++++++---------------------- eodag/rest/utils.py | 4 ++-- 2 files changed, 28 insertions(+), 28 deletions(-) diff --git a/eodag/rest/server.py b/eodag/rest/server.py index 976217cde..6183a5cbb 100755 --- a/eodag/rest/server.py +++ b/eodag/rest/server.py @@ -22,6 +22,7 @@ from contextlib import asynccontextmanager from distutils import dist from json.decoder import JSONDecodeError +from typing import Union import pkg_resources from fastapi import APIRouter as FastAPIRouter @@ -32,6 +33,7 @@ from fastapi.responses import FileResponse, ORJSONResponse from fastapi.types import Any, Callable, DecoratedCallable from starlette.exceptions import HTTPException as StarletteHTTPException +from pydantic import BaseModel from eodag.config import load_stac_api_config from eodag.rest.utils import ( @@ -271,16 +273,25 @@ def stac_extension_oseo(request: Request): return jsonable_encoder(response) +class SearchBody(BaseModel): + provider: str + collections: str + datetime: Union[str, None] = None + bbox: Union[str, None] = None + limit: Union[int, None] = 10 + page: Union[int, None] = 1 + @router.get("/search", tags=["STAC"]) @router.post("/search", tags=["STAC"]) -async def stac_search(request: Request): +def stac_search(request: Request, search_body: SearchBody = None): """STAC collections items""" url = request.state.url url_root = request.state.url_root - try: - body = await request.json() - except JSONDecodeError: + + if search_body is None: body = {} + else: + body = vars(search_body) arguments = dict(request.query_params, **body) provider = arguments.pop("provider", None) @@ -288,7 +299,6 @@ async def stac_search(request: Request): response = search_stac_items( url=url, arguments=arguments, root=url_root, provider=provider ) - resp = ORJSONResponse( content=response, status_code=200, media_type="application/json" ) @@ -296,17 +306,15 @@ async def stac_search(request: Request): @router.get("/collections", tags=["Capabilities"]) -async def collections(request: Request): +def collections(request: Request): """STAC collections Can be filtered using parameters: instrument, platform, platformSerialIdentifier, sensorType, processingLevel """ url = request.state.url url_root = request.state.url_root - try: - body = await request.json() - except JSONDecodeError: - body = {} + + body = {} arguments = dict(request.query_params, **body) provider = arguments.pop("provider", None) @@ -316,19 +324,16 @@ async def collections(request: Request): arguments=arguments, provider=provider, ) - return jsonable_encoder(response) @router.get("/collections/{collection_id}/items", tags=["Data"]) -async def stac_collections_items(collection_id, request: Request): +def stac_collections_items(collection_id, request: Request): """STAC collections items""" url = request.state.url url_root = request.state.url_root - try: - body = await request.json() - except JSONDecodeError: - body = {} + + body = {} arguments = dict(request.query_params, **body) provider = arguments.pop("provider", None) @@ -339,19 +344,16 @@ async def stac_collections_items(collection_id, request: Request): provider=provider, catalogs=[collection_id], ) - return jsonable_encoder(response) @router.get("/collections/{collection_id}", tags=["Capabilities"]) -async def collection_by_id(collection_id, request: Request): +def collection_by_id(collection_id, request: Request): """STAC collection by id""" url = request.state.url url_root = request.state.url_root - try: - body = await request.json() - except JSONDecodeError: - body = {} + + body = {} arguments = dict(request.query_params, **body) provider = arguments.pop("provider", None) @@ -370,10 +372,8 @@ async def stac_collections_item(collection_id, item_id, request: Request): """STAC collection item by id""" url = request.state.url url_root = request.state.url_root - try: - body = await request.json() - except JSONDecodeError: - body = {} + + body = {} arguments = dict(request.query_params, **body) provider = arguments.pop("provider", None) diff --git a/eodag/rest/utils.py b/eodag/rest/utils.py index c424a6470..04946b1aa 100644 --- a/eodag/rest/utils.py +++ b/eodag/rest/utils.py @@ -232,7 +232,7 @@ def get_geometry(arguments): geom = None - if "bbox" in arguments or "box" in arguments: + if ("bbox" in arguments and arguments["bbox"] is not None) or ("box" in arguments and arguments["box"] is not None): # get bbox request_bbox = arguments.pop("bbox", None) or arguments.pop("box", None) if request_bbox and isinstance(request_bbox, str): @@ -688,7 +688,7 @@ def search_stac_items(url, arguments, root="/", catalogs=[], provider=None): arguments.pop("ids") # get datetime - if "datetime" in arguments.keys(): + if "datetime" in arguments.keys() and arguments["datetime"] is not None: dtime_split = arguments.get("datetime", "").split("/") if len(dtime_split) > 1: arguments["dtstart"] = ( From dd67afcfe116f4eebf533eff45fa53d6e74aa7e3 Mon Sep 17 00:00:00 2001 From: Julia Lahovnik Date: Tue, 6 Jun 2023 16:40:27 +0200 Subject: [PATCH 03/11] refactor: use list for collections parameter in search request --- eodag/rest/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eodag/rest/server.py b/eodag/rest/server.py index 6183a5cbb..c8afd1deb 100755 --- a/eodag/rest/server.py +++ b/eodag/rest/server.py @@ -275,7 +275,7 @@ def stac_extension_oseo(request: Request): class SearchBody(BaseModel): provider: str - collections: str + collections: list[str] datetime: Union[str, None] = None bbox: Union[str, None] = None limit: Union[int, None] = 10 From 04f40db2b9aeedfcf93df5a22e50e999a897fb30 Mon Sep 17 00:00:00 2001 From: Julia Lahovnik Date: Tue, 6 Jun 2023 17:00:49 +0200 Subject: [PATCH 04/11] refactor: make provider optional in search request --- eodag/rest/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eodag/rest/server.py b/eodag/rest/server.py index c8afd1deb..6cc7082bb 100755 --- a/eodag/rest/server.py +++ b/eodag/rest/server.py @@ -274,7 +274,7 @@ def stac_extension_oseo(request: Request): class SearchBody(BaseModel): - provider: str + provider: Union[str, None] = None collections: list[str] datetime: Union[str, None] = None bbox: Union[str, None] = None From 012a5d500b973541623d3fd490bce1dcb35cdea7 Mon Sep 17 00:00:00 2001 From: Julia Lahovnik Date: Tue, 20 Jun 2023 10:10:43 +0200 Subject: [PATCH 05/11] docs: add class description --- eodag/rest/server.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/eodag/rest/server.py b/eodag/rest/server.py index 6cc7082bb..56e826468 100755 --- a/eodag/rest/server.py +++ b/eodag/rest/server.py @@ -22,7 +22,7 @@ from contextlib import asynccontextmanager from distutils import dist from json.decoder import JSONDecodeError -from typing import Union +from typing import Union, List import pkg_resources from fastapi import APIRouter as FastAPIRouter @@ -274,13 +274,17 @@ def stac_extension_oseo(request: Request): class SearchBody(BaseModel): + """ + class which describes the body of a search request + """ provider: Union[str, None] = None - collections: list[str] + collections: List[str] datetime: Union[str, None] = None bbox: Union[str, None] = None limit: Union[int, None] = 10 page: Union[int, None] = 1 + @router.get("/search", tags=["STAC"]) @router.post("/search", tags=["STAC"]) def stac_search(request: Request, search_body: SearchBody = None): From e4a8bdd47b3fc0dd90a8b1efe217d86502361b69 Mon Sep 17 00:00:00 2001 From: Julia Lahovnik Date: Tue, 20 Jun 2023 10:45:20 +0200 Subject: [PATCH 06/11] test: fix test --- eodag/rest/server.py | 3 ++- tests/units/test_http_server.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/eodag/rest/server.py b/eodag/rest/server.py index 56e826468..74727c5fe 100755 --- a/eodag/rest/server.py +++ b/eodag/rest/server.py @@ -281,8 +281,9 @@ class which describes the body of a search request collections: List[str] datetime: Union[str, None] = None bbox: Union[str, None] = None - limit: Union[int, None] = 10 + limit: Union[int, None] = 20 page: Union[int, None] = 1 + query: Union[dict, None] = None @router.get("/search", tags=["STAC"]) diff --git a/tests/units/test_http_server.py b/tests/units/test_http_server.py index 72f97057d..e9379ec75 100644 --- a/tests/units/test_http_server.py +++ b/tests/units/test_http_server.py @@ -517,7 +517,7 @@ def test_cloud_cover_post_search(self): protocol="POST", post_data={ "collections": [self.tested_product_type], - "bbox": [0, 43, 1, 44], + "bbox": "0, 43, 1, 44", "query": {"eo:cloud_cover": {"lte": 10}}, }, expected_search_kwargs=dict( From 2fb1b673b22d20ef2a6280057651ad7d47fbf940 Mon Sep 17 00:00:00 2001 From: Julia Lahovnik Date: Tue, 20 Jun 2023 11:35:07 +0200 Subject: [PATCH 07/11] style: fix import order --- eodag/rest/server.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/eodag/rest/server.py b/eodag/rest/server.py index 74727c5fe..face5def9 100755 --- a/eodag/rest/server.py +++ b/eodag/rest/server.py @@ -22,7 +22,7 @@ from contextlib import asynccontextmanager from distutils import dist from json.decoder import JSONDecodeError -from typing import Union, List +from typing import List, Union import pkg_resources from fastapi import APIRouter as FastAPIRouter @@ -32,8 +32,8 @@ from fastapi.openapi.utils import get_openapi from fastapi.responses import FileResponse, ORJSONResponse from fastapi.types import Any, Callable, DecoratedCallable -from starlette.exceptions import HTTPException as StarletteHTTPException from pydantic import BaseModel +from starlette.exceptions import HTTPException as StarletteHTTPException from eodag.config import load_stac_api_config from eodag.rest.utils import ( @@ -277,6 +277,7 @@ class SearchBody(BaseModel): """ class which describes the body of a search request """ + provider: Union[str, None] = None collections: List[str] datetime: Union[str, None] = None From f93d0853dcb0a894a718ddbf2360e183344c925e Mon Sep 17 00:00:00 2001 From: Julia Lahovnik Date: Tue, 20 Jun 2023 11:40:59 +0200 Subject: [PATCH 08/11] style: fix long line --- eodag/rest/utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/eodag/rest/utils.py b/eodag/rest/utils.py index 04946b1aa..de5d168a7 100644 --- a/eodag/rest/utils.py +++ b/eodag/rest/utils.py @@ -232,7 +232,9 @@ def get_geometry(arguments): geom = None - if ("bbox" in arguments and arguments["bbox"] is not None) or ("box" in arguments and arguments["box"] is not None): + if ("bbox" in arguments and arguments["bbox"] is not None) or ( + "box" in arguments and arguments["box"] is not None + ): # get bbox request_bbox = arguments.pop("bbox", None) or arguments.pop("box", None) if request_bbox and isinstance(request_bbox, str): From f6fd5148bc6cbc63632f8cd7509770299a315f0d Mon Sep 17 00:00:00 2001 From: Sylvain Brunato <61419125+sbrunato@users.noreply.github.com> Date: Tue, 20 Jun 2023 17:28:46 +0200 Subject: [PATCH 09/11] Update tests/units/test_http_server.py --- tests/units/test_http_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/units/test_http_server.py b/tests/units/test_http_server.py index e9379ec75..72f97057d 100644 --- a/tests/units/test_http_server.py +++ b/tests/units/test_http_server.py @@ -517,7 +517,7 @@ def test_cloud_cover_post_search(self): protocol="POST", post_data={ "collections": [self.tested_product_type], - "bbox": "0, 43, 1, 44", + "bbox": [0, 43, 1, 44], "query": {"eo:cloud_cover": {"lte": 10}}, }, expected_search_kwargs=dict( From 3218c63e55fcaeb18031beac319da2b2f9103c3d Mon Sep 17 00:00:00 2001 From: Sylvain Brunato <61419125+sbrunato@users.noreply.github.com> Date: Tue, 20 Jun 2023 17:28:59 +0200 Subject: [PATCH 10/11] Update eodag/rest/server.py --- eodag/rest/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eodag/rest/server.py b/eodag/rest/server.py index face5def9..24f31b2c3 100755 --- a/eodag/rest/server.py +++ b/eodag/rest/server.py @@ -279,7 +279,7 @@ class which describes the body of a search request """ provider: Union[str, None] = None - collections: List[str] + collections: Union[List[str], str] datetime: Union[str, None] = None bbox: Union[str, None] = None limit: Union[int, None] = 20 From b62e971c3c4fb27e4c3c1f43d6cd0df637beea64 Mon Sep 17 00:00:00 2001 From: Sylvain Brunato <61419125+sbrunato@users.noreply.github.com> Date: Tue, 20 Jun 2023 17:29:09 +0200 Subject: [PATCH 11/11] Update eodag/rest/server.py --- eodag/rest/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eodag/rest/server.py b/eodag/rest/server.py index 24f31b2c3..fc40a8b65 100755 --- a/eodag/rest/server.py +++ b/eodag/rest/server.py @@ -281,7 +281,7 @@ class which describes the body of a search request provider: Union[str, None] = None collections: Union[List[str], str] datetime: Union[str, None] = None - bbox: Union[str, None] = None + bbox: Union[list, str, None] = None limit: Union[int, None] = 20 page: Union[int, None] = 1 query: Union[dict, None] = None