From 4c373fd081a54f93b1bcda3d669f19aaf1a05c7d Mon Sep 17 00:00:00 2001 From: Sylvain Brunato <61419125+sbrunato@users.noreply.github.com> Date: Thu, 4 May 2023 17:22:17 +0200 Subject: [PATCH] feat: Flask to FastAPI (#701) --- .github/workflows/test.yml | 2 + CONTRIBUTING.rst | 4 +- NOTICE | 4 +- README.rst | 35 +- docker-compose.yml | 1 + docs/notebooks/api_user_guide/4_search.ipynb | 2 +- .../notebooks/api_user_guide/7_download.ipynb | 2 +- .../tutos/tuto_search_location_tile.ipynb | 2 +- docs/stac_rest.rst | 30 +- eodag/cli.py | 38 +- eodag/config.py | 1 - eodag/plugins/apis/base.py | 2 +- eodag/plugins/download/base.py | 2 +- eodag/resources/stac.yml | 2 +- eodag/rest/server.py | 735 ++++++++++-------- eodag/rest/stac.py | 13 +- eodag/rest/utils.py | 32 +- pytest.ini | 2 +- setup.cfg | 5 +- tests/context.py | 1 + ...039_N0206_R110_T21NYC_20171231T201802.json | 2 +- ...041_N0207_R110_T21NYF_20181231T155050.json | 2 +- ...041_N0207_R110_T22NBJ_20181231T155050.json | 2 +- ...451_N0208_R037_T30UUD_20191231T115143.json | 2 +- ...451_N0208_R037_T30VVH_20191231T115143.json | 2 +- tests/units/test_http_server.py | 216 ++++- 26 files changed, 717 insertions(+), 424 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2f719a7ab..d5da06b71 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -97,6 +97,7 @@ jobs: with: path: artifacts/unit-test-results-python3.11-ubuntu-latest/coverage.xml repo_token: ${{ secrets.GITHUB_TOKEN }} + pull_request_number: ${{ github.pull_request.number }} minimum_coverage: 70 fail_below_threshold: false only_changed_files: true @@ -108,6 +109,7 @@ jobs: with: path: artifacts/unit-test-results-python3.11-windows-latest/coverage.xml repo_token: ${{ secrets.GITHUB_TOKEN }} + pull_request_number: ${{ github.pull_request.number }} minimum_coverage: 70 fail_below_threshold: false only_changed_files: true diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 279d32267..02a8833a9 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -90,7 +90,7 @@ The documentation of EODAG consists of: `Markdown markup language `_ and Python code -`Sphinx `_ is used to create the website from these files. +`Sphinx `_ is used to create the website from these files. `nbsphinx `_ is a Sphinx extension that can parse Jupyter Notebooks. It can also execute notebooks. The notebooks used to document EODAG are not executed by default (see the `conf.py` file), to avoid @@ -119,7 +119,7 @@ If not, the outputs and the widgets (e.g. progress bar) won't be displayed in th `sphinx-autobuild `_ can be installed to rebuild Sphinx documentation on changes, with live-reload in the browser. Run it from the repository root with ``sphinx-autobuild docs docs/_build/html/`` -`Read the Docs `_ is a service that uses Sphinx to build a documentation website, +`Read the Docs `_ is a service that uses Sphinx to build a documentation website, which it then hosts for free for open source projects, such as EODAG. Release EODAG diff --git a/NOTICE b/NOTICE index 59fa3d2f6..89c14b15b 100644 --- a/NOTICE +++ b/NOTICE @@ -39,9 +39,9 @@ https://github.com/jswhit/pyproj https://github.com/kapadia/usgs https://github.com/jupyter-widgets/ipyleaflet https://github.com/GeospatialPython/pyshp -https://github.com/flasgger/flasgger https://github.com/python-visualization/folium https://github.com/Unidata/netcdf4-python +https://github.com/tiangolo/fastapi ================================================================ @@ -71,7 +71,7 @@ https://github.com/mapbox/rasterio https://github.com/lxml/lxml https://github.com/jupyter-widgets/ipywidgets https://github.com/jupyter/jupyter -https://github.com/pallets/flask +https://github.com/encode/uvicorn The function slugify, located at eodag/utils/__init__.py is a modified version of the function with the same name from the Django Project, licensed under the BSD-3-Clause Licence. Follow project link below for more information: diff --git a/README.rst b/README.rst index 661cd2a3d..24b016328 100644 --- a/README.rst +++ b/README.rst @@ -74,7 +74,7 @@ EODAG is available on `PyPI `_: python -m pip install eodag -And with ``conda`` from the `conda-forge channel `_: +And with ``conda`` from the `conda-forge channel `_: .. code-block:: bash @@ -124,17 +124,19 @@ An eodag instance can be exposed through a STAC compliant REST api from the comm Start eodag HTTP server + Set EODAG_CORS_ALLOWED_ORIGINS environment variable to configure Cross- + Origin Resource Sharing allowed origins as comma-separated URLs (e.g. + 'http://somewhere,htttp://somewhere.else'). + Options: -f, --config PATH File path to the user configuration file with its - credentials - -d, --daemon TEXT run in daemon mode - -w, --world run flask using IPv4 0.0.0.0 (all network interfaces), - otherwise bind to 127.0.0.1 (localhost). This maybe - necessary in systems that only run Flask [default: - False] + credentials, default is ~/.config/eodag/eodag.yml + -l, --locs PATH File path to the location shapefiles configuration file + -d, --daemon run in daemon mode + -w, --world run uvicorn using IPv4 0.0.0.0 (all network interfaces), + otherwise bind to 127.0.0.1 (localhost). -p, --port INTEGER The port on which to listen [default: 5000] - --debug Run in debug mode (for development purpose) [default: - False] + --debug Run in debug mode (for development purpose) --help Show this message and exit. # run server @@ -147,13 +149,6 @@ An eodag instance can be exposed through a STAC compliant REST api from the comm "S1_SAR_SLC" "S2_MSI_L1C" "S2_MSI_L2A" - "S3_EFR" - "S3_ERR" - "S3_LAN" - "S3_OLCI_L2LFR" - "S3_OLCI_L2LRR" - "S3_SLSTR_L1RBT" - "S3_SLSTR_L2LST" # search for items $ curl "http://127.0.0.1:5000/search?collections=S2_MSI_L1C&bbox=0,43,1,44&datetime=2018-01-20/2018-01-25" \ @@ -161,17 +156,17 @@ An eodag instance can be exposed through a STAC compliant REST api from the comm 6 # browse for items - $ curl "http://127.0.0.1:5000/S2_MSI_L1C/country/FRA/year/2021/month/01/day/25/cloud_cover/10/items" \ + $ curl "http://127.0.0.1:5000/catalogs/S2_MSI_L1C/country/FRA/year/2021/month/01/day/25/cloud_cover/10/items" \ | jq ".context.matched" 9 # get download link - $ curl "http://127.0.0.1:5000/S2_MSI_L1C/country/FRA/year/2021/month/01/day/25/cloud_cover/10/items" \ + $ curl "http://127.0.0.1:5000/catalogs/S2_MSI_L1C/country/FRA/year/2021/month/01/day/25/cloud_cover/10/items" \ | jq ".features[0].assets.downloadLink.href" - "http://127.0.0.1:5000/S2_MSI_L1C/country/FRA/year/2021/month/01/day/25/cloud_cover/10/items/S2A_MSIL1C_20210125T105331_N0209_R051_T31UCR_20210125T130733/download" + "http://127.0.0.1:5000/catalogs/S2_MSI_L1C/country/FRA/year/2021/month/01/day/25/cloud_cover/10/items/S2A_MSIL1C_20210125T105331_N0209_R051_T31UCR_20210125T130733/download" # download - $ wget "http://127.0.0.1:5000/S2_MSI_L1C/country/FRA/year/2021/month/01/day/25/cloud_cover/10/items/S2A_MSIL1C_20210125T105331_N0209_R051_T31UCR_20210125T130733/download" + $ wget "http://127.0.0.1:5000/catalogs/S2_MSI_L1C/country/FRA/year/2021/month/01/day/25/cloud_cover/10/items/S2A_MSIL1C_20210125T105331_N0209_R051_T31UCR_20210125T130733/download" ``eodag-server`` is available on `https://hub.docker.com/r/csspace/eodag-server `_: diff --git a/docker-compose.yml b/docker-compose.yml index f57fe2f16..f165f6de1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -21,6 +21,7 @@ services: dockerfile: docker/stac-server.dockerfile environment: - "EODAG_LOGGING=${EODAG_LOGGING}" + - "EODAG_CORS_ALLOWED_ORIGINS=http://127.0.0.1:5001" container_name: stac_server restart: unless-stopped networks: diff --git a/docs/notebooks/api_user_guide/4_search.ipynb b/docs/notebooks/api_user_guide/4_search.ipynb index 1fe169495..b7dffa316 100644 --- a/docs/notebooks/api_user_guide/4_search.ipynb +++ b/docs/notebooks/api_user_guide/4_search.ipynb @@ -2185,7 +2185,7 @@ ], "source": [ "products, estimated_total_number = dag.search(\n", - " cloudCover=10, # cloud cover Less than 10\n", + " cloudCover=10, # cloud cover Less than 10\n", " **default_search_criteria\n", ")" ] diff --git a/docs/notebooks/api_user_guide/7_download.ipynb b/docs/notebooks/api_user_guide/7_download.ipynb index 6dd7a8f63..4317884d9 100644 --- a/docs/notebooks/api_user_guide/7_download.ipynb +++ b/docs/notebooks/api_user_guide/7_download.ipynb @@ -645,7 +645,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "The storage status of a product can be obtained from its `storageStatus` field. The status of an `OFFLINE` product is updated by `eodag` to `STAGING` when ordered and to `ONLINE` when found available." + "The storage status of a product can be obtained from its `storageStatus` field. The status of an `OFFLINE` product is updated by `eodag` to `STAGING` when ordered and to `ONLINE` when found available." ] }, { diff --git a/docs/notebooks/tutos/tuto_search_location_tile.ipynb b/docs/notebooks/tutos/tuto_search_location_tile.ipynb index 3e7992d47..e9e7dd16c 100644 --- a/docs/notebooks/tutos/tuto_search_location_tile.ipynb +++ b/docs/notebooks/tutos/tuto_search_location_tile.ipynb @@ -825,7 +825,7 @@ " tooltip=folium.GeoJsonTooltip(\n", " fields=[\n", " \"title\", # The product's title\n", - " \"tileIdentifier\", # The tile number on the MGRS grid\n", + " \"tileIdentifier\", # The tile number on the MGRS grid\n", " ]\n", " ),\n", ").add_to(fmap)\n", diff --git a/docs/stac_rest.rst b/docs/stac_rest.rst index 8431c121e..3f48103d8 100644 --- a/docs/stac_rest.rst +++ b/docs/stac_rest.rst @@ -23,24 +23,26 @@ Below is the content of the help message of this command (`eodag serve-rest --he Start eodag HTTP server + Set EODAG_CORS_ALLOWED_ORIGINS environment variable to configure Cross- + Origin Resource Sharing allowed origins as comma-separated URLs (e.g. + 'http://somewhere,htttp://somewhere.else'). + Options: -f, --config PATH File path to the user configuration file with its credentials, default is ~/.config/eodag/eodag.yml - -d, --daemon run in daemon mode [default: False] - -w, --world run flask using IPv4 0.0.0.0 (all network interfaces), - otherwise bind to 127.0.0.1 (localhost). This maybe - necessary in systems that only run Flask [default: - False] + -l, --locs PATH File path to the location shapefiles configuration file + -d, --daemon run in daemon mode + -w, --world run uvicorn using IPv4 0.0.0.0 (all network interfaces), + otherwise bind to 127.0.0.1 (localhost). -p, --port INTEGER The port on which to listen [default: 5000] - --debug Run in debug mode (for development purpose) [default: - False] + --debug Run in debug mode (for development purpose) --help Show this message and exit. Searching --------- After you have launched the server, navigate to its home page. For example, for a local -development server launched with ``eodag serve-rest -f --debug``, go to +development server launched with ``eodag serve-rest -f --debug``, go to http://127.0.0.1:5000/service-doc. You will see a documentation of the interface. Available operations are: @@ -86,8 +88,8 @@ EODAG provides additional catalogs that extend browsing/filtering capabilities: Example URLs: -* http://127.0.0.1:5000/S2_MSI_L1C/country : lists available countries -* http://127.0.0.1:5000/S2_MSI_L1C/country/FRA/year/2019/month/10/cloud_cover/10 : catalog referencing S2_MSI_L1C +* http://127.0.0.1:5000/catalogs/S2_MSI_L1C/country : lists available countries +* http://127.0.0.1:5000/catalogs/S2_MSI_L1C/country/FRA/year/2019/month/10/cloud_cover/10 : catalog referencing S2_MSI_L1C products over France, aquired during October 2019, and having 10% maximum cloud cover Browsing over catalogs can be experienced connecting EODAG STAC API to @@ -146,14 +148,14 @@ Example 6 # browse for items - $ curl "http://127.0.0.1:5000/S2_MSI_L1C/country/FRA/year/2021/month/01/day/25/cloud_cover/10/items" \ + $ curl "http://127.0.0.1:5000/catalogs/S2_MSI_L1C/country/FRA/year/2021/month/01/day/25/cloud_cover/10/items" \ | jq ".context.matched" 9 # get download link - $ curl "http://127.0.0.1:5000/S2_MSI_L1C/country/FRA/year/2021/month/01/day/25/cloud_cover/10/items" \ + $ curl "http://127.0.0.1:5000/catalogs/S2_MSI_L1C/country/FRA/year/2021/month/01/day/25/cloud_cover/10/items" \ | jq ".features[0].assets.downloadLink.href" - "http://127.0.0.1:5000/S2_MSI_L1C/country/FRA/year/2021/month/01/day/25/cloud_cover/10/items/S2A_MSIL1C_20210125T105331_N0209_R051_T31UCR_20210125T130733/download" + "http://127.0.0.1:5000/catalogs/S2_MSI_L1C/country/FRA/year/2021/month/01/day/25/cloud_cover/10/items/S2A_MSIL1C_20210125T105331_N0209_R051_T31UCR_20210125T130733/download" # download - $ wget "http://127.0.0.1:5000/S2_MSI_L1C/country/FRA/year/2021/month/01/day/25/cloud_cover/10/items/S2A_MSIL1C_20210125T105331_N0209_R051_T31UCR_20210125T130733/download" + $ wget "http://127.0.0.1:5000/catalogs/S2_MSI_L1C/country/FRA/year/2021/month/01/day/25/cloud_cover/10/items/S2A_MSIL1C_20210125T105331_N0209_R051_T31UCR_20210125T130733/download" diff --git a/eodag/cli.py b/eodag/cli.py index 1208bdbf1..9669d47d7 100755 --- a/eodag/cli.py +++ b/eodag/cli.py @@ -53,6 +53,7 @@ from importlib_metadata import metadata # type: ignore import click +import uvicorn from eodag.api.core import DEFAULT_ITEMS_PER_PAGE, DEFAULT_PAGE, EODataAccessGateway from eodag.utils import parse_qs @@ -621,7 +622,11 @@ def serve_rpc(ctx, host, port, conf): server.serve() -@eodag.command(help="Start eodag HTTP server") +@eodag.command( + help="Start eodag HTTP server\n\n" + "Set EODAG_CORS_ALLOWED_ORIGINS environment variable to configure Cross-Origin Resource Sharing allowed origins as " + "comma-separated URLs (e.g. 'http://somewhere,htttp://somewhere.else')." +) @click.option( "-f", "--config", @@ -644,9 +649,8 @@ def serve_rpc(ctx, host, port, conf): is_flag=True, show_default=True, help=( - "run flask using IPv4 0.0.0.0 (all network interfaces), " + "run uvicorn using IPv4 0.0.0.0 (all network interfaces), " "otherwise bind to 127.0.0.1 (localhost). " - "This maybe necessary in systems that only run Flask" ), ) @click.option( @@ -677,11 +681,6 @@ def serve_rest(ctx, daemon, world, port, config, locs, debug): if locs: os.environ["EODAG_LOCS_CFG_FILE"] = locs - from eodag.rest.server import app, run_swagger, stac_api_config - - # run swagger / service-doc - run_swagger(app=app, config=stac_api_config) - bind_host = "127.0.0.1" if world: bind_host = "0.0.0.0" @@ -693,11 +692,30 @@ def serve_rest(ctx, daemon, world, port, config, locs, debug): if pid == 0: os.setsid() - app.run(threaded=True, host=bind_host, port=port) + uvicorn.run("eodag.rest.server:app", host=bind_host, port=port) else: sys.exit(0) else: - app.run(debug=debug, host=bind_host, port=port) + logging_config = uvicorn.config.LOGGING_CONFIG + if debug: + logging_config["loggers"]["uvicorn"]["level"] = "DEBUG" + logging_config["loggers"]["uvicorn.error"]["level"] = "DEBUG" + logging_config["loggers"]["uvicorn.access"]["level"] = "DEBUG" + logging_config["formatters"]["default"][ + "fmt" + ] = "%(asctime)-15s %(name)-32s [%(levelname)-8s] (%(module)-17s) %(message)s" + logging_config["loggers"]["eodag"] = { + "handlers": ["default"], + "level": "DEBUG" if debug else "INFO", + "propagate": False, + } + uvicorn.run( + "eodag.rest.server:app", + host=bind_host, + port=port, + reload=debug, + log_config=logging_config, + ) @eodag.command( diff --git a/eodag/config.py b/eodag/config.py index ab052fb59..310adfdcf 100644 --- a/eodag/config.py +++ b/eodag/config.py @@ -15,7 +15,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# import copy import logging import os import tempfile diff --git a/eodag/plugins/apis/base.py b/eodag/plugins/apis/base.py index 747cf3dec..e28d539c4 100644 --- a/eodag/plugins/apis/base.py +++ b/eodag/plugins/apis/base.py @@ -48,7 +48,7 @@ class Api(PluginTopic): is built on the MD5 hash of the product's ``remote_location`` attribute (``hashlib.md5(remote_location.encode("utf-8")).hexdigest()``) and whose content is the product's ``remote_location`` attribute itself. - - not try to download a product whose ``location`` attribute already points to an + - not try to download a product whose ``location`` attribute already points to an existing file/directory - not try to download a product if its *record* file exists as long as the expected product's file/directory. If the *record* file only is found, it must be deleted diff --git a/eodag/plugins/download/base.py b/eodag/plugins/download/base.py index a89c0b52f..8b0e6f2bb 100644 --- a/eodag/plugins/download/base.py +++ b/eodag/plugins/download/base.py @@ -67,7 +67,7 @@ class Download(PluginTopic): is built on the MD5 hash of the product's ``remote_location`` attribute (``hashlib.md5(remote_location.encode("utf-8")).hexdigest()``) and whose content is the product's ``remote_location`` attribute itself. - - not try to download a product whose ``location`` attribute already points to an + - not try to download a product whose ``location`` attribute already points to an existing file/directory - not try to download a product if its *record* file exists as long as the expected product's file/directory. If the *record* file only is found, it must be deleted diff --git a/eodag/resources/stac.yml b/eodag/resources/stac.yml index 55508c253..c69551550 100644 --- a/eodag/resources/stac.yml +++ b/eodag/resources/stac.yml @@ -35,7 +35,7 @@ landing_page: href: "{catalog[root]}/api" - rel: service-doc type: "text/html" - href: "{catalog[root]}/service-doc" + href: "{catalog[root]}/api.html" - rel: conformance type: "application/json" href: "{catalog[root]}/conformance" diff --git a/eodag/rest/server.py b/eodag/rest/server.py index c8f37fed5..026635be9 100755 --- a/eodag/rest/server.py +++ b/eodag/rest/server.py @@ -18,21 +18,27 @@ import io import logging import os -import sys import traceback +from contextlib import asynccontextmanager from distutils import dist -from functools import wraps +from json.decoder import JSONDecodeError -import flask -import geojson import pkg_resources -from flasgger import Swagger -from flask import abort, jsonify, make_response, request, send_file +from fastapi import APIRouter as FastAPIRouter +from fastapi import FastAPI, HTTPException, Request +from fastapi.encoders import jsonable_encoder +from fastapi.middleware.cors import CORSMiddleware +from fastapi.openapi.utils import get_openapi +from fastapi.responses import FileResponse, ORJSONResponse +from fastapi.types import Any, Callable, DecoratedCallable +from starlette.exceptions import HTTPException as StarletteHTTPException from eodag.config import load_stac_api_config from eodag.rest.utils import ( download_stac_item_by_id, + eodag_api_init, get_detailled_collections_list, + get_stac_api_version, get_stac_catalogs, get_stac_collection_by_id, get_stac_collections, @@ -41,7 +47,9 @@ get_stac_item_by_id, search_stac_items, ) +from eodag.utils import parse_header, update_nested_dict from eodag.utils.exceptions import ( + AuthenticationError, MisconfiguredError, NoMatchingProductType, NotAvailableError, @@ -52,182 +60,364 @@ logger = logging.getLogger("eodag.rest.server") -app = flask.Flask(__name__) -# eodag metadata -distribution = pkg_resources.get_distribution("eodag") -metadata_str = distribution.get_metadata(distribution.PKG_INFO) -metadata_obj = dist.DistributionMetadata() -metadata_obj.read_pkg_file(io.StringIO(metadata_str)) +class APIRouter(FastAPIRouter): + """API router""" + + def api_route( + self, path: str, *, include_in_schema: bool = True, **kwargs: Any + ) -> Callable[[DecoratedCallable], DecoratedCallable]: + """Creates API route decorator""" + if path == "/": + return super().api_route( + path, include_in_schema=include_in_schema, **kwargs + ) + + if path.endswith("/"): + path = path[:-1] + add_path = super().api_route( + path, include_in_schema=include_in_schema, **kwargs + ) + + alternate_path = path + "/" + add_alternate_path = super().api_route( + alternate_path, include_in_schema=False, **kwargs + ) + + def decorator(func: DecoratedCallable) -> DecoratedCallable: + add_alternate_path(func) + return add_path(func) + + return decorator + + +router = APIRouter() + + +@asynccontextmanager +async def lifespan(app: FastAPI): + """API init and tear-down""" + eodag_api_init() + yield + + +app = FastAPI(lifespan=lifespan, title="EODAG", docs_url="/api.html") # conf from resources/stac_api.yml stac_api_config = load_stac_api_config() -root_catalog = get_stac_catalogs(url="", fetch_providers=False) -stac_api_version = stac_api_config["info"]["version"] -stac_api_config["info"]["title"] = root_catalog["title"] + " / eodag" -stac_api_config["info"]["description"] = ( - root_catalog["description"] - + " (stac-api-spec {})".format(stac_api_version) - + "".join( - [ - "\n - [{0}](/collections/{0}): {1}".format(pt["ID"], pt["abstract"]) - for pt in get_detailled_collections_list(fetch_providers=False) - ] + + +@router.get("/api", tags=["Capabilities"]) +def eodag_openapi(): + """Customized openapi""" + if app.openapi_schema: + return app.openapi_schema + + # eodag metadata + distribution = pkg_resources.get_distribution("eodag") + metadata_str = distribution.get_metadata(distribution.PKG_INFO) + metadata_obj = dist.DistributionMetadata() + metadata_obj.read_pkg_file(io.StringIO(metadata_str)) + + root_catalog = get_stac_catalogs(url="", fetch_providers=False) + stac_api_version = get_stac_api_version() + + openapi_schema = get_openapi( + title=f"{root_catalog['title']} / eodag", + version=getattr(metadata_obj, "version", None), + # description="This is a very custom OpenAPI schema", + routes=app.routes, ) + + # stac_api_config + update_nested_dict(openapi_schema["paths"], stac_api_config["paths"]) + update_nested_dict(openapi_schema["components"], stac_api_config["components"]) + openapi_schema["tags"] = stac_api_config["tags"] + + detailled_collections_list = get_detailled_collections_list(fetch_providers=False) + + openapi_schema["info"]["description"] = ( + root_catalog["description"] + + " (stac-api-spec {})".format(stac_api_version) + + "
Available collections / product types" + + "".join( + [ + f"[{pt['ID']}](/collections/{pt['ID']} '{pt['title']}') - " + for pt in detailled_collections_list + ] + )[:-2] + + "
" + ) + + app.openapi_schema = openapi_schema + return app.openapi_schema + + +app.openapi = eodag_openapi + +# Cross-Origin Resource Sharing +allowed_origins = os.getenv("EODAG_CORS_ALLOWED_ORIGINS") +allowed_origins_list = allowed_origins.split(",") if allowed_origins else [] +app.add_middleware( + CORSMiddleware, + allow_origins=allowed_origins_list, + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], ) -stac_api_config["info"]["version"] = getattr( - metadata_obj, "version", stac_api_config["info"]["version"] -) -stac_api_config["info"]["contact"]["name"] = "EODAG" -stac_api_config["info"]["contact"]["url"] = getattr( - metadata_obj, "url", stac_api_config["info"]["contact"]["url"] -) -stac_api_config["title"] = root_catalog["title"] + " - service-doc" -stac_api_config["specs"] = [ - { - "endpoint": "service-desc", - "route": "/api", - "rule_filter": lambda rule: True, # all in - "model_filter": lambda tag: True, # all in - } -] -stac_api_config["static_url_path"] = "/service-static" -stac_api_config["specs_route"] = "/service-doc/" -stac_api_config.pop("servers", None) - - -def run_swagger(app=None, config=None, merge=True, **kwargs): - """Run `flasgger.Swagger`. - Needs to be run once after `eodag.rest.server` module import. - - :param app: flask application that will run flasgger - :type app: :class:`~flask.Flask` - :param config: flasgger configuration - :type config: dict - :param merge: merge config with flasgger defaults - :type merge: bool - :param kwargs: keyword arguments passed to `flasgger.Swagger` constructor - :type kwargs: Any - :returns: running Swagger object - :rtype: :class:`~flasgger.Swagger` - """ - return Swagger(app=app, config=config, merge=True, **kwargs) -def cross_origin(request_handler): - """Wraps a view to relax the need for csrf token""" +@app.middleware("http") +async def forward_middleware(request: Request, call_next): + """Middleware that handles forward headers and sets request.state.url*""" - @wraps(request_handler) - def wrapper(*args, **kwargs): - resp = make_response(request_handler(*args, **kwargs)) - resp.headers["Access-Control-Allow-Origin"] = "*" - return resp + forwarded_host = request.headers.get("x-forwarded-host", None) + forwarded_proto = request.headers.get("x-forwarded-proto", None) - return wrapper + if "forwarded" in request.headers: + header_forwarded = parse_header(request.headers["forwarded"]) + forwarded_host = header_forwarded.get_param("host", None) or forwarded_host + forwarded_proto = header_forwarded.get_param("proto", None) or forwarded_proto + request.state.url_root = f"{forwarded_proto or request.url.scheme}://{forwarded_host or request.url.netloc}" + request.state.url = f"{request.state.url_root}{request.url.path}" -@app.errorhandler(MisconfiguredError) -@app.errorhandler(NoMatchingProductType) -@app.errorhandler(UnsupportedProductType) -@app.errorhandler(UnsupportedProvider) -@app.errorhandler(ValidationError) -def handle_invalid_usage(error): - """Invalid usage [400] errors handle""" - status_code = 400 - response = jsonify( - {"code": status_code, "name": type(error).__name__, "description": str(error)} - ) - response.status_code = status_code - logger.warning(traceback.format_exc()) + response = await call_next(request) return response -@app.errorhandler(RuntimeError) -@app.errorhandler(Exception) -def handle_internal_error(error): - """Internal [500] errors handle""" - status_code = 500 - response = jsonify( - {"code": status_code, "name": type(error).__name__, "description": str(error)} +@app.exception_handler(StarletteHTTPException) +async def default_exception_handler(request: Request, error): + """Default errors handle""" + description = ( + getattr(error, "description", None) + or getattr(error, "detail", None) + or str(error) + ) + return ORJSONResponse( + status_code=error.status_code, + content={"description": description}, ) - response.status_code = status_code - logger.error(traceback.format_exc()) - return response -@app.errorhandler(NotAvailableError) -@app.errorhandler(404) -def handle_resource_not_found(e): - """Not found [404] errors handle""" - return jsonify(error=str(e)), 404 +@app.exception_handler(MisconfiguredError) +@app.exception_handler(NoMatchingProductType) +@app.exception_handler(UnsupportedProductType) +@app.exception_handler(UnsupportedProvider) +@app.exception_handler(ValidationError) +async def handle_invalid_usage(request: Request, error): + """Invalid usage [400] errors handle""" + logger.warning(traceback.format_exc()) + return await default_exception_handler( + request, + HTTPException( + status_code=400, + detail=f"{type(error).__name__}: {str(error)}", + ), + ) -@app.route("/conformance", methods=["GET"]) -@cross_origin -def conformance(): - """STAC conformance""" +@app.exception_handler(NotAvailableError) +async def handle_resource_not_found(request: Request, error): + """Not found [404] errors handle""" + return await default_exception_handler( + request, + HTTPException( + status_code=404, + detail=f"{type(error).__name__}: {str(error)}", + ), + ) - response = get_stac_conformance() - return jsonify(response), 200 +@app.exception_handler(AuthenticationError) +async def handle_auth_error(request: Request, error): + """Unauthorized [401] errors handle""" + return await default_exception_handler( + request, + HTTPException( + status_code=401, + detail=f"{type(error).__name__}: {str(error)}", + ), + ) -@app.route("/", methods=["GET"]) -@cross_origin -def catalogs_root(): +@router.get("/", tags=["Capabilities"]) +def catalogs_root(request: Request): """STAC catalogs root""" response = get_stac_catalogs( - url=request.url.split("?")[0], - root=request.url_root, + url=request.state.url, + root=request.state.url_root, catalogs=[], - provider=request.args.to_dict().get("provider", None), + provider=request.query_params.get("provider", None), ) - return jsonify(response), 200 + return jsonable_encoder(response) -@app.route("/", methods=["GET"]) -@cross_origin -def stac_catalogs(catalogs): - """Describe the given catalog and list available sub-catalogs - --- - tags: - - Capabilities - parameters: - - name: catalogs - in: path - required: true - description: |- - The catalog's path. +@router.get("/conformance", tags=["Capabilities"]) +def conformance(): + """STAC conformance""" + response = get_stac_conformance() - For a nested catalog, provide the root-related path to the catalog (for example `S2_MSI_L1C/year/2020`) - schema: - type: string - responses: - 200: - description: The catalog's description - content: - application/json: - schema: - $ref: '#/components/schemas/collection' - '500': - $ref: '#/components/responses/ServerError' + return jsonable_encoder(response) + + +@router.get("/extensions/oseo/json-schema/schema.json", include_in_schema=False) +def stac_extension_oseo(request: Request): + """STAC OGC / OpenSearch extension for EO""" + response = get_stac_extension_oseo(url=request.state.url) + + return jsonable_encoder(response) + + +@router.get("/search", tags=["STAC"]) +@router.post("/search", tags=["STAC"]) +async def stac_search(request: Request): + """STAC collections items""" + url = request.state.url + url_root = request.state.url_root + try: + body = await request.json() + except JSONDecodeError: + body = {} + + arguments = dict(request.query_params, **body) + provider = arguments.pop("provider", None) + + response = search_stac_items( + url=url, arguments=arguments, root=url_root, provider=provider + ) + + resp = ORJSONResponse( + content=response, status_code=200, media_type="application/json" + ) + return resp + + +@router.get("/collections", tags=["Capabilities"]) +async def collections(request: Request): + """STAC collections + + Can be filtered using parameters: instrument, platform, platformSerialIdentifier, sensorType, processingLevel """ + url = request.state.url + url_root = request.state.url_root + try: + body = await request.json() + except JSONDecodeError: + body = {} + arguments = dict(request.query_params, **body) + provider = arguments.pop("provider", None) - catalogs = catalogs.strip("/").split("/") - response = get_stac_catalogs( - url=request.url.split("?")[0], - root=request.url_root, - catalogs=catalogs, - provider=request.args.to_dict().get("provider", None), + response = get_stac_collections( + url=url, + root=url_root, + arguments=arguments, + provider=provider, + ) + + return jsonable_encoder(response) + + +@router.get("/collections/{collection_id}/items", tags=["Data"]) +async def stac_collections_items(collection_id, request: Request): + """STAC collections items""" + url = request.state.url + url_root = request.state.url_root + try: + body = await request.json() + except JSONDecodeError: + body = {} + arguments = dict(request.query_params, **body) + provider = arguments.pop("provider", None) + + response = search_stac_items( + url=url, + arguments=arguments, + root=url_root, + provider=provider, + catalogs=[collection_id], + ) + + return jsonable_encoder(response) + + +@router.get("/collections/{collection_id}", tags=["Capabilities"]) +async def collection_by_id(collection_id, request: Request): + """STAC collection by id""" + url = request.state.url + url_root = request.state.url_root + try: + body = await request.json() + except JSONDecodeError: + body = {} + arguments = dict(request.query_params, **body) + provider = arguments.pop("provider", None) + + response = get_stac_collection_by_id( + url=url, + root=url_root, + collection_id=collection_id, + provider=provider, + ) + + return jsonable_encoder(response) + + +@router.get("/collections/{collection_id}/items/{item_id}", tags=["Data"]) +async def stac_collections_item(collection_id, item_id, request: Request): + """STAC collection item by id""" + url = request.state.url + url_root = request.state.url_root + try: + body = await request.json() + except JSONDecodeError: + body = {} + arguments = dict(request.query_params, **body) + provider = arguments.pop("provider", None) + + response = get_stac_item_by_id( + url=url, + item_id=item_id, + root=url_root, + catalogs=[collection_id], + provider=provider, ) - return jsonify(response), 200 + if response: + return jsonable_encoder(response) + else: + raise HTTPException( + status_code=404, + detail="No item found matching `{}` id in collection `{}`".format( + item_id, collection_id + ), + ) + + +@router.get("/collections/{collection_id}/items/{item_id}/download", tags=["Data"]) +async def stac_collections_item_download(collection_id, item_id, request: Request): + """STAC collection item local download""" + try: + body = await request.json() + except JSONDecodeError: + body = {} + arguments = dict(request.query_params, **body) + provider = arguments.pop("provider", None) -@app.route("//items", methods=["GET"]) -@cross_origin -def stac_catalogs_items(catalogs): + response = download_stac_item_by_id( + catalogs=[collection_id], + item_id=item_id, + provider=provider, + ) + filename = os.path.basename(response) + + return FileResponse(response, filename=filename) + + +@router.get("/catalogs/{catalogs:path}/items", tags=["Data"]) +async def stac_catalogs_items(catalogs, request: Request): """Fetch catalog's features --- tags: @@ -258,25 +448,29 @@ def stac_catalogs_items(catalogs): '500': $ref: '#/components/responses/ServerError '""" + url = request.state.url + url_root = request.state.url_root + try: + body = await request.json() + except JSONDecodeError: + body = {} + arguments = dict(request.query_params, **body) + provider = arguments.pop("provider", None) catalogs = catalogs.strip("/").split("/") - arguments = request.args.to_dict() - provider = arguments.pop("provider", None) + response = search_stac_items( - url=request.url, + url=url, arguments=arguments, - root=request.url_root, + root=url_root, catalogs=catalogs, provider=provider, ) - return app.response_class( - response=geojson.dumps(response), status=200, mimetype="application/json" - ) + return jsonable_encoder(response) -@app.route("//items/", methods=["GET"]) -@cross_origin -def stac_catalogs_item(catalogs, item_id): +@router.get("/catalogs/{catalogs:path}/items/{item_id}", tags=["Data"]) +async def stac_catalogs_item(catalogs, item_id, request: Request): """Fetch catalog's single features --- tags: @@ -310,202 +504,99 @@ def stac_catalogs_item(catalogs, item_id): '500': $ref: '#/components/responses/ServerError' """ + url = request.state.url + url_root = request.state.url_root + try: + body = await request.json() + except JSONDecodeError: + body = {} + arguments = dict(request.query_params, **body) + provider = arguments.pop("provider", None) catalogs = catalogs.strip("/").split("/") response = get_stac_item_by_id( - url=request.url.split("?")[0], + url=url, item_id=item_id, - root=request.url_root, + root=url_root, catalogs=catalogs, - provider=request.args.to_dict().get("provider", None), + provider=provider, ) if response: - return app.response_class( - response=geojson.dumps(response), status=200, mimetype="application/json" - ) + return jsonable_encoder(response) else: - abort( - 404, - "No item found matching `{}` id in catalog `{}`".format(item_id, catalogs), + raise HTTPException( + status_code=404, + detail="No item found matching `{}` id in catalog `{}`".format( + item_id, catalogs + ), ) -@app.route("//items//download", methods=["GET"]) -@cross_origin -def stac_catalogs_item_download(catalogs, item_id): +@router.get("/catalogs/{catalogs:path}/items/{item_id}/download", tags=["Data"]) +async def stac_catalogs_item_download(catalogs, item_id, request: Request): """STAC item local download""" + try: + body = await request.json() + except JSONDecodeError: + body = {} + arguments = dict(request.query_params, **body) + provider = arguments.pop("provider", None) catalogs = catalogs.strip("/").split("/") response = download_stac_item_by_id( catalogs=catalogs, item_id=item_id, - provider=request.args.to_dict().get("provider", None), + provider=provider, ) filename = os.path.basename(response) - return send_file(response, as_attachment=True, attachment_filename=filename) + return FileResponse(response, filename=filename) -@app.route("/collections/", methods=["GET"]) -@app.route("/collections", methods=["GET"]) -@cross_origin -def collections(): - """STAC collections +@router.get("/catalogs/{catalogs:path}", tags=["Capabilities"]) +async def stac_catalogs(catalogs, request: Request): + """Describe the given catalog and list available sub-catalogs + --- + tags: + - Capabilities + parameters: + - name: catalogs + in: path + required: true + description: |- + The catalog's path. - Can be filtered using parameters: instrument, platform, platformSerialIdentifier, sensorType, processingLevel + For a nested catalog, provide the root-related path to the catalog (for example `S2_MSI_L1C/year/2020`) + schema: + type: string + responses: + 200: + description: The catalog's description + content: + application/json: + schema: + $ref: '#/components/schemas/collection' + '500': + $ref: '#/components/responses/ServerError' """ - arguments = request.args.to_dict() + url = request.state.url + url_root = request.state.url_root + try: + body = await request.json() + except JSONDecodeError: + body = {} + arguments = dict(request.query_params, **body) provider = arguments.pop("provider", None) - response = get_stac_collections( - url=request.url.split("?")[0], - root=request.url_root, - arguments=arguments, - provider=provider, - ) - - return jsonify(response), 200 - -@app.route("/collections/", methods=["GET"]) -@cross_origin -def collection_by_id(collection_id): - """STAC collection by id""" - - response = get_stac_collection_by_id( - url=request.url.split("?")[0], - root=request.url_root, - collection_id=collection_id, - provider=request.args.to_dict().get("provider", None), - ) - - return jsonify(response), 200 - - -@app.route("/collections//items", methods=["GET"]) -@cross_origin -def stac_collections_items(collection_id): - """STAC collections items""" - - arguments = request.args.to_dict() - provider = arguments.pop("provider", None) - response = search_stac_items( - url=request.url, - arguments=arguments, - root=request.url_root, + catalogs = catalogs.strip("/").split("/") + response = get_stac_catalogs( + url=url, + root=url_root, + catalogs=catalogs, provider=provider, - catalogs=[collection_id], - ) - return app.response_class( - response=geojson.dumps(response), status=200, mimetype="application/json" - ) - - -@app.route("/search", methods=["GET", "POST"]) -@cross_origin -def stac_search(): - """STAC collections items""" - - if request.get_json(silent=True, force=True): - arguments = dict(request.args.to_dict(), **request.get_json(force=True)) - else: - arguments = request.args.to_dict() - - provider = arguments.pop("provider", None) - response = search_stac_items( - url=request.url, arguments=arguments, root=request.url_root, provider=provider - ) - return app.response_class( - response=geojson.dumps(response), status=200, mimetype="application/json" - ) - - -@app.route("/extensions/oseo/json-schema/schema.json", methods=["GET"]) -@cross_origin -def stac_extension_oseo(): - """STAC OGC / OpenSearch extension for EO""" - - response = get_stac_extension_oseo(url=request.url.split("?")[0]) - - return app.response_class( - response=geojson.dumps(response), status=200, mimetype="application/json" - ) - - -@app.route("/collections//items/", methods=["GET"]) -@cross_origin -def stac_collections_item(collection_id, item_id): - """STAC collection item by id""" - - response = get_stac_item_by_id( - url=request.url.split("?")[0], - item_id=item_id, - root=request.url_root, - catalogs=[collection_id], - provider=request.args.to_dict().get("provider", None), - ) - - return app.response_class( - response=geojson.dumps(response), status=200, mimetype="application/json" - ) - - -@app.route("/collections//items//download", methods=["GET"]) -@cross_origin -def stac_collections_item_download(collection_id, item_id): - """STAC collection item local download""" - - response = download_stac_item_by_id( - catalogs=[collection_id], - item_id=item_id, - provider=request.args.to_dict().get("provider", None), ) - filename = os.path.basename(response) - - return send_file(response, as_attachment=True, attachment_filename=filename) - - -def main(): - """Launch the server""" - import argparse - - parser = argparse.ArgumentParser( - description="""Script for starting EODAG server""", epilog="""""" - ) - parser.add_argument( - "-d", "--daemon", action="store_true", help="run in daemon mode" - ) - parser.add_argument( - "-a", - "--all-addresses", - action="store_true", - help="run flask using IPv4 0.0.0.0 (all network interfaces), " - "otherwise bind to 127.0.0.1 (localhost). " - "This maybe necessary in systems that only run Flask", - ) - args = parser.parse_args() - - if args.all_addresses: - bind_host = "0.0.0.0" - else: - bind_host = "127.0.0.1" - - if args.daemon: - pid = None - try: - pid = os.fork() - except OSError as e: - raise Exception("%s [%d]" % (e.strerror, e.errno)) - - if pid == 0: - os.setsid() - app.run(threaded=True, host=bind_host) - else: - sys.exit(0) - else: - # For development - app.run(debug=True, use_reloader=True) + return jsonable_encoder(response) -if __name__ == "__main__": - main() +app.include_router(router) diff --git a/eodag/rest/stac.py b/eodag/rest/stac.py index 850b7d299..c27946cde 100644 --- a/eodag/rest/stac.py +++ b/eodag/rest/stac.py @@ -22,6 +22,7 @@ from collections import defaultdict import dateutil.parser +import geojson import shapefile from dateutil import tz from dateutil.relativedelta import relativedelta @@ -40,6 +41,7 @@ jsonpath_parse_dict_items, string_to_jsonpath, update_nested_dict, + urljoin, ) from eodag.utils.exceptions import ( MisconfiguredError, @@ -51,6 +53,7 @@ logger = logging.getLogger("eodag.rest.stac") DEFAULT_MISSION_START_DATE = "2015-01-01T00:00:00Z" +STAC_CATALOGS_PREFIX = "catalogs" class StacCommon(object): @@ -145,7 +148,7 @@ def as_dict(self): :returns: STAC data dictionnary :rtype: dict """ - return self.data + return geojson.loads(geojson.dumps(self.data)) __geo_interface__ = property(as_dict) @@ -319,7 +322,7 @@ def get_stac_items(self, search_results, catalog): items["features"] = self.__get_item_list(search_results, catalog) self.update_data(items) - return self.as_dict() + return geojson.loads(geojson.dumps(self.data)) def __filter_item_model_properties(self, item_model, product_type): """Filter item model depending on product type metadata and its extensions. @@ -644,7 +647,7 @@ def __init__( catalogs=[], fetch_providers=True, *args, - **kwargs + **kwargs, ): super(StacCatalog, self).__init__( url=url, @@ -1136,7 +1139,9 @@ def __build_stac_catalog(self, catalogs=[], fetch_providers=True): [ { "rel": "child", - "href": self.url + "/" + product_type["ID"], + "href": urljoin( + self.url, f"{STAC_CATALOGS_PREFIX}/{product_type['ID']}" + ), "title": product_type["ID"], } for product_type in product_types_list diff --git a/eodag/rest/utils.py b/eodag/rest/utils.py index 301f1b5cb..c424a6470 100644 --- a/eodag/rest/utils.py +++ b/eodag/rest/utils.py @@ -8,6 +8,7 @@ import os import re from collections import namedtuple +from shutil import make_archive import dateutil.parser from dateutil import tz @@ -465,7 +466,6 @@ def search_product_by_id(uid, product_type=None): """ try: products, total = eodag_api.search(id=uid, productType=product_type) - # products, total = eodag_api.search(id=uid, productType=product_type, provider=provider, raise_errors=True) return products except ValidationError: raise @@ -485,6 +485,15 @@ def get_stac_conformance(): return stac_config["conformance"] +def get_stac_api_version(): + """Get STAC API version + + :returns: STAC API version + :rtype: str + """ + return stac_config["stac_api_version"] + + def get_stac_collections(url, root, arguments, provider=None): """Build STAC collections @@ -580,9 +589,17 @@ def download_stac_item_by_id(catalogs, item_id, provider=None): eodag_api.providers_config[product.provider].download.extract = False - product_path = eodag_api.download(product) + product_path = eodag_api.download(product, extract=False) - return product_path + if os.path.isdir(product_path): + zipped_product_path = f"{product_path}.zip" + logger.debug( + f"Building archive for downloaded product path {zipped_product_path}" + ) + make_archive(product_path, "zip", product_path) + return zipped_product_path + else: + return product_path def get_stac_catalogs(url, root="/", catalogs=[], provider=None, fetch_providers=True): @@ -788,3 +805,12 @@ def get_stac_extension_oseo(url): return StacCommon.get_stac_extension( url=url, stac_config=stac_config, extension="oseo", properties=oseo_properties ) + + +def eodag_api_init(): + """Init EODataAccessGateway server instance, pre-running all time consuming tasks""" + eodag_api.fetch_product_types_list() + + # pre-build search plugins + for provider in eodag_api.available_providers(): + next(eodag_api._plugins_manager.get_search_plugins(provider=provider)) diff --git a/pytest.ini b/pytest.ini index 4964de78f..e1f2e4104 100644 --- a/pytest.ini +++ b/pytest.ini @@ -2,6 +2,6 @@ testpaths = eodag/utils tests -addopts = --doctest-modules --disable-socket +addopts = --doctest-modules --disable-socket --allow-unix-socket ; logging disabled to prevent conflicts with click, see https://github.com/pallets/click/issues/824 log_cli = 0 diff --git a/setup.cfg b/setup.cfg index 177496d74..85dd0f0de 100644 --- a/setup.cfg +++ b/setup.cfg @@ -51,10 +51,10 @@ install_requires = pyproj >= 2.1.0 usgs >= 0.3.1 boto3 - flasgger + fastapi >= 0.93.0 + uvicorn jsonpath-ng lxml - flask >= 1.0.2, != 2.2.0, != 2.2.1, < 2.3.0 whoosh pystac >= 1.0.0b1 ecmwf-api-client @@ -79,6 +79,7 @@ dev = flake8 pre-commit responses + fastapi[all] notebook = tqdm[notebook] tutorials = eodag-cube >= 0.2.0 diff --git a/tests/context.py b/tests/context.py index c320ae6e6..3cc2540c2 100644 --- a/tests/context.py +++ b/tests/context.py @@ -82,6 +82,7 @@ deepcopy, cached_parse, sanitize, + parse_header, ) from eodag.utils.exceptions import ( AddressNotFound, diff --git a/tests/resources/stac/country/FRA/year/2017/items/S2B_MSIL1C_20171231T141039_N0206_R110_T21NYC_20171231T201802/S2B_MSIL1C_20171231T141039_N0206_R110_T21NYC_20171231T201802.json b/tests/resources/stac/country/FRA/year/2017/items/S2B_MSIL1C_20171231T141039_N0206_R110_T21NYC_20171231T201802/S2B_MSIL1C_20171231T141039_N0206_R110_T21NYC_20171231T201802.json index c4513f31f..6ec2a57fe 100644 --- a/tests/resources/stac/country/FRA/year/2017/items/S2B_MSIL1C_20171231T141039_N0206_R110_T21NYC_20171231T201802/S2B_MSIL1C_20171231T141039_N0206_R110_T21NYC_20171231T201802.json +++ b/tests/resources/stac/country/FRA/year/2017/items/S2B_MSIL1C_20171231T141039_N0206_R110_T21NYC_20171231T201802/S2B_MSIL1C_20171231T141039_N0206_R110_T21NYC_20171231T201802.json @@ -84,7 +84,7 @@ "assets": { "downloadLink": { "title": "Download link", - "href": "http://127.0.0.1:5000/S2_MSI_L1C/country/FRA/year/2017/items/S2B_MSIL1C_20171231T141039_N0206_R110_T21NYC_20171231T201802/download", + "href": "http://127.0.0.1:5000/catalogs/S2_MSI_L1C/country/FRA/year/2017/items/S2B_MSIL1C_20171231T141039_N0206_R110_T21NYC_20171231T201802/download", "type": "application/zip" }, "thumbnail": { diff --git a/tests/resources/stac/country/FRA/year/2018/items/S2A_MSIL1C_20181231T141041_N0207_R110_T21NYF_20181231T155050/S2A_MSIL1C_20181231T141041_N0207_R110_T21NYF_20181231T155050.json b/tests/resources/stac/country/FRA/year/2018/items/S2A_MSIL1C_20181231T141041_N0207_R110_T21NYF_20181231T155050/S2A_MSIL1C_20181231T141041_N0207_R110_T21NYF_20181231T155050.json index fe68e729f..624e867c5 100644 --- a/tests/resources/stac/country/FRA/year/2018/items/S2A_MSIL1C_20181231T141041_N0207_R110_T21NYF_20181231T155050/S2A_MSIL1C_20181231T141041_N0207_R110_T21NYF_20181231T155050.json +++ b/tests/resources/stac/country/FRA/year/2018/items/S2A_MSIL1C_20181231T141041_N0207_R110_T21NYF_20181231T155050/S2A_MSIL1C_20181231T141041_N0207_R110_T21NYF_20181231T155050.json @@ -84,7 +84,7 @@ "assets": { "downloadLink": { "title": "Download link", - "href": "http://127.0.0.1:5000/S2_MSI_L1C/country/FRA/year/2018/items/S2A_MSIL1C_20181231T141041_N0207_R110_T21NYF_20181231T155050/download", + "href": "http://127.0.0.1:5000/catalogs/S2_MSI_L1C/country/FRA/year/2018/items/S2A_MSIL1C_20181231T141041_N0207_R110_T21NYF_20181231T155050/download", "type": "application/zip" }, "thumbnail": { diff --git a/tests/resources/stac/country/FRA/year/2018/items/S2A_MSIL1C_20181231T141041_N0207_R110_T22NBJ_20181231T155050/S2A_MSIL1C_20181231T141041_N0207_R110_T22NBJ_20181231T155050.json b/tests/resources/stac/country/FRA/year/2018/items/S2A_MSIL1C_20181231T141041_N0207_R110_T22NBJ_20181231T155050/S2A_MSIL1C_20181231T141041_N0207_R110_T22NBJ_20181231T155050.json index 6b717e79c..b256bb327 100644 --- a/tests/resources/stac/country/FRA/year/2018/items/S2A_MSIL1C_20181231T141041_N0207_R110_T22NBJ_20181231T155050/S2A_MSIL1C_20181231T141041_N0207_R110_T22NBJ_20181231T155050.json +++ b/tests/resources/stac/country/FRA/year/2018/items/S2A_MSIL1C_20181231T141041_N0207_R110_T22NBJ_20181231T155050/S2A_MSIL1C_20181231T141041_N0207_R110_T22NBJ_20181231T155050.json @@ -108,7 +108,7 @@ "assets": { "downloadLink": { "title": "Download link", - "href": "http://127.0.0.1:5000/S2_MSI_L1C/country/FRA/year/2018/items/S2A_MSIL1C_20181231T141041_N0207_R110_T22NBJ_20181231T155050/download", + "href": "http://127.0.0.1:5000/catalogs/S2_MSI_L1C/country/FRA/year/2018/items/S2A_MSIL1C_20181231T141041_N0207_R110_T22NBJ_20181231T155050/download", "type": "application/zip" }, "thumbnail": { diff --git a/tests/resources/stac/country/GBR/year/2019/items/S2A_MSIL1C_20191231T112451_N0208_R037_T30UUD_20191231T115143/S2A_MSIL1C_20191231T112451_N0208_R037_T30UUD_20191231T115143.json b/tests/resources/stac/country/GBR/year/2019/items/S2A_MSIL1C_20191231T112451_N0208_R037_T30UUD_20191231T115143/S2A_MSIL1C_20191231T112451_N0208_R037_T30UUD_20191231T115143.json index 6535347ad..6530f0a9e 100644 --- a/tests/resources/stac/country/GBR/year/2019/items/S2A_MSIL1C_20191231T112451_N0208_R037_T30UUD_20191231T115143/S2A_MSIL1C_20191231T112451_N0208_R037_T30UUD_20191231T115143.json +++ b/tests/resources/stac/country/GBR/year/2019/items/S2A_MSIL1C_20191231T112451_N0208_R037_T30UUD_20191231T115143/S2A_MSIL1C_20191231T112451_N0208_R037_T30UUD_20191231T115143.json @@ -113,7 +113,7 @@ "assets": { "downloadLink": { "title": "Download link", - "href": "http://127.0.0.1:5000/S2_MSI_L1C/country/GBR/year/2019/items/S2A_MSIL1C_20191231T112451_N0208_R037_T30UUD_20191231T115143/download", + "href": "http://127.0.0.1:5000/catalogs/S2_MSI_L1C/country/GBR/year/2019/items/S2A_MSIL1C_20191231T112451_N0208_R037_T30UUD_20191231T115143/download", "type": "application/zip" }, "thumbnail": { diff --git a/tests/resources/stac/country/GBR/year/2019/items/S2A_MSIL1C_20191231T112451_N0208_R037_T30VVH_20191231T115143/S2A_MSIL1C_20191231T112451_N0208_R037_T30VVH_20191231T115143.json b/tests/resources/stac/country/GBR/year/2019/items/S2A_MSIL1C_20191231T112451_N0208_R037_T30VVH_20191231T115143/S2A_MSIL1C_20191231T112451_N0208_R037_T30VVH_20191231T115143.json index a0b6f3f14..b6f3250d9 100644 --- a/tests/resources/stac/country/GBR/year/2019/items/S2A_MSIL1C_20191231T112451_N0208_R037_T30VVH_20191231T115143/S2A_MSIL1C_20191231T112451_N0208_R037_T30VVH_20191231T115143.json +++ b/tests/resources/stac/country/GBR/year/2019/items/S2A_MSIL1C_20191231T112451_N0208_R037_T30VVH_20191231T115143/S2A_MSIL1C_20191231T112451_N0208_R037_T30VVH_20191231T115143.json @@ -113,7 +113,7 @@ "assets": { "downloadLink": { "title": "Download link", - "href": "http://127.0.0.1:5000/S2_MSI_L1C/country/GBR/year/2019/items/S2A_MSIL1C_20191231T112451_N0208_R037_T30VVH_20191231T115143/download", + "href": "http://127.0.0.1:5000/catalogs/S2_MSI_L1C/country/GBR/year/2019/items/S2A_MSIL1C_20191231T112451_N0208_R037_T30VVH_20191231T115143/download", "type": "application/zip" }, "thumbnail": { diff --git a/tests/units/test_http_server.py b/tests/units/test_http_server.py index 4006e255c..ea2bf85d3 100644 --- a/tests/units/test_http_server.py +++ b/tests/units/test_http_server.py @@ -19,16 +19,28 @@ import importlib import json import os +import socket import unittest +from pathlib import Path from tempfile import TemporaryDirectory import geojson +from fastapi.testclient import TestClient from shapely import box from tests import mock -from tests.context import DEFAULT_ITEMS_PER_PAGE, SearchResult - - +from tests.context import ( + DEFAULT_ITEMS_PER_PAGE, + AuthenticationError, + SearchResult, + parse_header, +) + + +# AF_UNIX socket not supported on windows yet, see https://github.com/python/cpython/issues/77589 +@unittest.skipIf( + not hasattr(socket, "AF_UNIX"), "AF_UNIX socket not supported on this OS (windows)" +) class RequestTestCase(unittest.TestCase): @classmethod def setUpClass(cls): @@ -52,13 +64,6 @@ def setUpClass(cls): cls.eodag_http_server = eodag_http_server - # run swagger / service-doc - eodag_http_server.run_swagger( - app=eodag_http_server.app, - config=eodag_http_server.stac_api_config, - merge=True, - ) - # mock os.environ to empty env cls.mock_os_environ = mock.patch.dict(os.environ, {}, clear=True) cls.mock_os_environ.start() @@ -77,13 +82,32 @@ def tearDownClass(cls): cls.tmp_home_dir.cleanup() def setUp(self): - self.app = self.eodag_http_server.app.test_client() + self.app = TestClient(self.eodag_http_server.app) def test_route(self): + self._request_valid("/") + + def test_forward(self): response = self.app.get("/", follow_redirects=True) self.assertEqual(200, response.status_code) + resp_json = json.loads(response.content.decode("utf-8")) + self.assertEqual(resp_json["links"][0]["href"], "http://testserver") + + response = self.app.get( + "/", follow_redirects=True, headers={"Forwarded": "host=foo;proto=https"} + ) + self.assertEqual(200, response.status_code) + resp_json = json.loads(response.content.decode("utf-8")) + self.assertEqual(resp_json["links"][0]["href"], "https://foo") - self._request_valid(self.tested_product_type) + response = self.app.get( + "/", + follow_redirects=True, + headers={"X-Forwarded-Host": "bar", "X-Forwarded-Proto": "httpz"}, + ) + self.assertEqual(200, response.status_code) + resp_json = json.loads(response.content.decode("utf-8")) + self.assertEqual(resp_json["links"][0]["href"], "httpz://bar") @mock.patch( "eodag.rest.utils.eodag_api.search", @@ -221,7 +245,7 @@ def test_route(self): 2, ), ) - def _request_valid( + def _request_valid_raw( self, url, mock_search, @@ -236,7 +260,6 @@ def _request_valid( url, data=json.dumps(post_data), follow_redirects=True, - mimetype="application/json", ) if expected_search_kwargs is not None: @@ -244,12 +267,28 @@ def _request_valid( self.assertEqual(200, response.status_code) + return response + + def _request_valid( + self, + url, + expected_search_kwargs=None, + protocol="GET", + post_data=None, + ): + response = self._request_valid_raw( + url, + expected_search_kwargs=expected_search_kwargs, + protocol=protocol, + post_data=post_data, + ) + # Assert response format is GeoJSON - return geojson.loads(response.data.decode("utf-8")) + return geojson.loads(response.content.decode("utf-8")) def _request_not_valid(self, url): response = self.app.get(url, follow_redirects=True) - response_content = json.loads(response.data.decode("utf-8")) + response_content = json.loads(response.content.decode("utf-8")) self.assertEqual(400, response.status_code) self.assertIn("description", response_content) @@ -257,11 +296,11 @@ def _request_not_valid(self, url): def _request_not_found(self, url): response = self.app.get(url, follow_redirects=True) - response_content = json.loads(response.data.decode("utf-8")) + response_content = json.loads(response.content.decode("utf-8")) self.assertEqual(404, response.status_code) - self.assertIn("error", response_content) - self.assertIn("not found", response_content["error"]) + self.assertIn("description", response_content) + self.assertIn("not found", response_content["description"]) def test_request_params(self): self._request_not_valid(f"search?collections={self.tested_product_type}&bbox=1") @@ -296,9 +335,26 @@ def test_request_params(self): ) def test_not_found(self): - """A request to eodag server with a not supported product type must return a 404 HTTP error code""" # noqa + """A request to eodag server with a not supported product type must return a 404 HTTP error code""" self._request_not_found("search?collections=ZZZ&bbox=0,43,1,44") + @mock.patch( + "eodag.rest.utils.eodag_api.search", + autospec=True, + side_effect=AuthenticationError("you are no authorized"), + ) + def test_auth_error(self, mock_search): + """A request to eodag server raising a Authentication error must return a 401 HTTP error code""" + response = self.app.get( + f"search?collections={self.tested_product_type}", follow_redirects=True + ) + response_content = json.loads(response.content.decode("utf-8")) + + self.assertEqual(401, response.status_code) + self.assertIn("description", response_content) + self.assertIn("AuthenticationError", response_content["description"]) + self.assertIn("you are no authorized", response_content["description"]) + def test_filter(self): """latestIntersect filter should only keep the latest products once search area is fully covered""" result1 = self._request_valid( @@ -326,6 +382,7 @@ def test_filter(self): self.assertEqual(len(result2.features), 1) def test_date_search(self): + """Search through eodag server /search endpoint using dates filering should return a valid response""" self._request_valid( f"search?collections={self.tested_product_type}&bbox=0,43,1,44&datetime=2018-01-20/2018-01-25", expected_search_kwargs=dict( @@ -340,6 +397,7 @@ def test_date_search(self): ) def test_date_search_from_items(self): + """Search through eodag server collection/items endpoint using dates filering should return a valid response""" self._request_valid( f"collections/{self.tested_product_type}/items?bbox=0,43,1,44", expected_search_kwargs=dict( @@ -364,8 +422,9 @@ def test_date_search_from_items(self): ) def test_date_search_from_catalog_items(self): + """Search through eodag server catalog/items endpoint using dates filering should return a valid response""" results = self._request_valid( - f"{self.tested_product_type}/year/2018/month/01/items?bbox=0,43,1,44", + f"catalogs/{self.tested_product_type}/year/2018/month/01/items?bbox=0,43,1,44", expected_search_kwargs=dict( productType=self.tested_product_type, page=1, @@ -379,7 +438,8 @@ def test_date_search_from_catalog_items(self): self.assertEqual(len(results.features), 2) results = self._request_valid( - f"{self.tested_product_type}/year/2018/month/01/items?bbox=0,43,1,44&datetime=2018-01-20/2018-01-25", + f"catalogs/{self.tested_product_type}/year/2018/month/01/items" + "?bbox=0,43,1,44&datetime=2018-01-20/2018-01-25", expected_search_kwargs=dict( productType=self.tested_product_type, page=1, @@ -393,7 +453,8 @@ def test_date_search_from_catalog_items(self): self.assertEqual(len(results.features), 2) results = self._request_valid( - f"{self.tested_product_type}/year/2018/month/01/items?bbox=0,43,1,44&datetime=2018-01-20/2019-01-01", + f"catalogs/{self.tested_product_type}/year/2018/month/01/items" + "?bbox=0,43,1,44&datetime=2018-01-20/2019-01-01", expected_search_kwargs=dict( productType=self.tested_product_type, page=1, @@ -407,20 +468,50 @@ def test_date_search_from_catalog_items(self): self.assertEqual(len(results.features), 2) results = self._request_valid( - f"{self.tested_product_type}/year/2018/month/01/items?bbox=0,43,1,44&datetime=2019-01-01/2019-01-31", + f"catalogs/{self.tested_product_type}/year/2018/month/01/items" + "?bbox=0,43,1,44&datetime=2019-01-01/2019-01-31", ) self.assertEqual(len(results.features), 0) def test_catalog_browse(self): + """Browsing catalogs through eodag server should return a valid response""" result = self._request_valid( - f"{self.tested_product_type}/year/2018/month/01/day", + f"catalogs/{self.tested_product_type}/year/2018/month/01/day" ) self.assertListEqual( [str(i) for i in range(1, 32)], [it["title"] for it in result.get("links", []) if it["rel"] == "child"], ) + def test_search_item_id_from_catalog(self): + """Search by id through eodag server /catalog endpoint should return a valid response""" + self._request_valid( + f"catalogs/{self.tested_product_type}/items/foo", + expected_search_kwargs={ + "id": "foo", + "productType": self.tested_product_type, + }, + ) + + def test_search_item_id_from_collection(self): + """Search by id through eodag server /collection endpoint should return a valid response""" + self._request_valid( + f"collections/{self.tested_product_type}/items/foo", + expected_search_kwargs={ + "id": "foo", + "productType": self.tested_product_type, + }, + ) + + def test_collection(self): + """Requesting a collection through eodag server should return a valid response""" + result = self._request_valid(f"collections/{self.tested_product_type}") + self.assertEqual(result["id"], self.tested_product_type) + for link in result["links"]: + self.assertIn(link["rel"], ["self", "root", "items"]) + def test_cloud_cover_post_search(self): + """POST search with cloudCover filtering through eodag server should return a valid response""" self._request_valid( "search", protocol="POST", @@ -440,7 +531,7 @@ def test_cloud_cover_post_search(self): ) def test_search_response_contains_pagination_info(self): - """Responses to valid search requests must return a geojson with pagination info in properties""" # noqa + """Responses to valid search requests must return a geojson with pagination info in properties""" response = self._request_valid(f"search?collections={self.tested_product_type}") self.assertIn("numberMatched", response) self.assertIn("numberReturned", response) @@ -469,7 +560,7 @@ def test_list_product_types_ok(self, list_pt, guess_pt): ["S2_MSI_L1C", "S2_MSI_L2A"], [ it["title"] - for it in json.loads(r.data.decode("utf-8")).get("links", []) + for it in json.loads(r.content.decode("utf-8")).get("links", []) if it["rel"] == "child" ], ) @@ -484,7 +575,7 @@ def test_list_product_types_ok(self, list_pt, guess_pt): ["S2_MSI_L1C"], [ it["title"] - for it in json.loads(r.data.decode("utf-8")).get("links", []) + for it in json.loads(r.content.decode("utf-8")).get("links", []) if it["rel"] == "child" ], ) @@ -495,7 +586,7 @@ def test_list_product_types_ok(self, list_pt, guess_pt): return_value=[{"ID": "S2_MSI_L1C"}, {"ID": "S2_MSI_L2A"}], ) def test_list_product_types_nok(self, list_pt): - """A request for product types with a not supported filter must return all product types""" # noqa + """A request for product types with a not supported filter must return all product types""" url = "/collections?platform=gibberish" r = self.app.get(url) self.assertTrue(list_pt.called) @@ -504,17 +595,78 @@ def test_list_product_types_nok(self, list_pt): ["S2_MSI_L1C", "S2_MSI_L2A"], [ it["title"] - for it in json.loads(r.data.decode("utf-8")).get("links", []) + for it in json.loads(r.content.decode("utf-8")).get("links", []) if it["rel"] == "child" ], ) + @mock.patch( + "eodag.rest.utils.eodag_api.download", + autospec=True, + ) + def test_download_item_from_catalog(self, mock_download): + """Download through eodag server catalog should return a valid response""" + # download returns a folder that must be zipped + tmp_dl_dir = TemporaryDirectory() + mock_download.return_value = tmp_dl_dir.name + expected_file = f"{tmp_dl_dir.name}.zip" + + response = self._request_valid_raw( + f"catalogs/{self.tested_product_type}/items/foo/download" + ) + mock_download.assert_called_once() + + header_content_disposition = parse_header( + response.headers["content-disposition"] + ) + response_filename = header_content_disposition.get_param("filename", None) + self.assertEqual(response_filename, os.path.basename(expected_file)) + self.assertTrue(os.path.isfile(expected_file)) + + @mock.patch( + "eodag.rest.utils.eodag_api.download", + autospec=True, + ) + def test_download_item_from_collection(self, mock_download): + """Download through eodag server catalog should return a valid response""" + # download returns a file that must be returned as is + tmp_dl_dir = TemporaryDirectory() + expected_file = f"{tmp_dl_dir.name}.tar" + Path(expected_file).touch() + mock_download.return_value = expected_file + + response = self._request_valid_raw( + f"collections/{self.tested_product_type}/items/foo/download" + ) + mock_download.assert_called_once() + + header_content_disposition = parse_header( + response.headers["content-disposition"] + ) + response_filename = header_content_disposition.get_param("filename", None) + self.assertEqual(response_filename, os.path.basename(expected_file)) + self.assertTrue(os.path.isfile(expected_file)) + def test_conformance(self): + """Request to /conformance should return a valid response""" self._request_valid("conformance") def test_service_desc(self): - self._request_valid("api") + """Request to service_desc should return a valid response""" + service_desc = self._request_valid("api") + self.assertIn("openapi", service_desc.keys()) + self.assertIn("eodag", service_desc["info"]["title"].lower()) + self.assertGreater(len(service_desc["paths"].keys()), 0) + # test a 2nd call (ending slash must be ignored) + self._request_valid("api/") def test_service_doc(self): - response = self.app.get("service-doc", follow_redirects=True) + """Request to service_doc should return a valid response""" + response = self.app.get("api.html", follow_redirects=True) self.assertEqual(200, response.status_code) + + def test_stac_extension_oseo(self): + """Request to oseo extension should return a valid response""" + response = self._request_valid("/extensions/oseo/json-schema/schema.json") + self.assertEqual(response["title"], "OpenSearch for Earth Observation") + self.assertEqual(response["allOf"][0]["$ref"], "#/definitions/oseo")