From eddf7a77fdb04ee93cffe5842d60a205a4b7b64d Mon Sep 17 00:00:00 2001 From: vincentsarago Date: Mon, 16 Oct 2023 17:03:47 +0200 Subject: [PATCH] update to titiler-pgstac 0.8.0 --- .pre-commit-config.yaml | 2 +- raster_api/runtime/Dockerfile | 6 +- raster_api/runtime/setup.py | 8 +- raster_api/runtime/src/algorithms.py | 45 + raster_api/runtime/src/app.py | 108 +-- raster_api/runtime/src/config.py | 43 +- raster_api/runtime/src/datasetparams.py | 50 - raster_api/runtime/src/dependencies.py | 89 +- raster_api/runtime/src/extensions.py | 43 + raster_api/runtime/src/factory.py | 47 - .../runtime/src/templates/stac-viewer.html | 2 +- raster_api/runtime/src/templates/viewer.html | 895 ------------------ 12 files changed, 182 insertions(+), 1156 deletions(-) create mode 100644 raster_api/runtime/src/algorithms.py delete mode 100644 raster_api/runtime/src/datasetparams.py create mode 100644 raster_api/runtime/src/extensions.py delete mode 100644 raster_api/runtime/src/factory.py delete mode 100644 raster_api/runtime/src/templates/viewer.html diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c064fbc2..4912662d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -12,7 +12,7 @@ repos: language_version: python - repo: https://github.com/PyCQA/flake8 - rev: 3.8.3 + rev: 6.1.0 hooks: - id: flake8 language_version: python diff --git a/raster_api/runtime/Dockerfile b/raster_api/runtime/Dockerfile index c0cec90f..8b051765 100644 --- a/raster_api/runtime/Dockerfile +++ b/raster_api/runtime/Dockerfile @@ -1,9 +1,9 @@ -FROM public.ecr.aws/sam/build-python3.9:latest +FROM public.ecr.aws/sam/build-python3.11:latest WORKDIR /tmp COPY raster_api/runtime /tmp/raster -RUN pip install "mangum>=0.14,<0.15" /tmp/raster["psycopg-binary"] -t /asset --no-binary pydantic +RUN pip install "mangum>=0.14,<0.15" /tmp/raster["psycopg-binary"] -t /asset --no-binary pydantic RUN rm -rf /tmp/raster # # Reduce package size and remove useless files @@ -15,4 +15,4 @@ RUN rm -rdf /asset/numpy/doc/ /asset/boto3* /asset/botocore* /asset/bin /asset/g COPY raster_api/runtime/handler.py /asset/handler.py -CMD ["echo", "hello world"] \ No newline at end of file +CMD ["echo", "hello world"] diff --git a/raster_api/runtime/setup.py b/raster_api/runtime/setup.py index aa87c65d..9b57c7d4 100644 --- a/raster_api/runtime/setup.py +++ b/raster_api/runtime/setup.py @@ -6,12 +6,10 @@ long_description = f.read() inst_reqs = [ - "titiler.pgstac==0.2.3", - "titiler.application>=0.10,<0.11", - "importlib_resources>=1.1.0;python_version<='3.9'", # https://github.com/cogeotiff/rio-tiler/pull/379 + "titiler.pgstac==0.8.0", + "titiler.extensions[cogeo]>=0.15.0,<0.16", "aws_xray_sdk>=2.6.0,<3", "aws-lambda-powertools>=1.18.0", - "pydantic<2", ] extra_reqs = { @@ -26,7 +24,7 @@ setup( name="veda.raster_api", description="", - python_requires=">=3.7", + python_requires=">=3.8", packages=find_namespace_packages(exclude=["tests*"]), package_data={"src": ["templates/*.html"]}, include_package_data=True, diff --git a/raster_api/runtime/src/algorithms.py b/raster_api/runtime/src/algorithms.py new file mode 100644 index 00000000..08566959 --- /dev/null +++ b/raster_api/runtime/src/algorithms.py @@ -0,0 +1,45 @@ +"""veda custom algorithms""" + +import math + +import numpy +from rio_tiler.models import ImageData + +from titiler.core.algorithm import Algorithms +from titiler.core.algorithm.base import BaseAlgorithm + +# https://github.com/cogeotiff/rio-tiler/blob/master/rio_tiler/reader.py#L35-L37 + +# From eoAPI datasetparams edl_auth branch https://github.com/NASA-IMPACT/eoAPI/blob/edl_auth/src/eoapi/raster/eoapi/raster/datasetparams.py + + +class SWIR(BaseAlgorithm): + """SWIR Custom Algorithm.""" + + low_value: float = math.e + high_value: float = 255 + low_threshold: float = math.log(1000) + high_threshold: float = math.log(7500) + + def __call__(self, img: ImageData) -> ImageData: + """Apply processing.""" + data = numpy.log(img.array) + data[numpy.where(data <= self.low_threshold)] = self.low_value + data[numpy.where(data >= self.high_threshold)] = self.high_value + indices = numpy.where((data > self.low_value) & (data < self.high_value)) + data[indices] = ( + self.high_value + * (data[indices] - self.low_threshold) + / (self.high_threshold - self.low_threshold) + ) + img.array = data.astype("uint8") + return img + + +algorithms = Algorithms( + { + "swir": SWIR, + } +) + +PostProcessParams = algorithms.dependency diff --git a/raster_api/runtime/src/app.py b/raster_api/runtime/src/app.py index a95ea636..a4cee76e 100644 --- a/raster_api/runtime/src/app.py +++ b/raster_api/runtime/src/app.py @@ -1,58 +1,60 @@ """TiTiler+PgSTAC FastAPI application.""" import logging +from contextlib import asynccontextmanager from aws_lambda_powertools.metrics import MetricUnit -from rio_cogeo.cogeo import cog_info as rio_cogeo_info -from rio_cogeo.models import Info +from src.algorithms import PostProcessParams from src.config import ApiSettings -from src.datasetparams import DatasetParams -from src.factory import MultiBaseTilerFactory +from src.dependencies import ItemPathParams +from src.extensions import stacViewerExtension +from src.monitoring import LoggerRouteHandler, logger, metrics, tracer from src.version import __version__ as veda_raster_version -from fastapi import APIRouter, Depends, FastAPI, Query +from fastapi import APIRouter, FastAPI from starlette.middleware.cors import CORSMiddleware from starlette.requests import Request -from starlette.responses import HTMLResponse -from starlette.templating import Jinja2Templates from starlette_cramjam.middleware import CompressionMiddleware -from titiler.core.dependencies import DatasetPathParams from titiler.core.errors import DEFAULT_STATUS_CODES, add_exception_handlers -from titiler.core.factory import TilerFactory, TMSFactory +from titiler.core.factory import MultiBaseTilerFactory, TilerFactory, TMSFactory from titiler.core.middleware import CacheControlMiddleware from titiler.core.resources.enums import OptionalHeader from titiler.core.resources.responses import JSONResponse +from titiler.extensions import cogValidateExtension, cogViewerExtension from titiler.mosaic.errors import MOSAIC_STATUS_CODES from titiler.pgstac.db import close_db_connection, connect_to_db -from titiler.pgstac.dependencies import ItemPathParams from titiler.pgstac.factory import MosaicTilerFactory from titiler.pgstac.reader import PgSTACReader -try: - from importlib.resources import files as resources_files # type: ignore -except ImportError: - # Try backported to PY<39 `importlib_resources`. - from importlib_resources import files as resources_files # type: ignore - -from .monitoring import LoggerRouteHandler, logger, metrics, tracer - logging.getLogger("botocore.credentials").disabled = True logging.getLogger("botocore.utils").disabled = True logging.getLogger("rio-tiler").setLevel(logging.ERROR) settings = ApiSettings() -templates = Jinja2Templates(directory=str(resources_files(__package__) / "templates")) # type: ignore + if settings.debug: optional_headers = [OptionalHeader.server_timing, OptionalHeader.x_assets] else: optional_headers = [] + +@asynccontextmanager +async def lifespan(app: FastAPI): + """FastAPI Lifespan.""" + # Create Connection Pool + await connect_to_db(app, settings=settings.load_postgres_settings()) + yield + # Close the Connection Pool + await close_db_connection(app) + + path_prefix = settings.path_prefix app = FastAPI( title=settings.name, version=veda_raster_version, openapi_url=f"{path_prefix}/openapi.json", docs_url=f"{path_prefix}/docs", + lifespan=lifespan, ) # router to be applied to all titiler route factories (improves logs with FastAPI context) @@ -60,19 +62,24 @@ add_exception_handlers(app, DEFAULT_STATUS_CODES) add_exception_handlers(app, MOSAIC_STATUS_CODES) - -# Custom PgSTAC mosaic tiler +############################################################################### +# /mosaic - PgSTAC Mosaic titiler endpoint +############################################################################### mosaic = MosaicTilerFactory( router_prefix=f"{path_prefix}/mosaic", add_mosaic_list=settings.enable_mosaic_search, optional_headers=optional_headers, environment_dependency=settings.get_gdal_config, - dataset_dependency=DatasetParams, + post_process=PostProcessParams, router=APIRouter(route_class=LoggerRouteHandler), ) app.include_router(mosaic.router, prefix=f"{path_prefix}/mosaic", tags=["Mosaic"]) +# TODO +# prefix will be replaced by `/mosaics/{search_id}` in titiler-pgstac 0.9.0 -# Custom STAC titiler endpoint (not added to the openapi docs) +############################################################################### +# /stac - Custom STAC titiler endpoint +############################################################################### stac = MultiBaseTilerFactory( reader=PgSTACReader, path_dependency=ItemPathParams, @@ -80,45 +87,28 @@ router_prefix=f"{path_prefix}/stac", environment_dependency=settings.get_gdal_config, router=APIRouter(route_class=LoggerRouteHandler), + extensions=[ + stacViewerExtension(), + ], ) app.include_router(stac.router, tags=["Items"], prefix=f"{path_prefix}/stac") +# TODO +# in titiler-pgstac we replaced the prefix to `/collections/{collection_id}/items/{item_id}` +############################################################################### +# /cog - External Cloud Optimized GeoTIFF endpoints +############################################################################### cog = TilerFactory( router_prefix=f"{path_prefix}/cog", optional_headers=optional_headers, environment_dependency=settings.get_gdal_config, router=APIRouter(route_class=LoggerRouteHandler), + extensions=[ + cogValidateExtension(), + cogViewerExtension(), + ], ) - -@cog.router.get( - "/validate", - response_model=Info, - response_class=JSONResponse, -) -def cog_validate( - src_path: str = Depends(DatasetPathParams), - strict: bool = Query(False, description="Treat warnings as errors"), -): - """Validate a COG""" - return rio_cogeo_info(src_path, strict=strict, config=settings.get_gdal_config()) - - -@cog.router.get("/viewer", response_class=HTMLResponse) -def cog_demo(request: Request): - """COG Viewer.""" - return templates.TemplateResponse( - name="viewer.html", - context={ - "request": request, - "tilejson_endpoint": cog.url_for(request, "tilejson"), - "info_endpoint": cog.url_for(request, "info"), - "statistics_endpoint": cog.url_for(request, "statistics"), - }, - media_type="text/html", - ) - - app.include_router( cog.router, tags=["Cloud Optimized GeoTIFF"], prefix=f"{path_prefix}/cog" ) @@ -174,8 +164,10 @@ async def add_correlation_id(request: Request, call_next): except KeyError: # If empty, use uuid corr_id = "local" + # Add correlation id to logs logger.set_correlation_id(corr_id) + # Add correlation id to traces tracer.put_annotation(key="correlation_id", value=corr_id) @@ -192,15 +184,3 @@ async def validation_exception_handler(request, err): metrics.add_metric(name="UnhandledExceptions", unit=MetricUnit.Count, value=1) logger.exception("Unhandled exception") return JSONResponse(status_code=500, content={"detail": "Internal Server Error"}) - - -@app.on_event("startup") -async def startup_event() -> None: - """Connect to database on startup.""" - await connect_to_db(app, settings=settings.load_postgres_settings()) - - -@app.on_event("shutdown") -async def shutdown_event() -> None: - """Close database connection.""" - await close_db_connection(app) diff --git a/raster_api/runtime/src/config.py b/raster_api/runtime/src/config.py index 38e52948..d25167c8 100644 --- a/raster_api/runtime/src/config.py +++ b/raster_api/runtime/src/config.py @@ -6,9 +6,10 @@ from typing import Optional import boto3 -import pydantic -from pydantic import BaseSettings, Field +from pydantic import Field, field_validator +from pydantic_settings import BaseSettings from rasterio.session import AWSSession +from typing_extensions import Annotated from titiler.pgstac.settings import PostgresSettings @@ -58,9 +59,15 @@ class ApiSettings(BaseSettings): # MosaicTiler settings enable_mosaic_search: bool = False - pgstac_secret_arn: Optional[str] + pgstac_secret_arn: Optional[str] = None - @pydantic.validator("cors_origins") + model_config = { + "env_file": ".env", + "extra": "ignore", + "env_prefix": "VEDA_RASTER_", + } + + @field_validator("cors_origins") def parse_cors_origin(cls, v): """Parse CORS origins.""" return [origin.strip() for origin in v.split(",")] @@ -74,21 +81,25 @@ def load_postgres_settings(self) -> "PostgresSettings": postgres_user=secret["username"], postgres_pass=secret["password"], postgres_host=secret["host"], - postgres_port=str(secret["port"]), + postgres_port=int(secret["port"]), postgres_dbname=secret["dbname"], ) else: return PostgresSettings() - data_access_role_arn: Optional[str] = Field( - None, - description="Resource name of role permitting access to specified external S3 buckets", - ) + data_access_role_arn: Annotated[ + Optional[str], + Field( + description="Resource name of role permitting access to specified external S3 buckets" + ), + ] = None - export_assume_role_creds_as_envs: Optional[bool] = Field( - False, - description="enables 'get_gdal_config' flow to export AWS credentials as os env vars", - ) + export_assume_role_creds_as_envs: Annotated[ + bool, + Field( + description="enables 'get_gdal_config' flow to export AWS credentials as os env vars", + ), + ] = False def get_gdal_config(self): """return default aws session config or assume role data_access_role_arn credentials session""" @@ -131,9 +142,3 @@ def get_gdal_config(self): else: # Use the default role of this lambda return {} - - class Config: - """model config""" - - env_file = ".env" - env_prefix = "VEDA_RASTER_" diff --git a/raster_api/runtime/src/datasetparams.py b/raster_api/runtime/src/datasetparams.py deleted file mode 100644 index 1105d8bc..00000000 --- a/raster_api/runtime/src/datasetparams.py +++ /dev/null @@ -1,50 +0,0 @@ -"""From eoAPI datasetparams edl_auth branch https://github.com/NASA-IMPACT/eoAPI/blob/edl_auth/src/eoapi/raster/eoapi/raster/datasetparams.py""" -import math -from dataclasses import dataclass -from typing import Optional, Tuple - -import numpy - -from fastapi import Query -from titiler.core import dependencies - -# https://github.com/cogeotiff/rio-tiler/blob/master/rio_tiler/reader.py#L35-L37 - -# From eoAPI datasetparams edl_auth branch https://github.com/NASA-IMPACT/eoAPI/blob/edl_auth/src/eoapi/raster/eoapi/raster/datasetparams.py - - -def swir(data, mask) -> Tuple[numpy.ndarray, numpy.ndarray]: - """SWIR""" - low_value = math.e - high_value = 255 - - low_threshold = math.log(1000) - high_threshold = math.log(7500) - - data = numpy.log(data) - data[numpy.where(data <= low_threshold)] = low_value - data[numpy.where(data >= high_threshold)] = high_value - indices = numpy.where((data > low_value) & (data < high_value)) - data[indices] = ( - high_value * (data[indices] - low_threshold) / (high_threshold - low_threshold) - ) - return data.astype("uint8"), mask - - -pp_methods = { - "swir": swir, -} - - -@dataclass -class DatasetParams(dependencies.DatasetParams): - """Post processing parameters for map layers""" - - post_process: Optional[str] = Query(None, description="Post Process Name.") - - def __post_init__(self): - """.""" - super().__post_init__() - - if self.post_process is not None: - self.post_process = pp_methods.get(self.post_process) # type: ignore diff --git a/raster_api/runtime/src/dependencies.py b/raster_api/runtime/src/dependencies.py index c4914138..cd066427 100644 --- a/raster_api/runtime/src/dependencies.py +++ b/raster_api/runtime/src/dependencies.py @@ -1,76 +1,23 @@ """veda.raster.dependencies.""" -import json -from base64 import b64decode -from typing import Dict, Union -from urllib.parse import urlparse - -from cachetools import LRUCache, cached -from cachetools.keys import hashkey +import pystac +from typing_extensions import Annotated from fastapi import Query from starlette.requests import Request - - -@cached( - LRUCache(maxsize=512), - key=lambda pool, collection_id, item_id: hashkey(collection_id, item_id), -) -def get_item(pool, collection_id, item_id): - """Get STAC Item from PGStac.""" - - print("COLLECTION ID: ", collection_id) - print("ITEM ID: ", item_id) - - req = dict( - filter={ - "op": "and", - "args": [ - { - "op": "eq", - "args": [{"property": "collection"}, collection_id], - }, - {"op": "eq", "args": [{"property": "id"}, item_id]}, - ], - }, - ) - print("REQUEST: ", req) - with pool.connection() as conn: - with conn.cursor() as cursor: - cursor.execute( - "SELECT * FROM search(%s);", - (json.dumps(req),), - ) - resp = cursor.fetchone()[0] - features = resp.get("features", []) - if not len(features): - raise Exception( - "No item '{item_id}' found in '{collection_id}' collection" - ) - - return features[0] - - -def DatasetPathParams( - request: Request, url: str = Query(..., description="Dataset URL") -) -> Union[str, Dict]: - """Custom Dataset Param for the custom STAC Reader""" - parsed = urlparse(url) - - # stac://{base 64 encoded item} - if parsed.scheme == "stac": - return json.loads(b64decode(url.replace("stac://", ""))) - - # pgstac://{collectionId}/{itemId} - elif parsed.scheme == "pgstac": - collection_id = parsed.netloc - item_id = parsed.path.strip("/") - return get_item( - request.app.state.dbpool, - collection_id, - item_id, - ) - - # Default to passing the URL - else: - return url +from titiler.pgstac.dependencies import get_stac_item + + +def ItemPathParams( + request: Request, + collection: Annotated[ + str, + Query(description="STAC Collection ID"), + ], + item: Annotated[ + str, + Query(description="STAC Item ID"), + ], +) -> pystac.Item: + """STAC Item dependency.""" + return get_stac_item(request.app.state.dbpool, collection, item) diff --git a/raster_api/runtime/src/extensions.py b/raster_api/runtime/src/extensions.py new file mode 100644 index 00000000..51271d98 --- /dev/null +++ b/raster_api/runtime/src/extensions.py @@ -0,0 +1,43 @@ +"""Stac Viewer Extension.""" + +from dataclasses import dataclass + +import jinja2 + +from fastapi import Depends +from starlette.requests import Request +from starlette.responses import HTMLResponse +from starlette.templating import Jinja2Templates +from titiler.core.factory import BaseTilerFactory, FactoryExtension + +DEFAULT_TEMPLATES = Jinja2Templates( + directory="", + loader=jinja2.ChoiceLoader([jinja2.PackageLoader(__package__, "templates")]), +) # type:ignore + + +@dataclass +class stacViewerExtension(FactoryExtension): + """Add /viewer endpoint to the TilerFactory.""" + + templates: Jinja2Templates = DEFAULT_TEMPLATES + + def register(self, factory: BaseTilerFactory): + """Register endpoint to the tiler factory.""" + + @factory.router.get("/viewer", response_class=HTMLResponse) + def stac_viewer( + request: Request, + item=Depends(factory.path_dependency), + ): + """STAC Viewer.""" + return self.templates.TemplateResponse( + name="stac-viewer.html", + context={ + "request": request, + "endpoint": request.url.path.replace("/viewer", ""), + "collection": item.collection_id, + "item": item.id, + }, + media_type="text/html", + ) diff --git a/raster_api/runtime/src/factory.py b/raster_api/runtime/src/factory.py deleted file mode 100644 index a9b969e6..00000000 --- a/raster_api/runtime/src/factory.py +++ /dev/null @@ -1,47 +0,0 @@ -"""Custom MultiBaseTilerFactory.""" - -from dataclasses import dataclass -from typing import Any - -from fastapi import Depends -from starlette.requests import Request -from starlette.responses import HTMLResponse -from starlette.templating import Jinja2Templates -from titiler.core import factory as TitilerFactory - -try: - from importlib.resources import files as resources_files # type: ignore -except ImportError: - # Try backported to PY<39 `importlib_resources`. - from importlib_resources import files as resources_files # type: ignore - - -# TODO: mypy fails in python 3.9, we need to find a proper way to do this -templates = Jinja2Templates(directory=str(resources_files(__package__) / "templates")) # type: ignore - - -@dataclass -class MultiBaseTilerFactory(TitilerFactory.MultiBaseTilerFactory): - """Custom endpoints factory.""" - - def register_routes(self) -> None: - """This Method register routes to the router.""" - super().register_routes() - - # Add viewer - @self.router.get("/viewer", response_class=HTMLResponse) - def stac_demo( - request: Request, - item: Any = Depends(self.path_dependency), - ): - """STAC Viewer.""" - return templates.TemplateResponse( - name="stac-viewer.html", - context={ - "request": request, - "endpoint": request.url.path.replace("/viewer", ""), - "collection": request.query_params["collection"], - "item": request.query_params["item"], - }, - media_type="text/html", - ) diff --git a/raster_api/runtime/src/templates/stac-viewer.html b/raster_api/runtime/src/templates/stac-viewer.html index dacf18e4..957bf947 100644 --- a/raster_api/runtime/src/templates/stac-viewer.html +++ b/raster_api/runtime/src/templates/stac-viewer.html @@ -723,4 +723,4 @@ }) - \ No newline at end of file + diff --git a/raster_api/runtime/src/templates/viewer.html b/raster_api/runtime/src/templates/viewer.html deleted file mode 100644 index e34ea2af..00000000 --- a/raster_api/runtime/src/templates/viewer.html +++ /dev/null @@ -1,895 +0,0 @@ - - - - - TiTiler - Cloud Optimized GeoTIFF Viewer - - - - - - - - - - - -
-
-
Enter COG url
- - -
-
- - - -
-
-
-
- - -
-
-
-
-
- - -