Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: titiler-pgstac v1 upgrade #398

Merged
merged 22 commits into from
Jul 25, 2024
Merged
Show file tree
Hide file tree
Changes from 20 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
58 changes: 29 additions & 29 deletions .github/workflows/tests/test_raster.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,29 +19,31 @@ def test_raster_api():
def test_mosaic_api():
"""test mosaic."""
query = {"collections": ["noaa-emergency-response"], "filter-lang": "cql-json"}
resp = httpx.post(f"{raster_endpoint}/mosaic/register", json=query)
resp = httpx.post(f"{raster_endpoint}/searches/register", json=query)
assert resp.headers["content-type"] == "application/json"
assert resp.status_code == 200
assert resp.json()["searchid"]
assert resp.json()["id"]
assert resp.json()["links"]

searchid = resp.json()["searchid"]
searchid = resp.json()["id"]

resp = httpx.get(f"{raster_endpoint}/mosaic/{searchid}/-85.6358,36.1624/assets")
resp = httpx.get(f"{raster_endpoint}/searches/{searchid}/-85.6358,36.1624/assets")
assert resp.status_code == 200
assert len(resp.json()) == 1
assert list(resp.json()[0]) == ["id", "bbox", "assets", "collection"]
assert resp.json()[0]["id"] == "20200307aC0853900w361030"

resp = httpx.get(f"{raster_endpoint}/mosaic/{searchid}/tiles/15/8589/12849/assets")
resp = httpx.get(
f"{raster_endpoint}/searches/{searchid}/tiles/15/8589/12849/assets"
)
assert resp.status_code == 200
assert len(resp.json()) == 1
assert list(resp.json()[0]) == ["id", "bbox", "assets", "collection"]
assert resp.json()[0]["id"] == "20200307aC0853900w361030"

z, x, y = 15, 8589, 12849
resp = httpx.get(
f"{raster_endpoint}/mosaic/{searchid}/tiles/{z}/{x}/{y}",
f"{raster_endpoint}/searches/{searchid}/tiles/{z}/{x}/{y}",
params={"assets": "cog"},
headers={"Accept-Encoding": "br, gzip"},
timeout=10.0,
Expand Down Expand Up @@ -105,11 +107,11 @@ def test_mosaic_search():
},
]
for search in searches:
resp = httpx.post(f"{raster_endpoint}/mosaic/register", json=search)
resp = httpx.post(f"{raster_endpoint}/searches/register", json=search)
assert resp.status_code == 200
assert resp.json()["searchid"]
assert resp.json()["id"]

resp = httpx.get(f"{raster_endpoint}/mosaic/list")
resp = httpx.get(f"{raster_endpoint}/searches/list")
assert resp.headers["content-type"] == "application/json"
assert resp.status_code == 200
assert (
Expand All @@ -118,16 +120,18 @@ def test_mosaic_search():
assert resp.json()["context"]["returned"] == 10 # default limit is 10

# Make sure all mosaics returned have
for mosaic in resp.json()["searches"]:
assert mosaic["search"]["metadata"]["type"] == "mosaic"
for search in resp.json()["searches"]:
assert search["search"]["metadata"]["type"] == "mosaic"

links = resp.json()["links"]
assert len(links) == 2
assert links[0]["rel"] == "self"
assert links[1]["rel"] == "next"
assert links[1]["href"] == f"{raster_endpoint}/mosaic/list?limit=10&offset=10"
assert links[1]["href"] == f"{raster_endpoint}/searches/list?limit=10&offset=10"

resp = httpx.get(f"{raster_endpoint}/mosaic/list", params={"limit": 1, "offset": 1})
resp = httpx.get(
f"{raster_endpoint}/searches/list", params={"limit": 1, "offset": 1}
)
assert resp.status_code == 200
assert resp.json()["context"]["matched"] > 10
assert resp.json()["context"]["limit"] == 1
Expand All @@ -136,55 +140,51 @@ def test_mosaic_search():
links = resp.json()["links"]
assert len(links) == 3
assert links[0]["rel"] == "self"
assert links[0]["href"] == f"{raster_endpoint}/mosaic/list?limit=1&offset=1"
assert links[0]["href"] == f"{raster_endpoint}/searches/list?limit=1&offset=1"
assert links[1]["rel"] == "next"
assert links[1]["href"] == f"{raster_endpoint}/mosaic/list?limit=1&offset=2"
assert links[1]["href"] == f"{raster_endpoint}/searches/list?limit=1&offset=2"
assert links[2]["rel"] == "prev"
assert links[2]["href"] == f"{raster_endpoint}/mosaic/list?limit=1&offset=0"
assert links[2]["href"] == f"{raster_endpoint}/searches/list?limit=1&offset=0"

# Filter on mosaic metadata
resp = httpx.get(f"{raster_endpoint}/mosaic/list", params={"owner": "vincent"})
resp = httpx.get(f"{raster_endpoint}/searches/list", params={"owner": "vincent"})
assert resp.status_code == 200
assert resp.json()["context"]["matched"] == 7
assert resp.json()["context"]["limit"] == 10
assert resp.json()["context"]["returned"] == 7

# sortBy
resp = httpx.get(f"{raster_endpoint}/mosaic/list", params={"sortby": "lastused"})
resp = httpx.get(f"{raster_endpoint}/searches/list", params={"sortby": "lastused"})
assert resp.status_code == 200

resp = httpx.get(f"{raster_endpoint}/mosaic/list", params={"sortby": "usecount"})
resp = httpx.get(f"{raster_endpoint}/searches/list", params={"sortby": "usecount"})
assert resp.status_code == 200

resp = httpx.get(f"{raster_endpoint}/mosaic/list", params={"sortby": "-owner"})
resp = httpx.get(f"{raster_endpoint}/searches/list", params={"sortby": "-owner"})
assert resp.status_code == 200
assert (
"owner" not in resp.json()["searches"][0]["search"]["metadata"]
) # some mosaic don't have owners

resp = httpx.get(f"{raster_endpoint}/mosaic/list", params={"sortby": "owner"})
resp = httpx.get(f"{raster_endpoint}/searches/list", params={"sortby": "owner"})
assert resp.status_code == 200
assert "owner" in resp.json()["searches"][0]["search"]["metadata"]


def test_item():
"""test stac endpoints."""
collection_id = "noaa-emergency-response"
item_id = "20200307aC0853300w361200"
resp = httpx.get(
f"{raster_endpoint}/stac/assets",
params={
"collection": "noaa-emergency-response",
"item": "20200307aC0853300w361200",
},
f"{raster_endpoint}/collections/{collection_id}/items/{item_id}/assets"
)
assert resp.status_code == 200
assert resp.headers["content-type"] == "application/json"
assert resp.json() == ["cog"]

resp = httpx.get(
f"{raster_endpoint}/stac/tilejson.json",
f"{raster_endpoint}/collections/{collection_id}/items/{item_id}/WebMercatorQuad/tilejson.json",
params={
"collection": "noaa-emergency-response",
"item": "20200307aC0853300w361200",
"assets": "cog",
},
)
Expand Down
8 changes: 4 additions & 4 deletions raster_api/runtime/setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,10 @@
inst_reqs = [
"boto3",
"rio-tiler==6.5.0",
"titiler.pgstac==0.8.3",
"titiler.core>=0.15.5,<0.16",
"titiler.mosaic>=0.15.5,<0.16",
"titiler.extensions[cogeo]>=0.15.5,<0.16",
"titiler.pgstac==1.3.0",
"titiler.core>=0.18.5,<0.19",
"titiler.mosaic>=0.18.5,<0.19",
"titiler.extensions[cogeo]>=0.18.5,<0.19",
"starlette-cramjam>=0.3,<0.4",
"aws_xray_sdk>=2.6.0,<3",
"aws-lambda-powertools>=1.18.0",
Expand Down
106 changes: 87 additions & 19 deletions raster_api/runtime/src/app.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
from src.algorithms import PostProcessParams
from src.alternate_reader import PgSTACReaderAlt
from src.config import ApiSettings
from src.dependencies import ColorMapParams, ItemPathParams
from src.dependencies import ColorMapParams
from src.extensions import stacViewerExtension
from src.monitoring import LoggerRouteHandler, logger, metrics, tracer
from src.version import __version__ as veda_raster_version
Expand All @@ -16,14 +16,25 @@
from starlette.requests import Request
from starlette_cramjam.middleware import CompressionMiddleware
from titiler.core.errors import DEFAULT_STATUS_CODES, add_exception_handlers
from titiler.core.factory import MultiBaseTilerFactory, TilerFactory, TMSFactory
from titiler.core.factory import (
ColorMapFactory,
MultiBaseTilerFactory,
TilerFactory,
TMSFactory,
)
from titiler.core.middleware import CacheControlMiddleware
from titiler.core.resources.enums import OptionalHeader
from titiler.core.resources.responses import JSONResponse
from titiler.extensions import cogValidateExtension, cogViewerExtension
from titiler.mosaic.errors import MOSAIC_STATUS_CODES
from titiler.pgstac.db import close_db_connection, connect_to_db
from titiler.pgstac.factory import MosaicTilerFactory
from titiler.pgstac.dependencies import CollectionIdParams, ItemIdParams, SearchIdParams
from titiler.pgstac.extensions import searchInfoExtension
from titiler.pgstac.factory import (
MosaicTilerFactory,
add_search_list_route,
add_search_register_route,
)
from titiler.pgstac.reader import PgSTACReader

logging.getLogger("botocore.credentials").disabled = True
Expand Down Expand Up @@ -64,61 +75,112 @@ async def lifespan(app: FastAPI):
add_exception_handlers(app, MOSAIC_STATUS_CODES)

###############################################################################
# /mosaic - PgSTAC Mosaic titiler endpoint
# /searches - PgSTAC Mosaic titiler endpoint
###############################################################################
mosaic = MosaicTilerFactory(
router_prefix="/mosaic",
searches = MosaicTilerFactory(
router_prefix="/searches/{search_id}",
path_dependency=SearchIdParams,
optional_headers=optional_headers,
environment_dependency=settings.get_gdal_config,
process_dependency=PostProcessParams,
Copy link
Contributor Author

@smohiudd smohiudd Jul 22, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we still need process_dependency if we have tile_dependencies below?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@smohiudd the process dependency is different from the tile_dependency. I see that we have custom post_process method, so if they are still used we should keep the process_dependency

router=APIRouter(route_class=LoggerRouteHandler),
# add /list (default to False)
add_mosaic_list=settings.enable_mosaic_search,
# add /statistics [POST] (default to False)
add_statistics=True,
# add /map viewer (default to False)
add_viewer=False,
# add /bbox [GET] and /feature [POST] (default to False)
add_part=True,
colormap_dependency=ColorMapParams,
extensions=[
searchInfoExtension(),
],
)
app.include_router(searches.router, prefix="/searches/{search_id}", tags=["STAC Search"])

# add /register endpoint
add_search_register_route(
Copy link
Contributor

@vincentsarago vincentsarago Jul 24, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

add

if settings.enable_mosaic_search:
    add_search_register_route(

app,
prefix="/searches",
# any dependency we want to validate
# when creating the tilejson/map links
tile_dependencies=[
searches.layer_dependency,
searches.dataset_dependency,
searches.pixel_selection_dependency,
searches.process_dependency,
searches.rescale_dependency,
searches.colormap_dependency,
searches.render_dependency,
searches.pgstac_dependency,
searches.reader_dependency,
searches.backend_dependency,
],
tags=["STAC Search"]
)
app.include_router(mosaic.router, prefix="/mosaic", tags=["Mosaic"])
# TODO
# prefix will be replaced by `/mosaics/{search_id}` in titiler-pgstac 1.0.0
# add /list endpoint
if settings.enable_mosaic_search:
add_search_list_route(app, prefix="/searches", tags=["STAC Search"])

###############################################################################
# /stac - Custom STAC titiler endpoint
# STAC COLLECTION Endpoints
###############################################################################
collection = MosaicTilerFactory(
path_dependency=CollectionIdParams,
optional_headers=optional_headers,
router_prefix="/collections/{collection_id}",
add_statistics=True,
add_viewer=True,
add_part=True,
extensions=[
searchInfoExtension(),
],
)
app.include_router(
collection.router, tags=["STAC Collection"], prefix="/collections/{collection_id}"
)


###############################################################################
# /collections/{collection_id}/items/{item_id} - Custom STAC titiler endpoint
###############################################################################
stac = MultiBaseTilerFactory(
reader=PgSTACReader,
path_dependency=ItemPathParams,
path_dependency=ItemIdParams,
optional_headers=optional_headers,
router_prefix="/stac",
router_prefix="/collections/{collection_id}/items/{item_id}",
environment_dependency=settings.get_gdal_config,
router=APIRouter(route_class=LoggerRouteHandler),
extensions=[
stacViewerExtension(),
],
colormap_dependency=ColorMapParams,
)
app.include_router(stac.router, tags=["Items"], prefix="/stac")
app.include_router(
stac.router,
tags=["STAC Item"],
prefix="/collections/{collection_id}/items/{item_id}",
)

###############################################################################
# /stac-alt - Custom STAC titiler endpoint for alternate asset locations
# /alt/collections/{collection_id}/items/{item_id} - Custom STAC titiler endpoint for alternate asset locations
###############################################################################
stac_alt = MultiBaseTilerFactory(
reader=PgSTACReaderAlt,
path_dependency=ItemPathParams,
path_dependency=ItemIdParams,
optional_headers=optional_headers,
router_prefix="/stac-alt",
router_prefix="/alt/collections/{collection_id}/items/{item_id}",
environment_dependency=settings.get_gdal_config,
router=APIRouter(route_class=LoggerRouteHandler),
extensions=[
stacViewerExtension(),
],
colormap_dependency=ColorMapParams,
)
app.include_router(stac_alt.router, tags=["Items"], prefix="/stac-alt")
app.include_router(
stac_alt.router,
tags=["Alt STAC Item"],
anayeaye marked this conversation as resolved.
Show resolved Hide resolved
prefix="/alt/collections/{collection_id}/items/{item_id}",
anayeaye marked this conversation as resolved.
Show resolved Hide resolved
)

###############################################################################
# /cog - External Cloud Optimized GeoTIFF endpoints
Expand All @@ -137,6 +199,12 @@ async def lifespan(app: FastAPI):

app.include_router(cog.router, tags=["Cloud Optimized GeoTIFF"], prefix="/cog")

###############################################################################
# Colormaps endpoints
###############################################################################
cmaps = ColorMapFactory()
app.include_router(cmaps.router, tags=["ColorMaps"])


@app.get("/healthz", description="Health Check", tags=["Health Check"])
def ping():
Expand Down
21 changes: 0 additions & 21 deletions raster_api/runtime/src/dependencies.py
Original file line number Diff line number Diff line change
@@ -1,36 +1,15 @@
"""veda.raster.dependencies."""

import pystac
from rio_tiler.colormap import cmap as default_cmap
from typing_extensions import Annotated

from fastapi import Query
from starlette.requests import Request
from titiler.core.dependencies import create_colormap_dependency
from titiler.pgstac.dependencies import get_stac_item

try:
from importlib.resources import files as resources_files # type: ignore
except ImportError:
# Try backported to PY<39 `importlib_resources`.
from importlib_resources import files as resources_files # type: ignore


def ItemPathParams(
request: Request,
collection: Annotated[
str,
Query(description="STAC Collection ID"),
],
item: Annotated[
str,
Query(description="STAC Item ID"),
],
) -> pystac.Item:
"""STAC Item dependency."""
return get_stac_item(request.app.state.dbpool, collection, item)


VEDA_CMAPS_FILES = {
f.stem: str(f) for f in (resources_files(__package__) / "cmap_data").glob("*.npy") # type: ignore
}
Expand Down
9 changes: 5 additions & 4 deletions raster_api/runtime/src/extensions.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,10 @@
from titiler.core.factory import BaseTilerFactory, FactoryExtension

DEFAULT_TEMPLATES = Jinja2Templates(
directory="",
loader=jinja2.ChoiceLoader([jinja2.PackageLoader(__package__, "templates")]),
) # type:ignore
env=jinja2.Environment(
loader=jinja2.ChoiceLoader([jinja2.PackageLoader(__package__, "templates")])
)
)


@dataclass
Expand All @@ -32,9 +33,9 @@ def stac_viewer(
):
"""STAC Viewer."""
return self.templates.TemplateResponse(
request,
name="stac-viewer.html",
context={
"request": request,
"endpoint": request.url.path.replace("/viewer", ""),
"collection": item.collection_id,
"item": item.id,
Expand Down
Loading