From f62461e078326277d69032ef18ec93cf9b8b9e26 Mon Sep 17 00:00:00 2001 From: Matt Webster Date: Wed, 3 Mar 2021 22:52:38 -0800 Subject: [PATCH] Tune performance (caching API status calls) Fixes #990 - revisited caching - added new council_ids to geometry --- .../code/lacity_data_api/models/council.py | 18 +++------ .../code/lacity_data_api/models/geometry.py | 14 ++++++- .../api/code/lacity_data_api/models/region.py | 10 +---- .../lacity_data_api/models/request_type.py | 40 +++++-------------- .../lacity_data_api/models/service_request.py | 5 +-- .../code/lacity_data_api/routers/geojson.py | 2 + .../code/lacity_data_api/services/status.py | 2 +- 7 files changed, 33 insertions(+), 58 deletions(-) diff --git a/server/api/code/lacity_data_api/models/council.py b/server/api/code/lacity_data_api/models/council.py index 5772c95f8..81a96afb2 100644 --- a/server/api/code/lacity_data_api/models/council.py +++ b/server/api/code/lacity_data_api/models/council.py @@ -1,4 +1,4 @@ -from aiocache import cached, Cache, serializers +from aiocache import cached from sqlalchemy import and_ @@ -6,7 +6,6 @@ from .service_request import ServiceRequest from .region import Region from .request_type import RequestType -from ..config import CACHE_ENDPOINT class Council(db.Model): @@ -19,8 +18,10 @@ class Council(db.Model): region_id = db.Column(db.SmallInteger, db.ForeignKey('regions.region_id')) latitude = db.Column(db.Float) longitude = db.Column(db.Float) + data_code = db.Column(db.SmallInteger) @classmethod + @cached(key="councils:all", alias="default") async def all(cls): result = await ( db.select( @@ -53,12 +54,7 @@ async def one(cls, id: int): return result -@cached(cache=Cache.REDIS, - endpoint=CACHE_ENDPOINT, - namespace="councils", - key="dict", - serializer=serializers.PickleSerializer(), - ) +@cached(key="councils:dict", alias="default") async def get_councils_dict(): result = await db.all(Council.query) councils_dict = [ @@ -68,11 +64,7 @@ async def get_councils_dict(): return dict(councils_dict) -@cached(cache=Cache.REDIS, - endpoint=CACHE_ENDPOINT, - namespace="councils", - serializer=serializers.PickleSerializer(), - ) +@cached(alias="default") async def get_open_request_counts(council: int): result = await ( diff --git a/server/api/code/lacity_data_api/models/geometry.py b/server/api/code/lacity_data_api/models/geometry.py index e53ecca9f..1f92c1e6d 100644 --- a/server/api/code/lacity_data_api/models/geometry.py +++ b/server/api/code/lacity_data_api/models/geometry.py @@ -1,5 +1,8 @@ -from . import db from geoalchemy2 import Geometry +from aiocache import cached + +from . import db +from lacity_data_api.models.council import Council class Geometry(db.Model): @@ -9,14 +12,19 @@ class Geometry(db.Model): geometry = db.Column(Geometry(geometry_type="MULTIPOLYGON")) @classmethod + @cached(key="councils:geojson", alias="default") async def get_council_geojson(cls): result = await ( db.select( [ + Council.council_id, + Council.council_name, Geometry.nc_id, db.func.ST_AsGeoJSON(Geometry.geometry).label("geometry") ] + ).select_from( + Geometry.join(Council, Geometry.nc_id == Council.data_code) ).gino.all() ) return result @@ -27,8 +35,10 @@ async def get_enclosing_council(cls, latitude: float, longitude: float): result = await ( db.select( [ - Geometry.nc_id.label("council_id"), + Council.council_id ] + ).select_from( + Geometry.join(Council, Geometry.nc_id == Council.data_code) ).where( db.func.ST_WITHIN( db.func.ST_MakePoint(longitude, latitude), Geometry.geometry diff --git a/server/api/code/lacity_data_api/models/region.py b/server/api/code/lacity_data_api/models/region.py index f943a3184..430e2e741 100644 --- a/server/api/code/lacity_data_api/models/region.py +++ b/server/api/code/lacity_data_api/models/region.py @@ -1,6 +1,5 @@ -from aiocache import cached, Cache, serializers +from aiocache import cached -from ..config import CACHE_ENDPOINT from . import db @@ -13,12 +12,7 @@ class Region(db.Model): longitude = db.Column(db.Float) -@cached(cache=Cache.REDIS, - endpoint=CACHE_ENDPOINT, - namespace="regions", - key="dict", - serializer=serializers.PickleSerializer(), - ) +@cached(key="regions:dict", alias="default") async def get_regions_dict(): result = await db.all(Region.query) regions_dict = [ diff --git a/server/api/code/lacity_data_api/models/request_type.py b/server/api/code/lacity_data_api/models/request_type.py index 648f32dc0..396491c6f 100644 --- a/server/api/code/lacity_data_api/models/request_type.py +++ b/server/api/code/lacity_data_api/models/request_type.py @@ -1,8 +1,6 @@ from typing import List -from aiocache import cached, Cache, serializers - -from ..config import CACHE_ENDPOINT +from aiocache import cached from . import db @@ -17,12 +15,7 @@ class RequestType(db.Model): data_code = db.Column(db.String) @classmethod - @cached(cache=Cache.REDIS, - endpoint=CACHE_ENDPOINT, - namespace="types", - key="all", - serializer=serializers.PickleSerializer(), - ) + @cached(key="types:all", alias="default") async def all(cls): from .agency import Agency @@ -59,26 +52,21 @@ async def one(cls, id: int): return result @classmethod - @cached(cache=Cache.REDIS, - endpoint=CACHE_ENDPOINT, - namespace="types", - key="stats", - serializer=serializers.PickleSerializer(), - ) + @cached(key="types:stats", alias="default") async def get_type_stats(cls): query = db.text(""" SELECT service_requests.type_id, request_types.type_name, - min(closed_date - created_date), + min(closed_date::date - created_date::date), percentile_disc(0.25) within group - (order by closed_date - created_date) as q1, + (order by closed_date::date - created_date::date) as q1, percentile_disc(0.5) within group - (order by closed_date - created_date) as median, + (order by closed_date::date - created_date::date) as median, percentile_disc(0.75) within group - (order by closed_date - created_date) as q3, - max(closed_date - created_date) + (order by closed_date::date - created_date::date) as q3, + max(closed_date::date - created_date::date) FROM service_requests JOIN @@ -94,12 +82,7 @@ async def get_type_stats(cls): return result -@cached(cache=Cache.REDIS, - endpoint=CACHE_ENDPOINT, - namespace="types", - key="dict", - serializer=serializers.PickleSerializer(), - ) +@cached(key="types:dict", alias="default") async def get_types_dict(): '''This is a shim function to allow types to be retrieved by strings''' result = await db.all(RequestType.query) @@ -107,11 +90,6 @@ async def get_types_dict(): return dict(types_dict) -@cached(cache=Cache.REDIS, - endpoint=CACHE_ENDPOINT, - namespace="types", - serializer=serializers.PickleSerializer(), - ) async def get_type_ids_by_str_list(str_list: List[str]) -> List[int]: '''Get a list of RequestType IDs from their type_names using data code''' result = await db.all( diff --git a/server/api/code/lacity_data_api/models/service_request.py b/server/api/code/lacity_data_api/models/service_request.py index 3baf7fe49..8c6b8d660 100644 --- a/server/api/code/lacity_data_api/models/service_request.py +++ b/server/api/code/lacity_data_api/models/service_request.py @@ -87,7 +87,6 @@ async def get_request_reports( return result @classmethod - @cached(alias="default") async def get_recent_requests(cls, start_date: datetime.date): result = await ( @@ -109,7 +108,7 @@ async def get_full_request(srnumber: str): return result -@cached(alias="default") +@cached(key="requests:open", alias="default") async def get_open_requests() -> List[ServiceRequest]: result = await ( @@ -139,7 +138,7 @@ async def get_id_from_srnumber(srnumber: str): return result -@cached(alias="default") +@cached(key="requests:open_type_counts", alias="default") async def get_open_request_counts(): result = await ( db.select( diff --git a/server/api/code/lacity_data_api/routers/geojson.py b/server/api/code/lacity_data_api/routers/geojson.py index 359b0e5c3..96c153712 100644 --- a/server/api/code/lacity_data_api/routers/geojson.py +++ b/server/api/code/lacity_data_api/routers/geojson.py @@ -12,6 +12,8 @@ def createFeature(row): return { "type": "Feature", "properties": { + "council_id": item['council_id'], + "council_name": item['council_name'], "nc_id": item['nc_id'] }, "geometry": json.loads(item['geometry']) diff --git a/server/api/code/lacity_data_api/services/status.py b/server/api/code/lacity_data_api/services/status.py index e451d922e..d4eb10045 100644 --- a/server/api/code/lacity_data_api/services/status.py +++ b/server/api/code/lacity_data_api/services/status.py @@ -4,7 +4,7 @@ from ..config import CACHE_MAXMEMORY -@cached(alias="default") +@cached(key="status:last_updated", alias="default") async def get_last_updated(): query = db.text("SELECT max(created_time) as last_pulled FROM log WHERE status = 'INFO'") # noqa result = await db.first(query)