diff --git a/server/api/code/lacity_data_api/config.py b/server/api/code/lacity_data_api/config.py index 5d15fdd17..72de95bb1 100644 --- a/server/api/code/lacity_data_api/config.py +++ b/server/api/code/lacity_data_api/config.py @@ -47,6 +47,7 @@ # check whether running in legacy mode API_LEGACY_MODE = config('API_LEGACY_MODE', cast=bool, default=True) +# TODO: figure out how to remove dependency on DATABASE_URL from services # the legacy code needs these created as environment settings if API_LEGACY_MODE: environ['DATABASE_URL'] = str(DB_DSN) @@ -60,3 +61,6 @@ for k, v in sorted(os.environ.items()): print(f'\033[92m{k}\033[0m: {v}') print(f"\n\033[93mDatabase\033[0m: {DB_DSN}\n") + +# create empty cache object to populate at runtime +cache = {} diff --git a/server/api/code/lacity_data_api/models/clusters.py b/server/api/code/lacity_data_api/models/clusters.py index c2867f707..8630485fe 100644 --- a/server/api/code/lacity_data_api/models/clusters.py +++ b/server/api/code/lacity_data_api/models/clusters.py @@ -6,12 +6,13 @@ from sqlalchemy import and_ from .service_request import ServiceRequest -from .request_type import get_types_dict -from .council import Council +from .region import Region from . import db +from ..config import cache -DEFAULT_CITY_ZOOM = 12 # a click on a city point zooms from 10 to 12 +DEFAULT_CITY_ZOOM = 11 # a click on a city point zooms from 10 to 12 +DEFAULT_REGION_ZOOM = 12 # a click on a city point zooms from 10 to 12 DEFAULT_COUNCIL_ZOOM = 13 # a click on a council point zooms to 14 DEFAULT_LATITUDE = 34.0522 DEFAULT_LONGITUDE = -118.2437 @@ -71,15 +72,60 @@ async def get_clusters_for_city( return cluster_list -# TODO: same as above by group by region of each council -def get_clusters_for_regions(pins, zoom, bounds, options): +async def get_clusters_for_regions( + start_date: datetime.date, + end_date: datetime.date, + type_ids: List[int], + council_ids: List[int], + zoom_current: int +) -> List[Cluster]: """ - Cluster pins by region + Cluster service request pins by council regions + + Args: + start_date (date): beginning of date range service was requested + end_date (date): end of date range service was requested + type_ids (List[int]): the request type ids to match on + council_ids (List[int]): the council ids to match on + + Returns: + cluster: a list of cluster objects """ - print(zoom) + + # TODO: CACHE 'region-reqs:start-end-types-councils' + result = await ( + db.select( + [ + ServiceRequest.region_id, + db.func.count() + ] + ).where( + and_( + ServiceRequest.created_date >= start_date, + ServiceRequest.created_date <= end_date, + ServiceRequest.type_id.in_(type_ids), + ServiceRequest.council_id.in_(council_ids), + ) + ).group_by( + ServiceRequest.region_id + ).gino.all() + ) + + cluster_list = [] + + for row in result: + region = await Region.get(row[0]) + cluster_list.append(Cluster( + count=row[1], + expansion_zoom=DEFAULT_REGION_ZOOM, + id=region.region_id, + latitude=region.latitude, + longitude=region.longitude + )) + + return cluster_list -# TODO: same as above by group by council async def get_clusters_for_councils( start_date: datetime.date, end_date: datetime.date, @@ -88,17 +134,19 @@ async def get_clusters_for_councils( zoom_current: int ) -> List[Cluster]: """ - Cluster pins for the entire city + Cluster service request pins by council Args: start_date (date): beginning of date range service was requested end_date (date): end of date range service was requested type_ids (List[int]): the request type ids to match on + council_ids (List[int]): the council ids to match on Returns: - cluster: a cluster object + cluster: a list of cluster objects """ + # TODO: CACHE 'council-reqs:start-end-types-councils' result = await ( db.select( [ @@ -120,14 +168,23 @@ async def get_clusters_for_councils( # zoom_next = (zoom_current + 1) or DEFAULT_COUNCIL_ZOOM cluster_list = [] + # TODO: replace this with a caching solution + # returns dictionary with council id as key and name, lat, long + # council_result = await db.all(Council.query) + # councils = [ + # (i.council_id, [i.council_name, i.latitude, i.longitude]) + # for i in council_result + # ] + councils_dict = cache.get("councils_dict") + for row in result: - council = await Council.get(row[0]) + council = councils_dict.get(row[0]) cluster_list.append(Cluster( count=row[1], expansion_zoom=DEFAULT_COUNCIL_ZOOM, - id=council.council_id, - latitude=council.latitude, - longitude=council.longitude + id=row[0], + latitude=council[1], + longitude=council[2] )) return cluster_list @@ -149,7 +206,7 @@ async def get_points( council_ids: (List[int]): the council ids to match Returns: - a list of latitude and logitude pairs of service locations + a list of latitude and logitude pairs of service request locations """ result = await ( @@ -170,12 +227,11 @@ async def get_points( point_list = [] for row in result: - point_list.append([row[0], row[1]]) + point_list.append([row.latitude, row.longitude]) return point_list -# TODO: same as above by group by council async def get_clusters_for_bounds( start_date: datetime.date, end_date: datetime.date, @@ -219,7 +275,7 @@ async def get_clusters_for_bounds( ) # TODO: clean this up. goes in [longitude, latitude] format - points = [[i[2], i[1]] for i in result] + points = [[row.longitude, row.latitude] for row in result] index = pysupercluster.SuperCluster( numpy.array(points), @@ -235,14 +291,15 @@ async def get_clusters_for_bounds( zoom=zoom_current ) - types_dict = await get_types_dict() + # TODO: replace this with a proper caching solution + types_dict = cache.get("types_dict") for item in cluster_list: # change single item clusters into points if item['count'] == 1: pin = result[item['id']] # cluster id matches the result row - item['srnumber'] = "1-" + str(pin[0]) - item['requesttype'] = types_dict[pin[3]] + item['srnumber'] = "1-" + str(pin.request_id) + item['requesttype'] = types_dict[pin.type_id] del item['expansion_zoom'] return cluster_list diff --git a/server/api/code/lacity_data_api/models/council.py b/server/api/code/lacity_data_api/models/council.py index ceb4df737..b43cd8db6 100644 --- a/server/api/code/lacity_data_api/models/council.py +++ b/server/api/code/lacity_data_api/models/council.py @@ -9,3 +9,12 @@ class Council(db.Model): region_id = db.Column(db.SmallInteger) latitude = db.Column(db.Float) longitude = db.Column(db.Float) + + +async def get_councils_dict(): + result = await db.all(Council.query) + councils_dict = [ + (i.council_id, (i.council_name, i.latitude, i.longitude)) + for i in result + ] + return dict(councils_dict) diff --git a/server/api/code/lacity_data_api/models/region.py b/server/api/code/lacity_data_api/models/region.py index ecb14a261..b0af10bfd 100644 --- a/server/api/code/lacity_data_api/models/region.py +++ b/server/api/code/lacity_data_api/models/region.py @@ -8,3 +8,12 @@ class Region(db.Model): region_name = db.Column(db.String) latitude = db.Column(db.Float) longitude = db.Column(db.Float) + + +async def get_regions_dict(): + result = await db.all(Region.query) + regions_dict = [ + (i.region_id, (i.region_name, i.latitude, i.longitude)) + for i in result + ] + return dict(regions_dict) diff --git a/server/api/code/lacity_data_api/models/service_request.py b/server/api/code/lacity_data_api/models/service_request.py index 2aaf6c433..457000ac3 100644 --- a/server/api/code/lacity_data_api/models/service_request.py +++ b/server/api/code/lacity_data_api/models/service_request.py @@ -1,3 +1,5 @@ +from typing import List + from . import db @@ -9,6 +11,17 @@ class ServiceRequest(db.Model): closed_date = db.Column(db.Date) type_id = db.Column(db.SmallInteger) council_id = db.Column(db.SmallInteger) + region_id = db.Column(db.SmallInteger) address = db.Column(db.String) latitude = db.Column(db.Float) longitude = db.Column(db.Float) + + +async def get_open_requests() -> List[ServiceRequest]: + '''Get a list of RequestTypes from their type_names''' + result = await db.all( + ServiceRequest.query.where( + ServiceRequest.closed_date == None # noqa + ) + ) + return result diff --git a/server/api/code/lacity_data_api/routers/api_models.py b/server/api/code/lacity_data_api/routers/api_models.py new file mode 100644 index 000000000..d03918d4b --- /dev/null +++ b/server/api/code/lacity_data_api/routers/api_models.py @@ -0,0 +1,86 @@ +from typing import List, Optional +import datetime +from enum import Enum + +from pydantic import BaseModel, validator +from pydantic.dataclasses import dataclass + + +class Bounds(BaseModel): + north: float + south: float + east: float + west: float + + +class Filter(BaseModel): + startDate: str + endDate: str + ncList: List[int] + requestTypes: List[str] + zoom: Optional[int] = None + bounds: Optional[Bounds] = None + + @validator('startDate', 'endDate') + def parse_date(cls, v): + if isinstance(v, str): + try: + v = datetime.datetime.strptime(v, '%m/%d/%Y') + except ValueError: + try: + v = datetime.datetime.strptime(v, '%Y-%m-%d') + except ValueError: + pass + return v + + +class Pin(BaseModel): + srnumber: str + requesttype: str + latitude: float + longitude: float + + +class Cluster(BaseModel): + count: int + expansion_zoom: Optional[int] + id: int + latitude: float + longitude: float + + +@dataclass +class Set: + district: str + list: List[int] + + @validator('district') + def district_is_valid(cls, v): + assert v in ['cc', 'nc'], 'district must be either "nc" or "cc".' + return v + + def __getitem__(cls, item): + return getattr(cls, item) + + +class Comparison(BaseModel): + startDate: str + endDate: str + requestTypes: List[str] + set1: Set + set2: Set + + +class Feedback(BaseModel): + title: str + body: str + + +class StatusTypes(str, Enum): + api = "api" + database = "db" + system = "sys" + + +Pins = List[Pin] +Clusters = List[Cluster] diff --git a/server/api/code/lacity_data_api/routers/index.py b/server/api/code/lacity_data_api/routers/index.py index d26837768..9a5b4dd68 100644 --- a/server/api/code/lacity_data_api/routers/index.py +++ b/server/api/code/lacity_data_api/routers/index.py @@ -1,8 +1,12 @@ from fastapi import APIRouter +from .utilities import build_cache +from ..config import cache + router = APIRouter() @router.get("/") async def index(): - return {"message": "Hello, new index!"} + await build_cache() + return cache diff --git a/server/api/code/lacity_data_api/routers/legacy.py b/server/api/code/lacity_data_api/routers/legacy.py index c106f934d..ece34585e 100644 --- a/server/api/code/lacity_data_api/routers/legacy.py +++ b/server/api/code/lacity_data_api/routers/legacy.py @@ -1,13 +1,11 @@ -from typing import List, Optional -import datetime -from enum import Enum - from fastapi import responses from fastapi import APIRouter -from pydantic import BaseModel, validator -from pydantic.dataclasses import dataclass +from .api_models import ( + StatusTypes, Filter, Pins, Comparison, Feedback +) from services import status, map, visualizations, requests, comparison, github, email +from .utilities import build_cache router = APIRouter() @@ -17,92 +15,11 @@ """ -class Bounds(BaseModel): - north: float - south: float - east: float - west: float - - -class Filter(BaseModel): - startDate: str - endDate: str - ncList: List[int] - requestTypes: List[str] - zoom: Optional[int] = None - bounds: Optional[Bounds] = None - - @validator('startDate', 'endDate') - def parse_date(cls, v): - if isinstance(v, str): - try: - v = datetime.datetime.strptime(v, '%m/%d/%Y') - except ValueError: - try: - v = datetime.datetime.strptime(v, '%Y-%m-%d') - except ValueError: - pass - return v - - -class Pin(BaseModel): - srnumber: str - requesttype: str - latitude: float - longitude: float - - -Pins = List[Pin] - - -class Cluster(BaseModel): - count: int - expansion_zoom: Optional[int] - id: int - latitude: float - longitude: float - - -Clusters = List[Cluster] - - -@dataclass -class Set: - district: str - list: List[int] - - @validator('district') - def district_is_valid(cls, v): - assert v in ['cc', 'nc'], 'district must be either "nc" or "cc".' - return v - - def __getitem__(cls, item): - return getattr(cls, item) - - -class Comparison(BaseModel): - startDate: str - endDate: str - requestTypes: List[str] - set1: Set - set2: Set - - -class Feedback(BaseModel): - title: str - body: str - - -class StatusTypes(str, Enum): - api = "api" - database = "db" - system = "sys" - - @router.get("/status/{status_type}", description="Provides the status of backend systems") async def status_check(status_type: StatusTypes): if status_type == StatusTypes.api: + await build_cache() result = await status.api() if status_type == StatusTypes.database: result = await status.database() diff --git a/server/api/code/lacity_data_api/routers/shim.py b/server/api/code/lacity_data_api/routers/shim.py index 897149d0c..eafc11477 100644 --- a/server/api/code/lacity_data_api/routers/shim.py +++ b/server/api/code/lacity_data_api/routers/shim.py @@ -1,10 +1,15 @@ -from typing import List, Optional import datetime from fastapi import APIRouter from pydantic import BaseModel -from lacity_data_api.models import clusters, request_type +from .api_models import ( + Filter # StatusTypes, Pins, Comparison, Feedback +) +from ..models import ( + clusters, request_type, service_request +) +from .utilities import build_cache router = APIRouter() @@ -14,24 +19,84 @@ """ -class Bounds(BaseModel): - north: float - south: float - east: float - west: float +class SimpleServiceRequest(BaseModel): + request_id: int + type_id: int + latitude: float + longitude: float + class Config: + orm_mode = True -class Filter(BaseModel): - startDate: str - endDate: str - ncList: List[int] - requestTypes: List[str] - zoom: Optional[int] = None - bounds: Optional[Bounds] = None +@router.get("/status/api") +async def shim_get_api_status(): + currentTime = datetime.datetime.now() + last_pulled = datetime.datetime.now() + await build_cache() + + # SELECT last_pulled FROM metadata + return { + 'currentTime': currentTime, + 'gitSha': "DEVELOPMENT", + 'version': "0.1.1", + 'lastPulled': last_pulled + } + + +# TODO: return format is slightly different than current +@router.get("/servicerequest/{srnumber}", description=""" + The service request ID is the integer created from the srnumber + when the initial "1-" is removed. + """) +async def shim_get_service_request(srnumber: str): + id = int(srnumber[2:]) + result = await service_request.ServiceRequest.get_or_404(id) + return result.to_dict() + + +# TODO: return format is slightly different than current +@router.post("/open-requests") +async def get_open_requests(): + result = await service_request.get_open_requests() + + requests_list = [] + + types_dict = await request_type.get_types_dict() + + for i in result: + requests_list.append({ + 'srnumber': f"1-{i.request_id}", + 'requesttype': types_dict.get(i.type_id), + 'latitude': i.latitude, + 'longitude': i.longitude + }) + + return requests_list + + +@router.post("/map/clusters") +async def get_clusters(filter: Filter): + # convert type names to type ids + request_types = await request_type.get_types_by_str_list(filter.requestTypes) + type_ids = [i.type_id for i in request_types] + + result = await clusters.get_clusters_for_bounds( + filter.startDate, + filter.endDate, + type_ids, + filter.ncList, + filter.zoom, + filter.bounds + ) + + return result + + +# TODO: tries clustering by district and NC first @router.post("/new/clusters") -async def get_new_clusters(filter: Filter): +async def shim_get_clusters(filter: Filter): # have to convert the funky date formats start_date = datetime.datetime.strptime(filter.startDate, '%m/%d/%Y') end_date = datetime.datetime.strptime(filter.endDate, '%m/%d/%Y') @@ -42,15 +107,17 @@ async def get_new_clusters(filter: Filter): zoom = filter.zoom or 10 - if zoom < 11: - # get city clusters - result = await clusters.get_clusters_for_city( + if zoom < 12: + # get region clusters + result = await clusters.get_clusters_for_regions( start_date, end_date, type_ids, + filter.ncList, filter.zoom ) elif zoom < 14: + # get council clusters result = await clusters.get_clusters_for_councils( start_date, end_date, @@ -59,6 +126,7 @@ async def get_new_clusters(filter: Filter): filter.zoom ) else: + # use pysupercluster to cluster viewing area result = await clusters.get_clusters_for_bounds( start_date, end_date, @@ -71,19 +139,85 @@ async def get_new_clusters(filter: Filter): return result -@router.post("/new/heat") -async def get_new_heatmap(filter: Filter): - start_time = datetime.datetime.strptime(filter.startDate, '%m/%d/%Y') - end_time = datetime.datetime.strptime(filter.endDate, '%m/%d/%Y') +@router.post("/map/heat") +async def shim_get_heatmap(filter: Filter): # convert type names to type ids request_types = await request_type.get_types_by_str_list(filter.requestTypes) type_ids = [i.type_id for i in request_types] result = await clusters.get_points( - start_time, - end_time, + filter.startDate, + filter.endDate, type_ids, filter.ncList ) return result + + +# TODO: currently a placeholder +@router.post("/visualizations") +async def shim_get_visualizations(filter: Filter): + result_object = { + "frequency": { + "bins": [ + "2020-01-01", + "2020-01-21", + "2020-02-10", + "2020-03-01", + "2020-03-21", + "2020-04-10", + "2020-04-30", + "2020-05-20", + "2020-06-09", + "2020-06-29", + "2020-07-19", + "2020-08-08", + "2020-08-28", + "2020-09-17" + ], + "counts": { + "Dead Animal Removal": [ + 20, + 31, + 16, + 21, + 16, + 22, + 23, + 15, + 17, + 22, + 19, + 25, + 7 + ] + } + }, + "timeToClose": { + "Dead Animal Removal": { + "min": 0.001632, + "q1": 0.043319, + "median": 0.123883, + "q3": 0.693608, + "max": 2.700694, + "whiskerMin": 0.001632, + "whiskerMax": 1.03765, + "count": 254, + "outlierCount": 2 + } + }, + "counts": { + "type": { + "Dead Animal Removal": 254 + }, + "source": { + "Call": 165, + "Driver Self Report": 1, + "Mobile App": 36, + "Self Service": 50, + "Voicemail": 2 + } + } + } + return result_object diff --git a/server/api/code/lacity_data_api/routers/utilities.py b/server/api/code/lacity_data_api/routers/utilities.py new file mode 100644 index 000000000..775fe4c5d --- /dev/null +++ b/server/api/code/lacity_data_api/routers/utilities.py @@ -0,0 +1,12 @@ + +from ..models import request_type, council, region +from ..config import cache + + +async def build_cache(): + if cache.get('types_dict') is None: + cache['types_dict'] = await request_type.get_types_dict() + if cache.get('councils_dict') is None: + cache['councils_dict'] = await council.get_councils_dict() + if cache.get('regions_dict') is None: + cache['regions_dict'] = await region.get_regions_dict()