Investigate /requests performance

Fixes #986 - trying LIMIT OFFSET from PostgreSQL/Sql Alchemy - removing transaction/curson
hackforla · Mar 3, 2021 · 70b5616 · 70b5616
1 parent f30321f
commit 70b5616
Show file tree

Hide file tree

Showing 3 changed files with 38 additions and 32 deletions.
diff --git a/server/api/code/lacity_data_api/models/service_request.py b/server/api/code/lacity_data_api/models/service_request.py
@@ -168,7 +168,7 @@ async def get_filtered_requests(
         type_ids: List[int] = None,
         council_ids: List[int] = None,
         include_updated: bool = False,
-        skip: int = 0,
+        offset: int = 0,
         limit: int = 100000
 ):
     from .council import Council  # noqa ... avoiding circular import
@@ -196,35 +196,34 @@ async def get_filtered_requests(
     if (council_ids):
         where_text.append(f"service_requests.council_id IN ({', '.join([str(i) for i in council_ids])})")  # noqa
 
-    async with db.transaction():
-        cursor = await (
-            db.select(
-                [
-                    ServiceRequest,
-                    RequestType.type_name,
-                    Council.council_name,
-                    Agency.agency_name,
-                    Source.source_name,
-                ]
-            ).select_from(
-                ServiceRequest.join(
-                    RequestType
-                ).join(
-                    Council
-                ).join(
-                    Agency, ServiceRequest.agency_id == Agency.agency_id
-                ).join(
-                    Source, ServiceRequest.source_id == Source.source_id
-                )
-            ).where(
-                text(" AND ".join(where_text))
-            ).order_by(
-                desc(ServiceRequest.created_date)
-            ).gino.iterate()
-        )
-
-        if skip > 0:
-            await cursor.forward(skip)  # skip rows
-        result = await cursor.many(limit)  # and retrieve limit rows
+    result = await (
+        db.select(
+            [
+                ServiceRequest,
+                RequestType.type_name,
+                Council.council_name,
+                Agency.agency_name,
+                Source.source_name,
+            ]
+        ).select_from(
+            ServiceRequest.join(
+                RequestType
+            ).join(
+                Council
+            ).join(
+                Agency, ServiceRequest.agency_id == Agency.agency_id
+            ).join(
+                Source, ServiceRequest.source_id == Source.source_id
+            )
+        ).where(
+            text(" AND ".join(where_text))
+        ).order_by(
+            desc(ServiceRequest.created_date)
+        ).limit(
+            limit
+        ).offset(
+            offset
+        ).gino.all()
+    )
 
     return result
diff --git a/server/api/code/lacity_data_api/routers/reports.py b/server/api/code/lacity_data_api/routers/reports.py
@@ -66,6 +66,13 @@ async def run_report(
     fields = [field_dict[i] for i in field]
     group_by = fields.copy()
     fields.append(db.func.count().label("counts"))
+    # TODO: good idea, but need to figure out best way to present open vs. closed requests  # noqa
+    # fields.append(db.func.sum(
+    #     (cast(ServiceRequest.closed_date, DATE) - cast(ServiceRequest.created_date, DATE))  # noqa
+    # ).label("total_days"))
+    # fields.append(db.func.count(
+    #     ServiceRequest.closed_date
+    # ).label("total_closed"))
 
     # set up filters for where clause
     filters = []

diff --git a/server/api/code/lacity_data_api/routers/service_requests.py b/server/api/code/lacity_data_api/routers/service_requests.py
@@ -39,7 +39,7 @@ async def get_all_service_requests(
         end_date=end_date,
         type_ids=type_ids,
         council_ids=council_ids,
-        skip=skip,
+        offset=skip,
         limit=limit
     )