Merge branch 'dev' into 363_DOC_PRTemplate

server/src/app.py

@@ -11,6 +11,7 @@
 from services.time_to_close import time_to_close
 from services.frequency import frequency
 from services.pinService import PinService
+from services.requestCountsService import RequestCountsService
 from services.requestDetailService import RequestDetailService
 from services.ingress_service import ingress_service
 from services.sqlIngest import DataHandler
@@ -140,6 +141,25 @@ async def pinMap(request):
     return json(return_data)
 
 
+@app.route('/requestcounts', methods=["POST"])
+@compress.compress()
+async def requestCounts(request):
+    counts_worker = RequestCountsService(app.config['Settings'])
+    postArgs = request.json
+    start = postArgs.get('startDate', None)
+    end = postArgs.get('endDate', None)
+    ncs = postArgs.get('ncList', [])
+    requests = postArgs.get('requestTypes', [])
+    countFields = postArgs.get('countFields', [])
+
+    return_data = await counts_worker.get_req_counts(startDate=start,
+                                                     endDate=end,
+                                                     ncList=ncs,
+                                                     requestTypes=requests,
+                                                     countFields=countFields)
+    return json(return_data)
+
+
 @app.route('/servicerequest/<srnumber>', methods=["GET"])
 async def requestDetails(request, srnumber):
     detail_worker = RequestDetailService(app.config['Settings'])

server/src/services/dataService.py

@@ -1,6 +1,8 @@
 import datetime
 import pandas as pd
 import sqlalchemy as db
+from sqlalchemy.orm import sessionmaker
+from .databaseOrm import Ingest as Request
 
 
 class DataService(object):
@@ -9,7 +11,6 @@ def innerFunc(*args, **kwargs):
             dataResponse = func(*args, **kwargs)
             if 'Error' in dataResponse:
                 return dataResponse
-
             # Will represent last time the ingest pipeline ran
             lastPulledTimestamp = datetime.datetime.utcnow()
             withMeta = {'lastPulled': lastPulledTimestamp,
@@ -26,22 +27,74 @@ def __init__(self, config=None, tableName="ingest_staging_table"):
         self.table = tableName
         self.data = None
         self.engine = db.create_engine(self.dbString)
+        self.session = sessionmaker(bind=self.engine)()
+
+    def standardFilters(self,
+                        startDate=None,
+                        endDate=None,
+                        ncList=[],
+                        requestTypes=[]):
+        '''
+        Generates filters for dates, ncs, and request types.
+        '''
+
+        return [
+            Request.createddate > startDate if startDate else True,
+            Request.createddate < endDate if endDate else True,
+            Request.ncname.in_(ncList) if ncList else True,
+            Request.requesttype.in_(requestTypes) if requestTypes else True
+        ]
+
+    @includeMeta
+    def itemQuery(self, requestNumber):
+        '''
+        Returns a single request by its requestNumber.
+        '''
+
+        if not requestNumber or not isinstance(requestNumber, str):
+            return {'Error': 'Missing request number'}
+
+        return self.session \
+            .query(Request) \
+            .get(requestNumber) \
+            ._asdict()
 
     @includeMeta
-    def query(self, queryItems=None, queryfilters=[], limit=None):
+    def query(self, queryItems=[], queryFilters=[], limit=None):
+        '''
+        Returns the specified properties of each request,
+        after filtering by queryFilters and applying the limit.
+        '''
+
         if not queryItems or not isinstance(queryItems, list):
             return {'Error': 'Missing query items'}
 
-        items = ', '.join(queryItems)
-        query = 'SELECT {} FROM {}'.format(items, self.table)
-        if queryfilters:
-            filters = ' AND '.join(queryfilters)
-            query += ' WHERE {}'.format(filters)
-        if limit:
-            query += ' LIMIT {}'.format(limit)
+        selectFields = [getattr(Request, item) for item in queryItems]
+        records = self.session \
+            .query(*selectFields) \
+            .filter(*queryFilters) \
+            .limit(limit) \
+            .all()
+
+        return [rec._asdict() for rec in records]
+
+    @includeMeta
+    def aggregateQuery(self, countFields=[], queryFilters=[]):
+        '''
+        Returns the counts of distinct values in the specified fields,
+        after filtering by queryFilters.
+        '''
+
+        if not countFields or not isinstance(countFields, list):
+            return {'Error': 'Missing count fields'}
+
+        filteredData = self.query(countFields, queryFilters)
+        df = pd.DataFrame(data=filteredData['data'])
 
-        df = pd.read_sql_query(query, con=self.engine)
-        return df.to_dict(orient='records')
+        return [{
+            'field': field,
+            'counts': df.groupby(by=field).size().to_dict()
+        } for field in countFields]
 
     def storedProc(self):
         pass

server/src/services/databaseOrm.py

@@ -5,7 +5,16 @@
 Base = declarative_base()
 
 
-class Ingest(Base):
+class Mixin:
+    """
+    Adds `_asdict()` to easily serialize objects to dictionaries.
+    """
+    def _asdict(self):
+        cols = self.__table__.columns
+        return {col.name: getattr(self, col.name) for col in cols}
+
+
+class Ingest(Base, Mixin):
     __tablename__ = 'ingest_staging_table'
     srnumber = Column(String(50), primary_key=True, unique=True)
     createddate = Column(DateTime)

server/src/services/pinService.py

@@ -6,8 +6,8 @@ def __init__(self, config=None, tableName="ingest_staging_table"):
         self.dataAccess = DataService(config, tableName)
 
     async def get_base_pins(self,
-                            startDate='',
-                            endDate='',
+                            startDate=None,
+                            endDate=None,
                             ncList=[],
                             requestTypes=[]):
         """
@@ -28,13 +28,7 @@ async def get_base_pins(self,
                  'latitude',
                  'longitude']
 
-        ncs = '\'' + '\', \''.join(ncList) + '\''
-        requests = '\'' + '\', \''.join(requestTypes) + '\''
+        filters = self.dataAccess.standardFilters(
+            startDate, endDate, ncList, requestTypes)
 
-        filters = ['createddate > \'{}\''.format(startDate),
-                   'createddate < \'{}\''.format(endDate),
-                   'ncname IN ({})'.format(ncs),
-                   'requesttype IN ({})'.format(requests)]
-        result = self.dataAccess.query(items, filters)
-
-        return result
+        return self.dataAccess.query(items, filters)

server/src/services/requestCountsService.py

@@ -0,0 +1,44 @@
+from .dataService import DataService
+
+
+class RequestCountsService(object):
+    def __init__(self, config=None, tableName="ingest_staging_table"):
+        self.dataAccess = DataService(config, tableName)
+
+    async def get_req_counts(self,
+                             startDate=None,
+                             endDate=None,
+                             ncList=[],
+                             requestTypes=[],
+                             countFields=[]):
+        """
+        For each countField, returns the counts of each distinct value
+        in that field, given times, ncs, and request filters.
+        E.g. if countsFields is ['requesttype', 'requestsource'], returns:
+        {
+            'lastPulled': 'Timestamp',
+            'data': [
+                {
+                    'field': 'requesttype',
+                    'counts': {
+                        'Graffiti Removal': 'Int',
+                        'Bulky Items': 'Int',
+                        ...
+                    }
+                },
+                {
+                    'field': 'requestsource',
+                    'counts': {
+                        'Mobile App': 'Int',
+                        'Driver Self Report': 'Int',
+                        ...
+                    }
+                }
+            ]
+        }
+        """
+
+        filters = self.dataAccess.standardFilters(
+            startDate, endDate, ncList, requestTypes)
+
+        return self.dataAccess.aggregateQuery(countFields, filters)

server/src/services/requestDetailService.py

@@ -25,8 +25,4 @@ async def get_request_detail(self, requestNumber=None):
         }
         """
 
-        items = ['*']
-        filters = ['srnumber = \'{}\''.format(requestNumber)]
-        result = self.dataAccess.query(items, filters)
-
-        return result
+        return self.dataAccess.itemQuery(requestNumber)