From b20fb6064aab85fd3aad9d4af66e21183aa10d68 Mon Sep 17 00:00:00 2001 From: rgao Date: Thu, 26 Dec 2019 05:13:35 -0800 Subject: [PATCH 1/3] successfully returns data with sql->pandas df->json conversion --- server/src/app.py | 12 ++++-- server/src/services/time_to_close.py | 55 ++++++++++++++++++++++------ 2 files changed, 51 insertions(+), 16 deletions(-) diff --git a/server/src/app.py b/server/src/app.py index 275458af2..23775381a 100644 --- a/server/src/app.py +++ b/server/src/app.py @@ -7,6 +7,7 @@ from services.ingress_service import ingress_service from services.reporting import reports from configparser import ConfigParser +from json import loads app = Sanic(__name__) @@ -29,10 +30,13 @@ async def index(request): @app.route('/timetoclose') async def timetoclose(request): ttc_worker = time_to_close(app.config['Settings']) + data = [] # Insert time to close calculation here - return_data = ttc_worker.ttc_query() - - return return_data + column_names = ttc_worker.ttc_view_columns() + all_rows = loads(ttc_worker.ttc_view_all()) + data.append(column_names) + data.append(all_rows) + return json(data) @app.route('/requestfrequency') @@ -41,7 +45,7 @@ async def requestfrequency(request): # Insert frequency calculation here return_data = freq_worker.freq_query() - return return_data + return json(return_data) @app.route('/sample-data') diff --git a/server/src/services/time_to_close.py b/server/src/services/time_to_close.py index f390de0cf..17443b324 100644 --- a/server/src/services/time_to_close.py +++ b/server/src/services/time_to_close.py @@ -1,23 +1,54 @@ from configparser import ConfigParser -from sqlalchemy.types import Integer, Text, String, DateTime, Float -from sqlalchemy import create_engine +import sqlalchemy as db +import pandas as pd class time_to_close(object): - def __init__(self, config=None): + def __init__(self, config=None, tableName="ingest_staging_table"): + """ + Choose table from database by setting the value of tableName. Default table is the staging table. + """ self.config = config self.dbString = None if not self.config else self.config['Database']['DB_CONNECTION_STRING'] + self.table = tableName pass - def ttc_query(self): - engine = create_engine(self.dbString) + def ttc_view_columns(self): + """ + Returns all the columns' names + """ + engine = db.create_engine(self.dbString) - connection = engine.connect() - query = "SELECT row_to_json(row(status)) \ - FROM ingest_staging_table" - result = connection.execute(query) - connection.close() + query = pd.read_sql_query("SELECT * FROM %s" % self.table, con=engine) - return result + return query + + def ttc_view_all(self): + """ + Returns all entries + """ + engine = db.create_engine(self.dbString) + + # The following directly converts SQL data to json objects; for consistency, this function first converts the SQL data to pandas dataframe + # connection = engine.connect() + # query = "SELECT row_to_json(ingest_staging_table) \ + # FROM ingest_staging_table" + # result = connection.execute(query) + # connection.close() + + query = pd.read_sql_query("SELECT * FROM %s" % self.table, con=engine) + + return query.to_json(orient='index') + + ### todo: + ##### request as parameter + ##### filter by closed status + ##### + + ### math method + ##### return in days? + ##### + + ### plot method if __name__ == "__main__": ttc = time_to_close() @@ -25,4 +56,4 @@ def ttc_query(self): config.read("../setting.cfg") ttc.config = config ttc.dbString = config['Database']['DB_CONNECTION_STRING'] - ttc.ttc_query() + ttc.ttc_view_all() From 478448209a39f62943d2df392a6f9b5743c61e13 Mon Sep 17 00:00:00 2001 From: rgao Date: Sat, 28 Dec 2019 19:56:03 -0800 Subject: [PATCH 2/3] successfully converted dates to a consistent human-readable format as JSON objects for view_dates function --- server/src/app.py | 16 +++++--- server/src/services/time_to_close.py | 57 +++++++++++++++++++++------- 2 files changed, 54 insertions(+), 19 deletions(-) diff --git a/server/src/app.py b/server/src/app.py index 23775381a..152b9c162 100644 --- a/server/src/app.py +++ b/server/src/app.py @@ -31,11 +31,17 @@ async def index(request): async def timetoclose(request): ttc_worker = time_to_close(app.config['Settings']) data = [] - # Insert time to close calculation here - column_names = ttc_worker.ttc_view_columns() - all_rows = loads(ttc_worker.ttc_view_all()) - data.append(column_names) - data.append(all_rows) + + # column_names = ttc_worker.ttc_view_columns() + # all_rows = loads(ttc_worker.ttc_view_table(onlyClosed=True)) + + converted_time = loads(ttc_worker.ttc_created_closed_time(serviced=False)) + # average_time = loads(ttc_worker.ttc_average_time()) + + # data.append(column_names) + # data.append(all_rows) + data.append(converted_time) + # data.append(average_time) return json(data) diff --git a/server/src/services/time_to_close.py b/server/src/services/time_to_close.py index 17443b324..c50056a4a 100644 --- a/server/src/services/time_to_close.py +++ b/server/src/services/time_to_close.py @@ -1,6 +1,8 @@ from configparser import ConfigParser import sqlalchemy as db import pandas as pd +from datetime import datetime as dt +import numpy as np class time_to_close(object): def __init__(self, config=None, tableName="ingest_staging_table"): @@ -10,6 +12,7 @@ def __init__(self, config=None, tableName="ingest_staging_table"): self.config = config self.dbString = None if not self.config else self.config['Database']['DB_CONNECTION_STRING'] self.table = tableName + self.data = None pass def ttc_view_columns(self): @@ -18,13 +21,14 @@ def ttc_view_columns(self): """ engine = db.create_engine(self.dbString) - query = pd.read_sql_query("SELECT * FROM %s" % self.table, con=engine) + df = pd.read_sql_query("SELECT * FROM %s" % self.table, con=engine) - return query + return df - def ttc_view_all(self): + def ttc_view_table(self, onlyClosed=False): """ Returns all entries + Returns only those with Status as 'Closed' if onlyClosed is set to True """ engine = db.create_engine(self.dbString) @@ -35,20 +39,45 @@ def ttc_view_all(self): # result = connection.execute(query) # connection.close() - query = pd.read_sql_query("SELECT * FROM %s" % self.table, con=engine) + if onlyClosed: + df = pd.read_sql_query("SELECT * FROM %s WHERE Status = 'Closed'" % self.table, con=engine) + else: + df = pd.read_sql_query("SELECT * FROM %s" % self.table, con=engine) - return query.to_json(orient='index') + return df.to_json(orient='index') - ### todo: - ##### request as parameter - ##### filter by closed status - ##### + def ttc_created_closed_time(self, serviced=False): + """ + Returns all rows under the CreatedDate and ClosedDate columns in ISO8601 format + Returns all rows with a service date under CreatedDate, ClosedDate, and ServicedDate columns if serviced is True + """ + engine = db.create_engine(self.dbString) + + if serviced: + df = pd.read_sql_query("SELECT createddate, closeddate, servicedate FROM %s" % self.table, con=engine) + df = df[df['servicedate'].notnull()] + print(len(df)) + else: + df = pd.read_sql_query("SELECT createddate, closeddate FROM %s" % self.table, con=engine) + print(len(df)) + + self.data = df - ### math method - ##### return in days? - ##### + df['createddate'] = df['createddate'].apply(lambda x: x.strftime('%m/%d/%Y %I:%M:%S %p')) + + return df.to_json(orient='index') + + def ttc_average_time(self): + """ + Returns the average time in days or hours for a specific request type to be completed + """ + df = self.data + + + # pd.Timedelta.total_seconds(df['createddate']) + print(df.dtypes) - ### plot method + return df.to_json(orient='index', date_format='iso', date_unit='s') if __name__ == "__main__": ttc = time_to_close() @@ -56,4 +85,4 @@ def ttc_view_all(self): config.read("../setting.cfg") ttc.config = config ttc.dbString = config['Database']['DB_CONNECTION_STRING'] - ttc.ttc_view_all() + ttc.ttc_view_table() From ad680c6979eda1ed2c75e7c0f70fbf092deb30fa Mon Sep 17 00:00:00 2001 From: rgao Date: Sat, 28 Dec 2019 23:07:03 -0800 Subject: [PATCH 3/3] added function for returning time to close/service for all data --- server/src/app.py | 9 ++-- server/src/services/time_to_close.py | 66 ++++++++++++++++++++-------- 2 files changed, 51 insertions(+), 24 deletions(-) diff --git a/server/src/app.py b/server/src/app.py index 152b9c162..78137da0a 100644 --- a/server/src/app.py +++ b/server/src/app.py @@ -34,14 +34,13 @@ async def timetoclose(request): # column_names = ttc_worker.ttc_view_columns() # all_rows = loads(ttc_worker.ttc_view_table(onlyClosed=True)) - - converted_time = loads(ttc_worker.ttc_created_closed_time(serviced=False)) - # average_time = loads(ttc_worker.ttc_average_time()) + # all_dates = loads(ttc_worker.ttc_view_dates(serviced=False)) + time_diff = loads(ttc_worker.ttc_average_time(serviced=True)) # data.append(column_names) # data.append(all_rows) - data.append(converted_time) - # data.append(average_time) + # data.append(all_dates) + data.append(time_diff) return json(data) diff --git a/server/src/services/time_to_close.py b/server/src/services/time_to_close.py index c50056a4a..2fdf6c27e 100644 --- a/server/src/services/time_to_close.py +++ b/server/src/services/time_to_close.py @@ -4,8 +4,9 @@ from datetime import datetime as dt import numpy as np + class time_to_close(object): - def __init__(self, config=None, tableName="ingest_staging_table"): + def __init__(self, config=None, requestTypes=None, tableName="ingest_staging_table"): """ Choose table from database by setting the value of tableName. Default table is the staging table. """ @@ -23,8 +24,10 @@ def ttc_view_columns(self): df = pd.read_sql_query("SELECT * FROM %s" % self.table, con=engine) + self.data = df + return df - + def ttc_view_table(self, onlyClosed=False): """ Returns all entries @@ -40,49 +43,74 @@ def ttc_view_table(self, onlyClosed=False): # connection.close() if onlyClosed: - df = pd.read_sql_query("SELECT * FROM %s WHERE Status = 'Closed'" % self.table, con=engine) + df = pd.read_sql_query( + "SELECT * FROM %s WHERE Status = 'Closed'" % self.table, con=engine) else: df = pd.read_sql_query("SELECT * FROM %s" % self.table, con=engine) return df.to_json(orient='index') - def ttc_created_closed_time(self, serviced=False): + def ttc_view_dates(self, serviced=False): """ - Returns all rows under the CreatedDate and ClosedDate columns in ISO8601 format + Returns all rows under the CreatedDate and ClosedDate columns in human-readable format Returns all rows with a service date under CreatedDate, ClosedDate, and ServicedDate columns if serviced is True """ engine = db.create_engine(self.dbString) if serviced: - df = pd.read_sql_query("SELECT createddate, closeddate, servicedate FROM %s" % self.table, con=engine) + df = pd.read_sql_query( + "SELECT createddate, closeddate, servicedate FROM %s" % self.table, con=engine) df = df[df['servicedate'].notnull()] - print(len(df)) else: - df = pd.read_sql_query("SELECT createddate, closeddate FROM %s" % self.table, con=engine) - print(len(df)) - - self.data = df + df = pd.read_sql_query( + "SELECT createddate, closeddate FROM %s" % self.table, con=engine) - df['createddate'] = df['createddate'].apply(lambda x: x.strftime('%m/%d/%Y %I:%M:%S %p')) + df['createddate'] = df['createddate'].apply( + lambda x: x.strftime('%m/%d/%Y %I:%M:%S %p')) return df.to_json(orient='index') - def ttc_average_time(self): + def ttc_time_diff(self, serviced=False, all=False): """ Returns the average time in days or hours for a specific request type to be completed """ - df = self.data + engine = db.create_engine(self.dbString) + + if serviced: + df = pd.read_sql_query( + "SELECT createddate, closeddate, servicedate FROM %s" % self.table, con=engine) + df = df[df['servicedate'].notnull()] + df['servicedate'] = pd.to_datetime(df['servicedate']) + diff_df = pd.DataFrame( + df['servicedate'] - df['createddate'], columns=['time_to_service']) + else: + df = pd.read_sql_query( + "SELECT createddate, closeddate FROM %s" % self.table, con=engine) + diff_df = pd.DataFrame({'time_to_close': []}) + + df['createddate'] = pd.to_datetime(df['createddate']) + df['closeddate'] = pd.to_datetime(df['closeddate']) + diff_df['time_to_close'] = df['closeddate'] - df['createddate'] + + def dt_to_days(dt): + num_days = pd.Timedelta.total_seconds(dt)/(24.*3600) + if num_days <= .000001: + return 0 + return pd.Timedelta.total_seconds(dt)/(24.*3600) + + diff_df['time_to_close'] = diff_df.time_to_close.apply(dt_to_days) + diff_df['time_to_service'] = diff_df.time_to_service.apply(dt_to_days) - - # pd.Timedelta.total_seconds(df['createddate']) - print(df.dtypes) + ### Todo: Convert unix time to strings displaying days + ### Todo: Return averages and min/max + ### Todo: Implement function for considering request type - return df.to_json(orient='index', date_format='iso', date_unit='s') + return diff_df.to_json(orient='index') if __name__ == "__main__": ttc = time_to_close() config = ConfigParser() config.read("../setting.cfg") - ttc.config = config + ttc.config = config ttc.dbString = config['Database']['DB_CONNECTION_STRING'] ttc.ttc_view_table()