From d1aa1dd882467d8e9ac55811b322a52fec283668 Mon Sep 17 00:00:00 2001 From: Yi Heng Joshua Wu Date: Mon, 13 Jun 2022 13:46:51 -0700 Subject: [PATCH 01/53] Updte dashboard to newest version --- server/dash/app.py | 6 +- server/dash/dashboards/neighborhood.py | 73 +++++++++---------- server/dash/dashboards/neighborhood_recent.py | 70 ++++++++++++------ server/dash/dashboards/overview.py | 65 ++++++++--------- server/dash/dashboards/recent.py | 25 +++---- server/dash/dashboards/types_map.py | 26 +++---- server/dash/index.py | 10 +-- server/dash/requirements.txt | 32 ++++---- 8 files changed, 162 insertions(+), 145 deletions(-) diff --git a/server/dash/app.py b/server/dash/app.py index 36dbea8d9..98cc3a88f 100644 --- a/server/dash/app.py +++ b/server/dash/app.py @@ -1,14 +1,12 @@ import json import urllib -import dash -import dash_core_components as dcc -import dash_html_components as html +from dash import Dash, html, dcc import pandas as pd external_stylesheets = ['/static/reports.css'] -app = dash.Dash(__name__, external_stylesheets=external_stylesheets) +app = Dash(__name__, external_stylesheets=external_stylesheets) # set up default layout app.layout = html.Div([ diff --git a/server/dash/dashboards/neighborhood.py b/server/dash/dashboards/neighborhood.py index f7ec9a2cf..4a5215f96 100644 --- a/server/dash/dashboards/neighborhood.py +++ b/server/dash/dashboards/neighborhood.py @@ -1,20 +1,13 @@ import textwrap -import dash_core_components as dcc -import dash_html_components as html +from dash import Dash, dcc, html, callback import pandas as pd import plotly.express as px -from dash.dependencies import Input -from dash.dependencies import Output +from dash.dependencies import Input, Output -from app import app +# from app import app from config import API_HOST -from design import apply_figure_style -from design import CONFIG_OPTIONS -from design import DISCRETE_COLORS -from design import DISCRETE_COLORS_MAP -from design import LABELS - +from design import CONFIG_OPTIONS, DISCRETE_COLORS, DISCRETE_COLORS_MAP, LABELS, apply_figure_style # TITLE title = "NEIGHBORHOODS" @@ -41,13 +34,37 @@ def populate_options(): return values +layout = html.Div([ + html.H1(title), + dcc.Dropdown( + id='council_list', + clearable=False, + value="Arleta", + placeholder="Select a neighborhood", + options=populate_options() + ), + dcc.Graph( + id='ncAvgCompLineChart', + figure=fig, + config=CONFIG_OPTIONS + ), + dcc.Graph( + id='ncNumReqTypeLineChart', + figure=fig, + config=CONFIG_OPTIONS + ) +]) + + + + + # Define callback to update graph -@app.callback( - Output('graph1', 'figure'), +@callback( + Output('ncAvgCompLineChart', 'figure'), [Input("council_list", "value")] ) def update_figure(selected_council): - neighborhood_sum_df = df[df.council_name == selected_council].groupby(['created_date']).agg('sum').reset_index() # noqa total_sum_df = df.groupby(['created_date']).agg('sum').reset_index() total_sum_df["nc_avg"] = total_sum_df["counts"] / 99 @@ -59,7 +76,7 @@ def update_figure(selected_council): y=['counts', 'nc_avg'], color_discrete_sequence=DISCRETE_COLORS, labels=LABELS, - title="Comparison trend for " + selected_council + title= "Number of " + selected_council + " Requests compare with the average of all Neighborhood Councils requests" ) fig.update_xaxes( @@ -76,8 +93,8 @@ def update_figure(selected_council): # Define callback to update graph -@app.callback( - Output('graph2', 'figure'), +@callback( + Output('ncNumReqTypeLineChart', 'figure'), [Input("council_list", "value")] ) def update_council_figure(selected_council): @@ -93,7 +110,7 @@ def update_council_figure(selected_council): color_discrete_sequence=DISCRETE_COLORS, color_discrete_map=DISCRETE_COLORS_MAP, labels=LABELS, - title="Request type trend for " + selected_council + title="Number of Different Requests Types for " + selected_council ) fig.update_xaxes( @@ -109,23 +126,3 @@ def update_council_figure(selected_council): return fig -layout = html.Div([ - html.H1(title), - dcc.Dropdown( - id='council_list', - clearable=False, - value="Arleta", - placeholder="Select a neighborhood", - options=populate_options() - ), - dcc.Graph( - id='graph1', - figure=fig, - config=CONFIG_OPTIONS - ), - dcc.Graph( - id='graph2', - figure=fig, - config=CONFIG_OPTIONS - ) -]) diff --git a/server/dash/dashboards/neighborhood_recent.py b/server/dash/dashboards/neighborhood_recent.py index badce73a1..0e59dab5c 100644 --- a/server/dash/dashboards/neighborhood_recent.py +++ b/server/dash/dashboards/neighborhood_recent.py @@ -1,12 +1,12 @@ import datetime import textwrap +import json +import urllib -import dash_core_components as dcc -import dash_html_components as html -import dash_table +from dash import dcc, html, dash_table, callback import pandas as pd import plotly.express as px -from app import app, batch_get_data +#from app import app, batch_get_data from config import API_HOST from dash.dependencies import Input, Output from design import CONFIG_OPTIONS, DISCRETE_COLORS, DISCRETE_COLORS_MAP, LABELS, apply_figure_style @@ -31,6 +31,33 @@ # DATA df_path = f"/requests/updated?start_date={start_date}&end_date={end_date}" print(" * Downloading data for dataframe") + +BATCH_SIZE = 10000 + +def batch_get_data(url): + # set up your query + if '?' in url: + batch_url = f"{url}&limit={BATCH_SIZE}" + else: + batch_url = f"{url}?limit={BATCH_SIZE}" + + response_size = BATCH_SIZE + result_list = [] + skip = 0 + + # loop through query results and add to a list (better performance!) + while (response_size == BATCH_SIZE): + batch = json.loads(urllib.request.urlopen(f"{batch_url}&skip={skip}").read()) + result_list.extend(batch) + response_size = len(batch) + skip = skip + BATCH_SIZE + + # convert JSON object list to dataframe + df = pd.DataFrame.from_records(result_list) + + return df + + df = batch_get_data(API_HOST + df_path) df['createdDate'] = pd.to_datetime( df['createdDate'], errors='coerce').dt.strftime('%Y-%m-%d') @@ -106,31 +133,31 @@ def populate_options(): html.Div(f"Weekly report ({start_date.strftime('%b %d')} to {end_date.strftime('%b %d')})"), # noqa html.Div([ html.Div( - [html.H2(id="created_txt"), html.Label("New Requests")], + [html.H2(id="numNewRequests"), html.Label("New Requests")], className="stats-label" ), html.Div( - [html.H2(id="closed_txt"), html.Label("Closed Requests")], + [html.H2(id="numClosedRequests"), html.Label("Closed Requests")], className="stats-label" ), html.Div( - [html.H2(id="net_txt"), html.Label("Net Change")], + [html.H2(id="closeMinusNew"), html.Label("Net Change")], className="stats-label" ), ], className="graph-row"), html.Div([ html.Div( - dcc.Graph(id='graph', figure=fig, config=CONFIG_OPTIONS), + dcc.Graph(id='newRequestsLineChart', figure=fig, config=CONFIG_OPTIONS), className="half-graph" ), html.Div( - dcc.Graph(id='pie_graph', figure=pie_fig, config=CONFIG_OPTIONS), + dcc.Graph(id='neighRecentReqTypePieChart', figure=pie_fig, config=CONFIG_OPTIONS), className="half-graph" ) ]), html.Div( dash_table.DataTable( - id='council_table', + id='neighCouncilTbl', columns=[ {"name": pretty_columns[i], "id": i} for i in table_df.columns ], @@ -156,8 +183,8 @@ def populate_options(): # Define callback to update graph -@app.callback( - Output("council_table", "data"), +@callback( + Output("neighCouncilTbl", "data"), Input("council_list", "value") ) def update_table(selected_council): @@ -165,11 +192,11 @@ def update_table(selected_council): return table_df.to_dict('records') -@app.callback( +@callback( [ - Output("created_txt", "children"), - Output("closed_txt", "children"), - Output("net_txt", "children"), + Output("numNewRequests", "children"), + Output("numClosedRequests", "children"), + Output("closeMinusNew", "children"), ], Input("council_list", "value") ) @@ -179,8 +206,8 @@ def update_text(selected_council): return create_count, close_count, create_count - close_count -@app.callback( - Output("graph", "figure"), +@callback( + Output("newRequestsLineChart", "figure"), Input("council_list", "value") ) def update_figure(selected_council): @@ -194,7 +221,7 @@ def update_figure(selected_council): color="typeName", color_discrete_sequence=DISCRETE_COLORS, labels=LABELS, - title="New Requests" + title="Number of new " + selected_council + " Requests" ) fig.update_xaxes( tickformat="%a\n%m/%d", @@ -208,8 +235,8 @@ def update_figure(selected_council): return fig -@app.callback( - Output("pie_graph", "pie_fig"), +@callback( + Output("neighRecentReqTypePieChart", "pie_fig"), Input("council_list", "value") ) def update_council_figure(selected_council): @@ -221,6 +248,7 @@ def update_council_figure(selected_council): values="srnumber", color_discrete_sequence=DISCRETE_COLORS, labels=LABELS, + title="Share of new requests types for " + selected_council ) apply_figure_style(pie_fig) diff --git a/server/dash/dashboards/overview.py b/server/dash/dashboards/overview.py index e4649be46..f4d9427fb 100644 --- a/server/dash/dashboards/overview.py +++ b/server/dash/dashboards/overview.py @@ -1,17 +1,12 @@ import textwrap -import dash_core_components as dcc -import dash_html_components as html +from dash import dcc, html import pandas as pd import plotly.express as px import plotly.graph_objects as go from config import API_HOST -from design import apply_figure_style -from design import CONFIG_OPTIONS -from design import DISCRETE_COLORS -from design import LABELS -from design import DISCRETE_COLORS_MAP +from design import CONFIG_OPTIONS, DISCRETE_COLORS, DISCRETE_COLORS_MAP, LABELS, apply_figure_style # TITLE @@ -23,13 +18,13 @@ query_string = "/reports?field=council_name&filter=created_date>=2016-01-01" df1 = pd.read_json(API_HOST + query_string) df1 = df1.groupby(['council_name'])['counts'].sum().sort_values().to_frame() -fig1 = px.bar( +reqByNcBarChart = px.bar( df1, x=df1.index, y='counts', color_discrete_sequence=DISCRETE_COLORS, labels=LABELS, - title="Total Requests by Neighborhood", + title="Total Requests by Neighborhood Councils", ) # year totals figure @@ -37,7 +32,7 @@ query_string = "/reports?field=created_year&filter=created_date>=2016-01-01" df2 = pd.read_json(API_HOST + query_string) df2 = df2.groupby(['created_year'])['counts'].sum().to_frame() -fig2 = px.bar( +numReqByYearBarChart = px.bar( df2, x=df2.index, y='counts', @@ -57,7 +52,7 @@ df5 = df5[:5] df5.index = df5.index.map(lambda x: '
'.join(textwrap.wrap(x, width=16))) -fig5 = px.pie( +shareReqByAgencyPieChart = px.pie( df5, names=df5.index, values='counts', @@ -76,7 +71,7 @@ df6.loc['Others'] = df6[4:].sum() df6.sort_values('counts', ascending=False, inplace=True) df6 = df6[:5] -fig6 = px.bar( +reqSourceBarchart = px.bar( df6, x=df6.index, y='counts', @@ -90,7 +85,7 @@ query_string = "/reports?field=type_name&filter=created_date>=2016-01-01" df3 = pd.read_json(API_HOST + query_string) df3 = df3.groupby(['type_name'], as_index=False)['counts'].sum() -fig3 = px.pie( +reqTypeSharePieChart = px.pie( df3, names="type_name", values="counts", @@ -106,9 +101,9 @@ stas_df = pd.read_json(API_HOST + '/types/stats') stas_df = stas_df.sort_values('median', ascending=False) -fig4 = go.Figure() +medDaysToCloseBoxPlot = go.Figure() -fig4.add_trace( +medDaysToCloseBoxPlot.add_trace( go.Box( y=stas_df.type_name, q1=stas_df['q1'], @@ -119,22 +114,22 @@ ) ) -fig4.update_xaxes( +medDaysToCloseBoxPlot.update_xaxes( dtick=5 ) -fig4.update_layout( +medDaysToCloseBoxPlot.update_layout( title="Total Median Days to Close by Type", ) # apply shared styles -apply_figure_style(fig1) -apply_figure_style(fig2) -apply_figure_style(fig3) -apply_figure_style(fig4) -apply_figure_style(fig5) -apply_figure_style(fig6) +apply_figure_style(reqByNcBarChart) +apply_figure_style(numReqByYearBarChart) +apply_figure_style(reqTypeSharePieChart) +apply_figure_style(medDaysToCloseBoxPlot) +apply_figure_style(shareReqByAgencyPieChart) +apply_figure_style(reqSourceBarchart) # LAYOUT layout = html.Div([ @@ -147,43 +142,43 @@ ], className="graph-row"), html.Div([ dcc.Graph( - id='graph2', - figure=fig2, + id='numReqByYearBarChart', + figure=numReqByYearBarChart, config=CONFIG_OPTIONS, className="half-graph" ), dcc.Graph( - id='graph3', - figure=fig3, + id='reqTypeSharePieChart', + figure=reqTypeSharePieChart, config=CONFIG_OPTIONS, className="half-graph" )], className="graph-row"), dcc.Graph( - id='graph4', - figure=fig4, + id='medDaysToCloseBoxPlot', + figure=medDaysToCloseBoxPlot, config=CONFIG_OPTIONS, responsive=True, ), html.Div([ dcc.Graph( - id='graph5', - figure=fig5, + id='shareReqByAgencyPieChart', + figure=shareReqByAgencyPieChart, config=CONFIG_OPTIONS, className="half-graph", responsive=True, ), dcc.Graph( - id='graph6', - figure=fig6, + id='reqSourceBarchart', + figure=reqSourceBarchart, config=CONFIG_OPTIONS, className="half-graph", responsive=True, )], className="graph-row"), dcc.Graph( - id='graph1', - figure=fig1, + id='reqByNcBarChart', + figure=reqByNcBarChart, config=CONFIG_OPTIONS, responsive=True, ), diff --git a/server/dash/dashboards/recent.py b/server/dash/dashboards/recent.py index 14307828c..bfcc90eaa 100644 --- a/server/dash/dashboards/recent.py +++ b/server/dash/dashboards/recent.py @@ -1,8 +1,7 @@ import datetime import textwrap -import dash_core_components as dcc -import dash_html_components as html +from dash import dcc, html import pandas as pd import plotly.express as px import plotly.graph_objects as go @@ -27,7 +26,7 @@ report_df.type_name = report_df.type_name.map(lambda x: '
'.join(textwrap.wrap(x, width=16))) # noqa # Line Graph -fig = px.line( +numCreatedReqLineChart = px.line( report_df, x="created_date", y="counts", @@ -37,17 +36,17 @@ labels=LABELS, ) -fig.update_xaxes( +numCreatedReqLineChart.update_xaxes( tickformat="%a\n%m/%d", ) -fig.update_traces( +numCreatedReqLineChart.update_traces( mode='markers+lines' ) # add markers to lines # Pie Chart pie_df = df.groupby(['type_name']).agg('sum').reset_index() -pie_fig = px.pie( +recentReqTypeSharePieChart = px.pie( pie_df, names="type_name", values="counts", @@ -61,7 +60,7 @@ df['created_date'] = pd.to_datetime(df['created_date']) dow_df = df.groupby(['created_date']).agg('sum').reset_index() dow_df['day_of_week'] = dow_df['created_date'].dt.day_name() -dow_fig = px.bar( +numReqByDayOfWeekBarChart = px.bar( dow_df, x="day_of_week", y="counts", @@ -69,9 +68,9 @@ labels=LABELS, ) -apply_figure_style(fig) -apply_figure_style(pie_fig) -apply_figure_style(dow_fig) +apply_figure_style(numCreatedReqLineChart) +apply_figure_style(recentReqTypeSharePieChart) +apply_figure_style(numReqByDayOfWeekBarChart) # LAYOUT layout = html.Div([ @@ -82,9 +81,9 @@ html.Div([html.H2(f"{start_date.strftime('%b %d')}"), html.Label("Report Start Date")], className="stats-label"), # noqa html.Div([html.H2(f"{end_date.strftime('%b %d')}"), html.Label("Report End Date")], className="stats-label"), # noqa ], className="graph-row", style={'color':'white'}), - dcc.Graph(id='graph', figure=fig, config=CONFIG_OPTIONS), + dcc.Graph(id='graph', figure=numCreatedReqLineChart, config=CONFIG_OPTIONS), html.Div([ - dcc.Graph(id='graph3', figure=dow_fig, config=CONFIG_OPTIONS, className="half-graph"), # noqa - dcc.Graph(id='graph2', figure=pie_fig, config=CONFIG_OPTIONS, className="half-graph"), # noqa + dcc.Graph(id='numReqByDayOfWeekBarChart', figure=numReqByDayOfWeekBarChart, config=CONFIG_OPTIONS, className="half-graph"), # noqa + dcc.Graph(id='recentReqTypeSharePieChart', figure=recentReqTypeSharePieChart, config=CONFIG_OPTIONS, className="half-graph"), # noqa ], className="graph-row"), ]) diff --git a/server/dash/dashboards/types_map.py b/server/dash/dashboards/types_map.py index dad551eb9..6c0ac6c29 100644 --- a/server/dash/dashboards/types_map.py +++ b/server/dash/dashboards/types_map.py @@ -2,18 +2,15 @@ import json from urllib.request import urlopen -import dash_core_components as dcc -import dash_html_components as html +from dash import dcc, html, callback import pandas as pd import plotly.express as px -from dash.dependencies import Input -from dash.dependencies import Output +from dash.dependencies import Input, Output -from app import app +# from app import app from config import API_HOST -from design import apply_figure_style -from design import CONFIG_OPTIONS -from design import LABELS +from design import CONFIG_OPTIONS, LABELS, apply_figure_style + # TITLE @@ -43,21 +40,20 @@ def populate_options(): layout = html.Div([ html.H1(title), dcc.Dropdown( - id="types", + id="typesMapReqtypes", clearable=False, value="Illegal Dumping", placeholder="Select a type", searchable=False, options=populate_options(), ), - dcc.Graph(id="choropleth", config=CONFIG_OPTIONS, className="hidden-graph"), + dcc.Graph(id="reqTypeChoroplethGraph", config=CONFIG_OPTIONS, className="hidden-graph"), ]) -@app.callback( - Output("choropleth", "figure"), - Output("choropleth", "className"), - [Input("types", "value")] +@callback( + Output("reqTypeChoroplethGraph", "figure"), + [Input("typesMapReqtypes", "value")] ) def display_choropleth(selected_value): fig = px.choropleth( @@ -90,4 +86,4 @@ def display_choropleth(selected_value): colorbar_len=0.3, ) apply_figure_style(fig) - return fig, "visible-graph" + return fig diff --git a/server/dash/index.py b/server/dash/index.py index 88f98effe..4395b4728 100644 --- a/server/dash/index.py +++ b/server/dash/index.py @@ -5,10 +5,8 @@ import re import signal -import dash_core_components as dcc -import dash_html_components as html -from dash.dependencies import Input -from dash.dependencies import Output +from dash import Dash, html, dcc, callback, Input, Output +# from dash.dependencies import Input, Output from app import app from config import DASH_FILES @@ -37,12 +35,12 @@ # callback to handle requests -@app.callback(Output('page-content', 'children'), +@callback(Output('page-content', 'children'), Input('url', 'pathname')) def display_page(pathname): last_part = re.search("([\w]*)$", pathname).group(1) # noqa - + print(last_part) # run the dashboard if last_part in available_dashboards: logger.log(logging.INFO, f"Running dashboard: {last_part}") diff --git a/server/dash/requirements.txt b/server/dash/requirements.txt index 15d9a34b0..afc64990a 100644 --- a/server/dash/requirements.txt +++ b/server/dash/requirements.txt @@ -1,22 +1,28 @@ +# Dash dependencies Brotli==1.0.9 -click==7.1.2 -dash==1.19.0 -dash-core-components==1.15.0 -dash-html-components==1.1.2 -dash-renderer==1.9.0 -dash-table==4.11.2 -Flask==1.1.2 -Flask-Compress==1.8.0 +click==8.1.3 +colorama==0.4.4 +dash==2.4.1 +dash-core-components==2.0.0 +dash-html-components==2.0.0 +dash-table==5.0.0 +Flask==2.1.2 +Flask-Compress==1.12 +importlib-metadata==4.11.4 +itsdangerous==2.1.2 +Jinja2==3.1.2 +MarkupSafe==2.1.1 +plotly==5.8.2 +tenacity==8.0.1 +Werkzeug==2.1.2 +zipp==3.8.0 + +# Other dependencies future==0.18.2 gunicorn==20.0.4 -itsdangerous==1.1.0 -Jinja2==2.11.3 -MarkupSafe==1.1.1 numpy==1.19.5 pandas==1.1.5 -plotly==4.14.3 python-dateutil==2.8.1 pytz==2021.1 retrying==1.3.3 six==1.15.0 -Werkzeug==1.0.1 From 39e63b05618d00cca067a4d668570121607968ef Mon Sep 17 00:00:00 2001 From: Yi Heng Joshua Wu Date: Tue, 14 Jun 2022 18:50:08 -0700 Subject: [PATCH 02/53] Resolve interaction issue with flask --- server/dash/app.py | 40 +-- server/dash/dashboards/ncSumComp.py | 309 ++++++++++++++++++ server/dash/dashboards/neighborhood.py | 10 +- server/dash/dashboards/neighborhood_recent.py | 9 +- server/dash/dashboards/overview.py | 6 +- server/dash/dashboards/recent.py | 6 +- server/dash/dashboards/types_map.py | 9 +- server/dash/requirements.txt | 10 + 8 files changed, 338 insertions(+), 61 deletions(-) create mode 100644 server/dash/dashboards/ncSumComp.py diff --git a/server/dash/app.py b/server/dash/app.py index 98cc3a88f..82695e137 100644 --- a/server/dash/app.py +++ b/server/dash/app.py @@ -1,44 +1,12 @@ -import json -import urllib - +import flask from dash import Dash, html, dcc -import pandas as pd - external_stylesheets = ['/static/reports.css'] -app = Dash(__name__, external_stylesheets=external_stylesheets) +server = flask.Flask(__name__) # define flask app.server +app = Dash(__name__, external_stylesheets=external_stylesheets, suppress_callback_exceptions=True, server=server) # set up default layout app.layout = html.Div([ dcc.Location(id='url', refresh=False), html.Div(id='page-content') -]) - -server = app.server -app.config.suppress_callback_exceptions = True - -BATCH_SIZE = 10000 - - -def batch_get_data(url): - # set up your query - if '?' in url: - batch_url = f"{url}&limit={BATCH_SIZE}" - else: - batch_url = f"{url}?limit={BATCH_SIZE}" - - response_size = BATCH_SIZE - result_list = [] - skip = 0 - - # loop through query results and add to a list (better performance!) - while (response_size == BATCH_SIZE): - batch = json.loads(urllib.request.urlopen(f"{batch_url}&skip={skip}").read()) - result_list.extend(batch) - response_size = len(batch) - skip = skip + BATCH_SIZE - - # convert JSON object list to dataframe - df = pd.DataFrame.from_records(result_list) - - return df +]) \ No newline at end of file diff --git a/server/dash/dashboards/ncSumComp.py b/server/dash/dashboards/ncSumComp.py new file mode 100644 index 000000000..b032de097 --- /dev/null +++ b/server/dash/dashboards/ncSumComp.py @@ -0,0 +1,309 @@ +import datetime +import dash_daq as daq +import numpy as np +import pandas as pd +import plotly.express as px +import plotly.graph_objects as go +import requests as re + +from config import API_HOST +from design import apply_figure_style +from dash import dcc, html, callback +from dash.dependencies import Input, Output +from dash.exceptions import PreventUpdate + +# Setting 1 week worth of data +start_date = datetime.date.today() - datetime.timedelta(days=200) +end_date = datetime.date.today() - datetime.timedelta(days=100) + +df_path = f"/requests/updated?start_date={start_date}&end_date={end_date}" +results = re.get(API_HOST + df_path) +data_json = results.json() +data_2020 = pd.json_normalize(data_json) + + +layout = html.Div([ + + html.Div(children=[ + + ## Neighborhood Council Dashboard + html.Div(children=[ + html.H2("LA 311 Requests - Neighborhood Council Summary Dashboard", style={'vertical-align':'middle'} ), + html.Div([daq.ToggleSwitch(id='dataQualitySwitch', value=True, style={'height':'vh'}, size = 35), + html.Div(id='dataQualityOutput')], style={'font-family':'Open Sans'}) + ], style={'display':'flex', "justify-content": "space-between", 'vertical-align':'middle' , 'height':'5vh', 'width':'97.5vw'}), + + ## Summary Dropdown + html.Div(children=[ + html.Div(dcc.Dropdown(sorted([n for n in set(data_2020['councilName'])]), ' ', id='nc_dropdown', placeholder="Select a Neighborhood Council..."), style={'display': 'inline-block', 'width':'48.5vw'}), + html.Div(dcc.Dropdown(id='nc_dropdown_filter', multi=True, placeholder="Select a Request Type..."), style={'display': 'inline-block', 'width':'48.5vw'}) + ], style={'display':'flex' , "justify-content": "space-between", "width": "97.5vw", "height": "10vh"}), + + html.Div(html.Br(), style={"height":"0.5vh"}), + + # Line Chart for Number of Request throughout the day + html.Div(dcc.Graph(id='numReqLineChart', style={"height":"40vh", 'width':'97.4vw'}), style={"border":"0.5px black solid", "height":"40vh", 'width':'97.4vw'}), + + html.Div(html.Br(), style={"height":"1vh"}), + + html.Div(children=[ + html.Div( + # Pie Chart for the share of request type + dcc.Graph( + id="reqTypePieChart", style={"height":"40vh", 'width':'48.5vw'} + ), style={'display': 'inline-block', 'width':'48.5vw', "border":"0.5px black solid", "height":"40vh"}), # for border-radius , add stuff later + html.Div( + # Histogram for the request timeToClose + dcc.Graph( + id="timeCloseHist", style={"height":"40vh", 'width':'48vw'} + ), style={'display': 'inline-block', 'width':'48vw', "border":"0.5px black solid", "height":"40vh"}) + ], style={'display':'flex', "justify-content": "space-between", "width":'97.5vw'}) + + ]), + + html.Div(html.Br(), style={"height":"vh"}), + + # Neighborhood Council Summarization Dashboard + html.Div(children=[ + html.H2("LA 311 Requests - Neighborhood Council Comparison Dashboard") + ], style={'textAlign':'center', 'height':'5vh'}), + + ## Comparison Dropdowns + html.Div(children=[ + html.Div(dcc.Dropdown(sorted([n for n in set(data_2020['councilName'])]), ' ', id='nc_comp_dropdown', placeholder="Select a Neighborhood Council..."), style={'display': 'inline-block', 'width':'48.5vw'}), + html.Div(dcc.Dropdown(sorted([n for n in set(data_2020['councilName'])]), ' ', id='nc_comp_dropdown2', placeholder="Select a Neighborhood Council..."), style={'display': 'inline-block', 'width':'48.5vw'}), + ], style={'display':'flex' , "justify-content": "space-between", "width": "97.5vw", "height": "12vh"}), + + + html.Div(html.Br(), style={"height":"1vh"}), + + ## NC Comparison - Indicator Visuals + html.Div(children = [ + html.Div(children = [ + + + # Indicator Visuals for Total number of requests and the number of days the data spans across + html.Div([ + html.H6("Total Number of Requests", style={"text-align":'center'}), + html.H1(id='totalReqCard', style={"text-align":'center'})], + style={'display': 'inline-block', 'width':'24vw', 'height':'16vh', "border":"0.5px black solid"}), + html.Div([ + html.H6("Number of Days", style={"text-align":'center'}), + html.H1(id='numDaysCard', style={"text-align":'center'})], + style={'display': 'inline-block', 'width':'24vw', 'height':'16vh', "border":"0.5px black solid"}) + + + ], style={'display':'flex' , "justify-content": "space-between", 'width':'48.5vw'}), + + + # Indicator Visuals for Total number of requests and the number of days the data spans across + html.Div(children=[ + html.Div([ + html.H6("Total Number of Requests", style={"text-align":'center'}), + html.H1(id='totalReqCard2', style={"text-align":'center'})], + style={'display': 'inline-block', 'width':'24vw', 'height':'16vh', "border":"0.5px black solid"}), + html.Div([ + html.H6("Number of Days", style={"text-align":'center'}), + html.H1(id='numDaysCard2', style={"text-align":'center'})], + style={'display': 'inline-block', 'width':'24vw', 'height':'16vh', "border":"0.5px black solid"}) + + + ], style={'display':'flex' , "justify-content": "space-between", 'width':'48.5vw'}) + ] , style={'display':'flex' , "justify-content": "space-between", "width": "97.5vw"}), + + + html.Div(html.Br(), style={"height":"1vh"}), + + ## NC Comparison - Request Source Bar Charts + html.Div(children = [ + html.Div(dcc.Graph(id='reqSourceBarChart', style={"height":"30vh"}), style={'display': 'inline-block', 'width':'48.5vw', "border":"0.5px black solid", "height":"30vh"}), + html.Div(dcc.Graph(id='reqSourceBarChart2', style={"height":"30vh"}), style={'display': 'inline-block', 'width':'48.5vw', "border":"0.5px black solid", "margin-left": "10px", "height":"30vh"}) + ] , style={'display':'flex' , "justify-content": "space-between", "width": "97.5vw"}), + + html.Div(html.Br(), style={"height":"1vh"}), + + ## NC Comparison - Number of Requests per day Overlapping line chart + html.Div(dcc.Graph(id='overlayReqTimeLineChart', style={"height":"32vh", "width":"97.5vw"}), style={"border":"0.5px black solid", "height":"32vh", "width":"97.5vw"}) +]) + + + +# Callback Function to generate Dynamic Filter Selection - +# Removing request types that doesn't exist in NC +@callback( + [Output('nc_dropdown_filter', 'options'), + Output('nc_dropdown_filter', 'value')], + Input('nc_dropdown', 'value') + +) +def generate_dynamic_filter(nc_dropdown): + if not nc_dropdown: + df = data_2020 + else: + df = data_2020[data_2020['councilName'] == nc_dropdown] + + rTypes = sorted([n for n in set(df['typeName'])]) + return rTypes, ' ' + + +## Generate the charts and filters for NC Summary Dashboard, including: +## Share of Request Type Pie Chart, Distribution of request timeToClose, +## Number of Request throughout the day line chart, +@callback( + Output('reqTypePieChart', 'figure'), + Output('timeCloseHist', 'figure'), + Output('numReqLineChart', 'figure'), + Output('dataQualityOutput', 'children'), + Input('nc_dropdown', 'value'), + [Input('nc_dropdown_filter', 'value')], + Input('dataQualitySwitch', 'value') +) +def generate_nc_summary_charts(nc_dropdown, nc_dropdown_filter=None, dataQualitySwitch=True): + # NC Dropdown + if not nc_dropdown: + df = data_2020 + else: + df = data_2020[data_2020['councilName'] == nc_dropdown] + # Filter as per selection on Reqquest Type Dropdown + if not nc_dropdown_filter and not nc_dropdown: + df = data_2020 + elif nc_dropdown_filter: + df = df[df['typeName'].isin(nc_dropdown_filter)] + + # Pie Chart for the distribution of Request Types + rtype = pd.DataFrame(df['typeName'].value_counts()) + rtype = rtype.reset_index() + reqTypePieChart = px.pie(rtype, values="typeName", names="index", title="Share of each Request Type") + reqTypePieChart.update_layout(margin=dict(l=50, r=50, b=50, t=50), legend_title = dict(font = dict(size = 10)), font=dict(size=9)) + + # Distribution of Time to Close Date of each request + ## Calculate the Time to Closed + df.loc[:, 'createDateDT'] = pd.to_datetime(df.loc[:, 'createdDate'].str[:-4].str.split("T").str.join(" ")) + df.loc[:, 'closeDateDT'] = pd.to_datetime(df.loc[:, 'closedDate'].str[:-4].str.split("T").str.join(" ")) + df.loc[:, 'timeToClose'] = (df.loc[:, 'closeDateDT'] - df.loc[:, 'createDateDT']).dt.days + + ## Calculate the Optimal number of bins based on Freedman-Diaconis Rule + + # Replace empty rows with 0.0000001 To avoid log(0) error later + df.loc[:, "timeToClose"] = df.loc[:, "timeToClose"].fillna(0.0000001) + + # Replace negative values + # TODO: figure out what to do when there is no data avaialble + df = df[df['timeToClose'] > 0] + if df.shape[0] == 0: + raise PreventUpdate() + else: + q3, q1 = np.percentile(df.loc[:, "timeToClose"].astype(int), [75 ,25]) + iqr = q3 - q1 + if not iqr: + numBins = 100 + else: + numBins = int((2*iqr)/(df.shape[0]**(1/3))) + + # Log Transform, Compute IQR, then exclude outliers + df.loc[:, "logTimeToClose"] = np.log(df.loc[:, "timeToClose"]) + log_q3, log_q1 = np.percentile(df.loc[:, "logTimeToClose"], [75 ,25]) + log_iqr = log_q3 - log_q1 + + # Data Quality switch to remove outliers as defined by Median +- 1.5*IQR + if dataQualitySwitch: + # TODO: figure out what happens when the filtering mechanism output no data at all + temp = df[(df.loc[:, "logTimeToClose"] > 1.5*log_iqr - np.median(df.loc[:, "logTimeToClose"])) & (df.loc[:, "logTimeToClose"] < 1.5*log_iqr + np.median(df.loc[:, "logTimeToClose"]))] + if temp.shape[0] > 0: + df = temp + dataQualityOutput = "Quality Filter: On" + else: + dataQualityOutput = "Quality Filter: Off" + + # Distribution for the total number of requests + timeCloseHist = px.histogram(df, x="timeToClose", title="Distribution of Time to Close Request", nbins= numBins, range_x=[min(df.loc[:, 'timeToClose']), max(df.loc[:, 'timeToClose'])], labels={"timeToClose":"Request Duration", "count":"Frequency"}) + timeCloseHist.update_layout(margin=dict(l=50, r=50, b=50, t=50), font=dict(size=9)) + + ## Time Series for the Total Number of Requests + + # Implementation with Datetime + rtime = pd.DataFrame(df.groupby('createDateDT', as_index=False)['srnumber'].count()) + numReqLineChart = px.line(rtime, x="createDateDT", y = 'srnumber', title="Total Number of 311 Requests Overtime", labels={"createDateDT":"DateTime", "srnumber":"Frequency"} ) + numReqLineChart.update_layout(margin=dict(l=25, r=25, b=25, t=50), font=dict(size=9)) + + + ## Potentially applying stylistic changes according to previous version + apply_figure_style(reqTypePieChart) + apply_figure_style(timeCloseHist) + apply_figure_style(numReqLineChart) + + return reqTypePieChart, timeCloseHist, numReqLineChart, dataQualityOutput + +## Generate the charts and filters for NC Comparison Dashboards, including the following +@callback( + Output('reqSourceBarChart', 'figure'), + Output('reqSourceBarChart2', 'figure'), + Output('totalReqCard', 'children'), + Output('totalReqCard2', 'children'), + Output('numDaysCard', 'children'), + Output('numDaysCard2', 'children'), + Output('overlayReqTimeLineChart', 'figure'), + Input('nc_comp_dropdown', 'value'), + Input('nc_comp_dropdown2', 'value'), + prevent_initial_call=True +) +def generate_nc_comparison_charts(nc_comp_dropdown, nc_comp_dropdown2): + if not nc_comp_dropdown: + df_nc1 = data_2020 + else: + df_nc1 = data_2020[data_2020['councilName'] == nc_comp_dropdown] + if not nc_comp_dropdown2: + df_nc2 = data_2020 + else: + df_nc2 = data_2020[data_2020['councilName'] == nc_comp_dropdown2] + + + df_nc1.loc[:, 'createDateDT'] = pd.to_datetime(df_nc1.loc[:, 'createdDate'].str[:-4].str.split("T").str.join(" ")) + df_nc2.loc[:, 'createDateDT'] = pd.to_datetime(df_nc2.loc[:, 'createdDate'].str[:-4].str.split("T").str.join(" ")) + + + + # Bar chart of different Request Type Sources + rSource = pd.DataFrame(df_nc1['sourceName'].value_counts()) + rSource = rSource.reset_index() + reqSourceBarChart = px.bar(rSource, x="sourceName", y="index", orientation='h', title='Number of Requests by Source', labels={"index":"Request Source", "sourceName":"Frequency"}) + reqSourceBarChart.update_layout(margin=dict(l=25, r=25, b=25, t=50), font=dict(size=9)) + + rSource2 = pd.DataFrame(df_nc2['sourceName'].value_counts()) + rSource2 = rSource2.reset_index() + reqSourceBarChart2 = px.bar(rSource2, x="sourceName", y="index", orientation='h', title='Number of Requests by Source', labels={"index":"Request Source", "sourceName":"Frequency"}) + reqSourceBarChart2.update_layout(margin=dict(l=25, r=25, b=25, t=50) , font=dict(size=9)) + + # Indicator Variables for Total Requests 1 + totalReqCard = df_nc1.shape[0] + + # Indicator Variables for Total Requests 2 + totalReqCard2 = df_nc2.shape[0] + + # Indicator Visuals for the Number of Days the dataset spans 1 + + numDaysCard = np.max(df_nc1['createDateDT'].dt.day) - np.min(df_nc1['createDateDT'].dt.day) + 1 + + # Indicator Visuals for the Number of Days the dataset spans 2 + numDaysCard2 = np.max(df_nc2['createDateDT'].dt.day) - np.min(df_nc2['createDateDT'].dt.day) + 1 + + + # Overlapping line chart for number of request throughout the day + rTime = pd.DataFrame(df_nc1.groupby('createDateDT', as_index=False)['srnumber'].count()) + rTime2 = pd.DataFrame(df_nc2.groupby('createDateDT', as_index=False)['srnumber'].count()) + overlayReqTimeLineChart = go.Figure() + overlayReqTimeLineChart.add_trace(go.Scatter(x=rTime['createDateDT'], y=rTime['srnumber'], mode='lines', name='NC1')) + overlayReqTimeLineChart.add_trace(go.Scatter(x=rTime2['createDateDT'], y=rTime2['srnumber'], mode='lines', name='NC2')) + + overlayReqTimeLineChart.update_layout(title='Number of Request Throughout the Day', margin=dict(l=25, r=25, b=35, t=50), xaxis_range=[min(min(rTime['createDateDT']), min(rTime2['createDateDT'])), max(max(rTime['createDateDT']), max(rTime2['createDateDT']))], font=dict(size=9)) + + apply_figure_style(reqSourceBarChart) + apply_figure_style(reqSourceBarChart2) + apply_figure_style(overlayReqTimeLineChart) + + + + return reqSourceBarChart, reqSourceBarChart2, totalReqCard, totalReqCard2, numDaysCard, numDaysCard2, overlayReqTimeLineChart + + + diff --git a/server/dash/dashboards/neighborhood.py b/server/dash/dashboards/neighborhood.py index 4a5215f96..e0a656eca 100644 --- a/server/dash/dashboards/neighborhood.py +++ b/server/dash/dashboards/neighborhood.py @@ -1,13 +1,11 @@ -import textwrap - -from dash import Dash, dcc, html, callback import pandas as pd import plotly.express as px -from dash.dependencies import Input, Output +import textwrap -# from app import app +from dash import dcc, html, callback +from dash.dependencies import Input, Output from config import API_HOST -from design import CONFIG_OPTIONS, DISCRETE_COLORS, DISCRETE_COLORS_MAP, LABELS, apply_figure_style +from design import CONFIG_OPTIONS, DISCRETE_COLORS, DISCRETE_COLORS_MAP, LABELS, apply_figure_style # TITLE title = "NEIGHBORHOODS" diff --git a/server/dash/dashboards/neighborhood_recent.py b/server/dash/dashboards/neighborhood_recent.py index 0e59dab5c..ed4ec49e6 100644 --- a/server/dash/dashboards/neighborhood_recent.py +++ b/server/dash/dashboards/neighborhood_recent.py @@ -1,13 +1,12 @@ import datetime -import textwrap import json -import urllib - -from dash import dcc, html, dash_table, callback import pandas as pd import plotly.express as px -#from app import app, batch_get_data +import textwrap +import urllib + from config import API_HOST +from dash import dash_table, dcc, html ,callback from dash.dependencies import Input, Output from design import CONFIG_OPTIONS, DISCRETE_COLORS, DISCRETE_COLORS_MAP, LABELS, apply_figure_style from flask import request diff --git a/server/dash/dashboards/overview.py b/server/dash/dashboards/overview.py index f4d9427fb..6ac1f4687 100644 --- a/server/dash/dashboards/overview.py +++ b/server/dash/dashboards/overview.py @@ -1,14 +1,12 @@ -import textwrap - -from dash import dcc, html import pandas as pd import plotly.express as px import plotly.graph_objects as go +import textwrap +from dash import dcc, html from config import API_HOST from design import CONFIG_OPTIONS, DISCRETE_COLORS, DISCRETE_COLORS_MAP, LABELS, apply_figure_style - # TITLE title = "311 DATA OVERVIEW" diff --git a/server/dash/dashboards/recent.py b/server/dash/dashboards/recent.py index bfcc90eaa..5b69bd759 100644 --- a/server/dash/dashboards/recent.py +++ b/server/dash/dashboards/recent.py @@ -1,11 +1,9 @@ import datetime -import textwrap - -from dash import dcc, html import pandas as pd import plotly.express as px -import plotly.graph_objects as go +import textwrap +from dash import dcc, html from config import API_HOST from design import CONFIG_OPTIONS, DISCRETE_COLORS, DISCRETE_COLORS_MAP, LABELS, apply_figure_style diff --git a/server/dash/dashboards/types_map.py b/server/dash/dashboards/types_map.py index 6c0ac6c29..408382831 100644 --- a/server/dash/dashboards/types_map.py +++ b/server/dash/dashboards/types_map.py @@ -1,16 +1,13 @@ import datetime +import pandas as pd +import plotly.express as px import json -from urllib.request import urlopen from dash import dcc, html, callback -import pandas as pd -import plotly.express as px from dash.dependencies import Input, Output - -# from app import app from config import API_HOST from design import CONFIG_OPTIONS, LABELS, apply_figure_style - +from urllib.request import urlopen # TITLE diff --git a/server/dash/requirements.txt b/server/dash/requirements.txt index afc64990a..4c00e8f6e 100644 --- a/server/dash/requirements.txt +++ b/server/dash/requirements.txt @@ -26,3 +26,13 @@ python-dateutil==2.8.1 pytz==2021.1 retrying==1.3.3 six==1.15.0 + +# Other dependencies related to dash +requests==2.27.1 +urllib3==1.26.9 +idna==3.3 +certifi==2021.10.8 +charset-normalizer==2.0.12 +dash-daq==0.5.0 +dash-bootstrap-components==1.1.0 +flask \ No newline at end of file From e3d4b5975774c6b7cb758f53b28d9d60020d4510 Mon Sep 17 00:00:00 2001 From: joshuayhwu <77765986+joshuayhwu@users.noreply.github.com> Date: Tue, 21 Jun 2022 16:28:45 -0700 Subject: [PATCH 03/53] Remove flask comment --- server/dash/app.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/dash/app.py b/server/dash/app.py index 82695e137..95c2e2a09 100644 --- a/server/dash/app.py +++ b/server/dash/app.py @@ -2,11 +2,11 @@ from dash import Dash, html, dcc external_stylesheets = ['/static/reports.css'] -server = flask.Flask(__name__) # define flask app.server +server = flask.Flask(__name__) app = Dash(__name__, external_stylesheets=external_stylesheets, suppress_callback_exceptions=True, server=server) # set up default layout app.layout = html.Div([ dcc.Location(id='url', refresh=False), html.Div(id='page-content') -]) \ No newline at end of file +]) From 4d51ab1e41793053b94b0334ce83591210e61c27 Mon Sep 17 00:00:00 2001 From: joshuayhwu <77765986+joshuayhwu@users.noreply.github.com> Date: Tue, 21 Jun 2022 16:44:05 -0700 Subject: [PATCH 04/53] Update batch_get_data function docstring --- server/dash/dashboards/neighborhood_recent.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/server/dash/dashboards/neighborhood_recent.py b/server/dash/dashboards/neighborhood_recent.py index ed4ec49e6..36e627047 100644 --- a/server/dash/dashboards/neighborhood_recent.py +++ b/server/dash/dashboards/neighborhood_recent.py @@ -34,7 +34,21 @@ BATCH_SIZE = 10000 def batch_get_data(url): - # set up your query + '''Requesting data from specified API endpoint by loading data in chunks. + + The function concatenates the BATCH_SIZE argument to the specified API endpoint (url), + loads the data in individual chunks, and returns a pandas datafrane. In other words, + the function will first store row 1 to row BATCH_SIZE in the first iteration, then + loads row BATCH_SIZE+1 to 2*BATCH_SIZE and so on until all data loaded. + By default, BATCH_SIZE is 10000. + + Typical Usage Example: + + API_HOST = "https://dev-api.311-data.org" + df_path = f"/requests/updated?start_date={'2022-05-21'}&end_date={'2022-06-21'}" + request_df = batch_get_data(API_HOST + df_path) + + ''' if '?' in url: batch_url = f"{url}&limit={BATCH_SIZE}" else: From 8d35608ea3ae8e39a478e680b7dec90a98217ba6 Mon Sep 17 00:00:00 2001 From: joshuayhwu <77765986+joshuayhwu@users.noreply.github.com> Date: Tue, 21 Jun 2022 17:02:06 -0700 Subject: [PATCH 05/53] Update import alphabetical order --- server/dash/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/dash/app.py b/server/dash/app.py index 95c2e2a09..c1b02a80d 100644 --- a/server/dash/app.py +++ b/server/dash/app.py @@ -1,5 +1,5 @@ import flask -from dash import Dash, html, dcc +from dash import Dash, dcc, html external_stylesheets = ['/static/reports.css'] server = flask.Flask(__name__) From f5a34a542312bd093dc64bc483e081d7d8531917 Mon Sep 17 00:00:00 2001 From: joshuayhwu <77765986+joshuayhwu@users.noreply.github.com> Date: Tue, 21 Jun 2022 18:53:56 -0700 Subject: [PATCH 06/53] Update import format on neighborhood.py --- server/dash/dashboards/neighborhood.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/server/dash/dashboards/neighborhood.py b/server/dash/dashboards/neighborhood.py index e0a656eca..1a1233cad 100644 --- a/server/dash/dashboards/neighborhood.py +++ b/server/dash/dashboards/neighborhood.py @@ -1,9 +1,10 @@ -import pandas as pd -import plotly.express as px import textwrap -from dash import dcc, html, callback +import pandas as pd +import plotly.express as px +from dash import callback, dcc, html from dash.dependencies import Input, Output + from config import API_HOST from design import CONFIG_OPTIONS, DISCRETE_COLORS, DISCRETE_COLORS_MAP, LABELS, apply_figure_style From 7306652b81086e82ad035f1725fb25d25e720119 Mon Sep 17 00:00:00 2001 From: joshuayhwu <77765986+joshuayhwu@users.noreply.github.com> Date: Tue, 21 Jun 2022 18:57:08 -0700 Subject: [PATCH 07/53] Removed obsolete import statement --- server/dash/index.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/server/dash/index.py b/server/dash/index.py index 4395b4728..006040726 100644 --- a/server/dash/index.py +++ b/server/dash/index.py @@ -5,8 +5,7 @@ import re import signal -from dash import Dash, html, dcc, callback, Input, Output -# from dash.dependencies import Input, Output +from dash import callback, Dash, dcc, html, Input, Output from app import app from config import DASH_FILES From 0b7cd2aa4539f183efd94d7b10711ce76580175b Mon Sep 17 00:00:00 2001 From: Yi Heng Joshua Wu Date: Thu, 23 Jun 2022 17:01:56 -0700 Subject: [PATCH 08/53] Neighborhood Combine draft --- server/dash/dashboards/neighborComb.py | 54 ++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 server/dash/dashboards/neighborComb.py diff --git a/server/dash/dashboards/neighborComb.py b/server/dash/dashboards/neighborComb.py new file mode 100644 index 000000000..ebf771353 --- /dev/null +++ b/server/dash/dashboards/neighborComb.py @@ -0,0 +1,54 @@ +# import pandas as pd +# import plotly.express as px +# from dash import callback, dcc, html +# from dash.dependencies import Input, Output + +# from config import API_HOST +# from design import CONFIG_OPTIONS, DISCRETE_COLORS, DISCRETE_COLORS_MAP, LABELS, apply_figure_style + + +# title = "NEIGHBORHOODS" + +# # DATA +# print(" * Downloading data for dataframe") +# query_string = '/reports?field=type_name&field=council_name&field=created_date' +# df = pd.read_json(API_HOST + query_string) +# print(" * Dataframe has been loaded") + +# fig = px.line() +# apply_figure_style(fig) + + +# def populate_options(): +# values = [] +# for i in df.sort_values('council_name').council_name.unique(): +# values.append( +# { +# 'label': i, +# 'value': i +# } +# ) +# return values + + +# # LAYOUT +# layout = html.Div([ +# html.H1(title), +# dcc.Dropdown( +# id='council_list', +# clearable=False, +# value="Arleta", +# placeholder="Select a neighborhood", +# options=populate_options() +# ), +# dcc.Graph( +# id='ncAvgCompLineChart', +# figure=fig, +# config=CONFIG_OPTIONS +# ), +# dcc.Graph( +# id='ncNumReqTypeLineChart', +# figure=fig, +# config=CONFIG_OPTIONS +# ) +# ]) From 5aa43e45d5a542937b0c79089c191d76c60629f2 Mon Sep 17 00:00:00 2001 From: Yi Heng Joshua Wu Date: Mon, 27 Jun 2022 20:38:30 -0700 Subject: [PATCH 09/53] Overview first page draft --- server/dash/dashboards/neighborComb.py | 54 -------- server/dash/dashboards/overviewComb.py | 166 +++++++++++++++++++++++++ 2 files changed, 166 insertions(+), 54 deletions(-) delete mode 100644 server/dash/dashboards/neighborComb.py create mode 100644 server/dash/dashboards/overviewComb.py diff --git a/server/dash/dashboards/neighborComb.py b/server/dash/dashboards/neighborComb.py deleted file mode 100644 index ebf771353..000000000 --- a/server/dash/dashboards/neighborComb.py +++ /dev/null @@ -1,54 +0,0 @@ -# import pandas as pd -# import plotly.express as px -# from dash import callback, dcc, html -# from dash.dependencies import Input, Output - -# from config import API_HOST -# from design import CONFIG_OPTIONS, DISCRETE_COLORS, DISCRETE_COLORS_MAP, LABELS, apply_figure_style - - -# title = "NEIGHBORHOODS" - -# # DATA -# print(" * Downloading data for dataframe") -# query_string = '/reports?field=type_name&field=council_name&field=created_date' -# df = pd.read_json(API_HOST + query_string) -# print(" * Dataframe has been loaded") - -# fig = px.line() -# apply_figure_style(fig) - - -# def populate_options(): -# values = [] -# for i in df.sort_values('council_name').council_name.unique(): -# values.append( -# { -# 'label': i, -# 'value': i -# } -# ) -# return values - - -# # LAYOUT -# layout = html.Div([ -# html.H1(title), -# dcc.Dropdown( -# id='council_list', -# clearable=False, -# value="Arleta", -# placeholder="Select a neighborhood", -# options=populate_options() -# ), -# dcc.Graph( -# id='ncAvgCompLineChart', -# figure=fig, -# config=CONFIG_OPTIONS -# ), -# dcc.Graph( -# id='ncNumReqTypeLineChart', -# figure=fig, -# config=CONFIG_OPTIONS -# ) -# ]) diff --git a/server/dash/dashboards/overviewComb.py b/server/dash/dashboards/overviewComb.py new file mode 100644 index 000000000..631aef7d9 --- /dev/null +++ b/server/dash/dashboards/overviewComb.py @@ -0,0 +1,166 @@ +import textwrap +import datetime +import pandas as pd +import plotly.express as px +import plotly.graph_objects as go +from dash import callback, dcc, html +from dash.dependencies import Input, Output + +from config import API_HOST +from design import CONFIG_OPTIONS, DISCRETE_COLORS, DISCRETE_COLORS_MAP, LABELS, apply_figure_style + +title = "OVERVIEW COMBINED DASHBOARD" + +# DATA +print(" * Downloading data for dataframe") +query_string = '/reports?field=type_name&field=council_name&field=created_date' +df = pd.read_json(API_HOST + query_string) +print(" * Dataframe has been loaded") + +# Loading the dataframe for the NCs and correspoding requests +print(" * Downloading data for dataframe") +query_string = "/reports?field=council_name&filter=created_date>=2016-01-01" +df1 = pd.read_json(API_HOST + query_string) +df1 = df1.groupby(['council_name'])['counts'].sum().sort_values().to_frame() + +# Loading the data for the number of new requests +print(" * Downloading data for dataframe") +query_string = "/reports?field=created_year&filter=created_date>=2016-01-01" +df2 = pd.read_json(API_HOST + query_string) +df2 = df2.groupby(['created_year'])['counts'].sum().to_frame() + +# Loading the count of each request types overall +print(" * Downloading data for dataframe") +query_string = "/reports?field=type_name&filter=created_date>=2016-01-01" +df3 = pd.read_json(API_HOST + query_string) +df3 = df3.groupby(['type_name'], as_index=False)['counts'].sum() + +# Loading the total number of requests +# start_date = datetime.date.today() - datetime.timedelta(days=200) +# df_path = f"/requests/updated?start_date={start_date}" +# df4 = pd.read_json(df_path) +# df4.loc[:, 'createDateDT'] = pd.to_datetime( +# df4.loc[:, 'createdDate'].str[:-4].str.split("T").str.join(" ")) +# df4.loc[:, 'closeDateDT'] = pd.to_datetime( +# df4.loc[:, 'closedDate'].str[:-4].str.split("T").str.join(" ")) +# df4.loc[:, 'timeToClose'] = (df4.loc[:, 'closeDateDT'] - df4.loc[:, 'createDateDT']).dt.days +# df4.loc[:, "timeToClose"] = df4.loc[:, "timeToClose"].fillna(0.0000001) + + +# Request Share by Agency Pie Chart +print(" * Downloading data for dataframe") +query_string = "/reports?field=agency_name&filter=created_date>=2016-01-01" +df5 = pd.read_json(API_HOST + query_string) +df5 = df5.groupby(['agency_name'])['counts'].sum().to_frame() +df5.sort_values('counts', ascending=False, inplace=True) +df5.loc['Others'] = df5[4:].sum() +df5.sort_values('counts', ascending=False, inplace=True) +df5 = df5[:5] +df5.index = df5.index.map(lambda x: '
'.join(textwrap.wrap(x, width=16))) + +shareReqByAgencyPieChart = px.pie( + df5, + names=df5.index, + values='counts', + color_discrete_sequence=DISCRETE_COLORS, + labels=LABELS, + hole=.3, + title="Total Requests by Agency", +) +shareReqByAgencyPieChart.update_layout(margin=dict(l=125, r=125, b=125, t=125)) +apply_figure_style(shareReqByAgencyPieChart) + + +# Request Type by Source Bar Chart +print(" * Downloading data for dataframe") +query_string = "/reports?field=source_name&filter=created_date>=2016-01-01" +df6 = pd.read_json(API_HOST + query_string) +df6 = df6.groupby(['source_name'])['counts'].sum().to_frame() +df6.sort_values('counts', ascending=False, inplace=True) +df6.loc['Others'] = df6[4:].sum() +df6.sort_values('counts', ascending=False, inplace=True) +df6 = df6[:5] +reqSourceBarchart = px.bar( + df6, + y=df6.index, + x='counts', + color_discrete_sequence=['#1D6996'], + labels=LABELS, + title="Total Requests by Source", + orientation='h' +) +apply_figure_style(reqSourceBarchart) + + +# Median Request Days to Close Box Plot +stas_df = pd.read_json(API_HOST + '/types/stats') +stas_df = stas_df.sort_values('median', ascending=False) +medDaysToCloseBoxPlot = go.Figure() +medDaysToCloseBoxPlot.add_trace( + go.Box( + y=stas_df.type_name, + q1=stas_df['q1'], + median=stas_df['median'], + q3=stas_df['q3'], + marker_color='#29404F', + fillcolor='#E17C05', + ) +) +medDaysToCloseBoxPlot.update_xaxes( + + dtick=5 +) +medDaysToCloseBoxPlot.update_layout( + title="Total Median Days to Close by Type", +) +apply_figure_style(medDaysToCloseBoxPlot) + +# LAYOUT +layout = html.Div([ + # Page 2 with Cards + Stuf + html.H1(title), + html.P("The figures below represent the total number of 311 requests made across LA County from 2016-2021. In 2020, we saw an all-time high with more than 1.4 million requests.", + style={ 'font-size': '18px', 'font-style': 'italic'}), + html.Div(children=[ + html.Div([ + html.Div([html.H2(f"{df2['counts'].sum():,}"), html.Label( + "Total Requests")], style={"text-align": 'center', "border": "#666666 1px solid", 'width': '18vw'}), + html.Div([html.H2(df1.shape[0] - 1), html.Label("Neighborhoods")], + style={"text-align": 'center', "border": "#666666 1px solid", 'width': '18vw'}), + html.Div([html.H2(df3.shape[0]), html.Label("Request Types")], style={"text-align": 'center', "border": "#666666 1px solid", 'width': '18vw'})], + style={"width": "57vw", 'display': 'flex', "justify-content": "space-between"}) + + # , + # html.Div([dcc.Graph( + # id='reqSourceBarchart', + # figure=reqSourceBarchart, + # config=CONFIG_OPTIONS, + # className="half-graph", + # responsive=True, + # style={"width":"35vw", "height":"50vh","text-align":'center'} + # )]) + + ], style={'display': 'flex', "justify-content": "space-between"} + # , className="graph-row" + ), + html.Div([ + dcc.Graph( + id='medDaysToCloseBoxPlot', + figure=medDaysToCloseBoxPlot, + config=CONFIG_OPTIONS, + responsive=True, + style={"width": "57vw", "height": "70vh"} + ), + dcc.Graph( + id='shareReqByAgencyPieChart', + figure=shareReqByAgencyPieChart, + config=CONFIG_OPTIONS, + className="half-graph", + responsive=True, + style={"width": "35vw", "height": "70vh"} + ) + + ], className="graph-row") + + +]) From f2f84dfa5b809e7ae7bb9bfeb5d9287e24946ff8 Mon Sep 17 00:00:00 2001 From: Yi Heng Joshua Wu Date: Thu, 30 Jun 2022 17:02:22 -0700 Subject: [PATCH 10/53] Updating combined dashboard --- server/dash/dashboards/ncSumComp.py | 46 +++++++++++++++++++++++++---- 1 file changed, 40 insertions(+), 6 deletions(-) diff --git a/server/dash/dashboards/ncSumComp.py b/server/dash/dashboards/ncSumComp.py index b032de097..74fe2c03c 100644 --- a/server/dash/dashboards/ncSumComp.py +++ b/server/dash/dashboards/ncSumComp.py @@ -7,15 +7,15 @@ import requests as re from config import API_HOST -from design import apply_figure_style +from design import DISCRETE_COLORS, LABELS, apply_figure_style from dash import dcc, html, callback from dash.dependencies import Input, Output from dash.exceptions import PreventUpdate # Setting 1 week worth of data -start_date = datetime.date.today() - datetime.timedelta(days=200) -end_date = datetime.date.today() - datetime.timedelta(days=100) - +start_date = datetime.date.today() - datetime.timedelta(days=300) +end_date = datetime.date.today() - datetime.timedelta(days=200) + df_path = f"/requests/updated?start_date={start_date}&end_date={end_date}" results = re.get(API_HOST + df_path) data_json = results.json() @@ -42,7 +42,7 @@ html.Div(html.Br(), style={"height":"0.5vh"}), # Line Chart for Number of Request throughout the day - html.Div(dcc.Graph(id='numReqLineChart', style={"height":"40vh", 'width':'97.4vw'}), style={"border":"0.5px black solid", "height":"40vh", 'width':'97.4vw'}), + html.Div(dcc.Graph(id='ncAvgCompLineChart', style={"height":"40vh", 'width':'97.4vw'}), style={"border":"0.5px black solid", "height":"40vh", 'width':'97.4vw'}), html.Div(html.Br(), style={"height":"1vh"}), @@ -190,6 +190,7 @@ def generate_nc_summary_charts(nc_dropdown, nc_dropdown_filter=None, dataQuality # Replace negative values # TODO: figure out what to do when there is no data avaialble df = df[df['timeToClose'] > 0] + print(df.shape) if df.shape[0] == 0: raise PreventUpdate() else: @@ -257,7 +258,7 @@ def generate_nc_comparison_charts(nc_comp_dropdown, nc_comp_dropdown2): else: df_nc2 = data_2020[data_2020['councilName'] == nc_comp_dropdown2] - + print(df_nc1.shape) df_nc1.loc[:, 'createDateDT'] = pd.to_datetime(df_nc1.loc[:, 'createdDate'].str[:-4].str.split("T").str.join(" ")) df_nc2.loc[:, 'createDateDT'] = pd.to_datetime(df_nc2.loc[:, 'createdDate'].str[:-4].str.split("T").str.join(" ")) @@ -307,3 +308,36 @@ def generate_nc_comparison_charts(nc_comp_dropdown, nc_comp_dropdown2): +# Define callback to update graph +@callback( + Output('ncAvgCompLineChart', 'figure'), + [Input("nc_dropdown", "value")] +) +def update_figure(nc_dropdown): + if not nc_dropdown: + df = data_2020 + neighborhood_sum_df = df[df.council_name == nc_dropdown].groupby(['created_date']).agg('sum').reset_index() # noqa + total_sum_df = df.groupby(['created_date']).agg('sum').reset_index() + total_sum_df["nc_avg"] = total_sum_df["counts"] / 99 + merged_df = neighborhood_sum_df.merge(total_sum_df["nc_avg"].to_frame(), left_index=True, right_index=True) # noqa + + fig = px.line( + merged_df, + x="created_date", + y=['counts', 'nc_avg'], + color_discrete_sequence=DISCRETE_COLORS, + labels=LABELS, + title= "Number of " + nc_dropdown + " Requests compare with the average of all Neighborhood Councils requests" + ) + + fig.update_xaxes( + tickformat="%a\n%m/%d", + ) + + fig.update_traces( + mode='markers+lines' + ) # add markers to lines + + apply_figure_style(fig) + + return fig \ No newline at end of file From 8118f9e80c84a52aaed4ce0134aef7eaa6652e4e Mon Sep 17 00:00:00 2001 From: Yi Heng Joshua Wu Date: Sun, 3 Jul 2022 13:54:25 -0700 Subject: [PATCH 11/53] Combined overview dashboard --- server/dash/dashboards/overviewComb.py | 134 ++++++++++++++----------- server/dash/static/reports.css | 11 +- 2 files changed, 79 insertions(+), 66 deletions(-) diff --git a/server/dash/dashboards/overviewComb.py b/server/dash/dashboards/overviewComb.py index 631aef7d9..b3fb267c6 100644 --- a/server/dash/dashboards/overviewComb.py +++ b/server/dash/dashboards/overviewComb.py @@ -35,17 +35,16 @@ df3 = pd.read_json(API_HOST + query_string) df3 = df3.groupby(['type_name'], as_index=False)['counts'].sum() -# Loading the total number of requests -# start_date = datetime.date.today() - datetime.timedelta(days=200) -# df_path = f"/requests/updated?start_date={start_date}" -# df4 = pd.read_json(df_path) -# df4.loc[:, 'createDateDT'] = pd.to_datetime( -# df4.loc[:, 'createdDate'].str[:-4].str.split("T").str.join(" ")) -# df4.loc[:, 'closeDateDT'] = pd.to_datetime( -# df4.loc[:, 'closedDate'].str[:-4].str.split("T").str.join(" ")) -# df4.loc[:, 'timeToClose'] = (df4.loc[:, 'closeDateDT'] - df4.loc[:, 'createDateDT']).dt.days -# df4.loc[:, "timeToClose"] = df4.loc[:, "timeToClose"].fillna(0.0000001) +# Loading the total number of request source +print(" * Downloading data for dataframe") +query_string = "/reports?field=source_name&filter=created_date>=2016-01-01" +df6 = pd.read_json(API_HOST + query_string) +reqSourceLab = df6.groupby(['source_name'])['counts'].sum() +# Loading the number of Request Agencies +query_string = "/reports?field=agency_name&filter=created_date>=2016-01-01" +df5 = pd.read_json(API_HOST + query_string) +reqAgency = df5.groupby(['agency_name'])['counts'].sum() # Request Share by Agency Pie Chart print(" * Downloading data for dataframe") @@ -62,14 +61,11 @@ df5, names=df5.index, values='counts', - color_discrete_sequence=DISCRETE_COLORS, labels=LABELS, hole=.3, title="Total Requests by Agency", ) -shareReqByAgencyPieChart.update_layout(margin=dict(l=125, r=125, b=125, t=125)) -apply_figure_style(shareReqByAgencyPieChart) - +shareReqByAgencyPieChart.update_layout(margin=dict(l=100, r=100, b=100, t=100)) # Request Type by Source Bar Chart print(" * Downloading data for dataframe") @@ -84,13 +80,9 @@ df6, y=df6.index, x='counts', - color_discrete_sequence=['#1D6996'], - labels=LABELS, title="Total Requests by Source", orientation='h' ) -apply_figure_style(reqSourceBarchart) - # Median Request Days to Close Box Plot stas_df = pd.read_json(API_HOST + '/types/stats') @@ -113,54 +105,74 @@ medDaysToCloseBoxPlot.update_layout( title="Total Median Days to Close by Type", ) -apply_figure_style(medDaysToCloseBoxPlot) + +# Day of Week Bar Chart: +start_date = datetime.date.today() - datetime.timedelta(days=30) +end_date = datetime.date.today() - datetime.timedelta(days=1) +query_string = f"/reports?filter=created_date>={start_date}&filter=created_date<={end_date}" # noqa +print(" * Downloading data for dataframe") +df = pd.read_json(API_HOST + query_string) +df['created_date'] = pd.to_datetime(df['created_date']) +dow_df = df.groupby(['created_date']).agg('sum').reset_index() +dow_df['day_of_week'] = dow_df['created_date'].dt.day_name() +numReqByDayOfWeekBarChart = px.bar( + dow_df, + x="day_of_week", + y="counts", + labels=LABELS, +) +numReqByDayOfWeekBarChart.update_xaxes(categoryorder='array', categoryarray= ["Sunday", "Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday"]) + +# Total Request by NC +print(" * Downloading data for dataframe") +query_string = "/reports?field=council_name&filter=created_date>=2016-01-01" +df1 = pd.read_json(API_HOST + query_string) +df1 = df1.groupby(['council_name'])['counts'].sum().sort_values().to_frame() +reqByNcBarChart = px.bar( + df1, + x=df1.index, + y='counts', + labels=LABELS, + title="Total Requests by Neighborhood Councils", +) +reqByNcBarChart.update_layout(font=dict(size=12)) # LAYOUT layout = html.Div([ # Page 2 with Cards + Stuf - html.H1(title), + html.H1(title + " Pt. 1"), html.P("The figures below represent the total number of 311 requests made across LA County from 2016-2021. In 2020, we saw an all-time high with more than 1.4 million requests.", - style={ 'font-size': '18px', 'font-style': 'italic'}), - html.Div(children=[ - html.Div([ - html.Div([html.H2(f"{df2['counts'].sum():,}"), html.Label( - "Total Requests")], style={"text-align": 'center', "border": "#666666 1px solid", 'width': '18vw'}), - html.Div([html.H2(df1.shape[0] - 1), html.Label("Neighborhoods")], - style={"text-align": 'center', "border": "#666666 1px solid", 'width': '18vw'}), - html.Div([html.H2(df3.shape[0]), html.Label("Request Types")], style={"text-align": 'center', "border": "#666666 1px solid", 'width': '18vw'})], - style={"width": "57vw", 'display': 'flex', "justify-content": "space-between"}) - - # , - # html.Div([dcc.Graph( - # id='reqSourceBarchart', - # figure=reqSourceBarchart, - # config=CONFIG_OPTIONS, - # className="half-graph", - # responsive=True, - # style={"width":"35vw", "height":"50vh","text-align":'center'} - # )]) - - ], style={'display': 'flex', "justify-content": "space-between"} - # , className="graph-row" - ), - html.Div([ - dcc.Graph( - id='medDaysToCloseBoxPlot', - figure=medDaysToCloseBoxPlot, - config=CONFIG_OPTIONS, - responsive=True, - style={"width": "57vw", "height": "70vh"} - ), - dcc.Graph( - id='shareReqByAgencyPieChart', - figure=shareReqByAgencyPieChart, - config=CONFIG_OPTIONS, - className="half-graph", - responsive=True, - style={"width": "35vw", "height": "70vh"} - ) - - ], className="graph-row") + style={'font-size': '18px', 'font-style': 'italic'}), + html.Div([ + html.Div([html.H2(f"{df2['counts'].sum():,}"), html.Label( + "Total Requests")], style={"text-align": 'center', "border": "0.5px black solid", 'width': '18vw', 'display': 'inline-block'}), + html.Div([html.H2(df1.shape[0] - 1), html.Label("Neighborhoods")], + style={"text-align": 'center', "border": "0.5px black solid", 'width': '18vw', 'display': 'inline-block'}), + html.Div([html.H2(df3.shape[0]), html.Label("Request Types")], style={ + "text-align": 'center', "border": "0.5px black solid", 'width': '18vw', 'display': 'inline-block'}), + html.Div([html.H2(reqSourceLab.shape[0]), html.Label("Request Source")], style={ + "text-align": 'center', "border": "0.5px black solid", 'width': '18vw', 'display': 'inline-block'}), + html.Div([html.H2(reqAgency.shape[0]), html.Label("Request Agency")], style={ + "text-align": 'center', "border": "0.5px black solid", 'width': '18vw', 'display': 'inline-block'}) + ], style={'display': 'flex', "justify-content": "space-between"}), + + html.Div(html.Br(), style={"height": "3vh"}), + html.Div([ + html.Div(dcc.Graph(id='medDaysToCloseBoxPlot', figure=medDaysToCloseBoxPlot, responsive=True, style={ + "width": "60vw", "height": "60vh"}), style={"border": "0.5px black solid"}), + html.Div(dcc.Graph(id='shareReqByAgencyPieChart', figure=shareReqByAgencyPieChart, className="half-graph", + responsive=True, style={"width": "35vw", "height": "60vh"}), style={"border": "0.5px black solid"}) + ], className="graph-row", style={'display': 'flex', "justify-content": "space-between"}), + html.Div(html.Br(), style={"height": "2vh"}), + html.H1(title + " Pt. 2"), + html.Div([ + html.Div(dcc.Graph(id='numReqByDayOfWeekBarChart', figure=numReqByDayOfWeekBarChart, className="half-graph", style={"width": "48vw", "height": "40vh"}), style={"border": "0.5px black solid"}), # noqa + html.Div(dcc.Graph(id='reqSourceBarchart', figure=reqSourceBarchart, className="half-graph", + responsive=True, style={"width": "48vw", "height": "40vh"}), style={"border": "0.5px black solid"}) + ], className="graph-row", style={'display': 'flex', "justify-content": "space-between"}), + html.Div(html.Br(), style={"height": "2vh"}), + html.Div(dcc.Graph(id='reqByNcBarChart', figure=reqByNcBarChart, responsive=True, + style={"height": "45vh"}), style={"border": "0.5px black solid"}) ]) diff --git a/server/dash/static/reports.css b/server/dash/static/reports.css index b041ccd07..cc941a30e 100644 --- a/server/dash/static/reports.css +++ b/server/dash/static/reports.css @@ -1,7 +1,8 @@ body { - color: #ececec; - background-color: #0f181f; - font-family: 'Robota, Gill Sans', 'Gill Sans MT', sans-serif; + /* color: #ececec; */ + /* background-color: #0f181f; */ + /* font-family: 'Robota, Gill Sans', 'Gill Sans MT', sans-serif; */ + font-family: 'Open Sans', sans-serif;; font-size: 1em; font-weight: 300; } @@ -47,8 +48,8 @@ h1 { } .stats-label h2 { font-size: 2em; - margin-bottom: 3pt; - margin-top: 3pt; + margin-bottom: 1pt; + margin-top: 1pt; } .stats-label label { color: #999999; From 86a2803ab6d3bc665217f6e553aed28c650bee10 Mon Sep 17 00:00:00 2001 From: Yi Heng Joshua Wu Date: Tue, 19 Jul 2022 17:29:02 -0700 Subject: [PATCH 12/53] Deleted redundant dashboards --- server/dash/dashboards/neighborhood.py | 127 --------- server/dash/dashboards/neighborhood_recent.py | 269 ------------------ server/dash/dashboards/overview.py | 183 ------------ server/dash/dashboards/recent.py | 87 ------ server/dash/dashboards/types_map.py | 86 ------ 5 files changed, 752 deletions(-) delete mode 100644 server/dash/dashboards/neighborhood.py delete mode 100644 server/dash/dashboards/neighborhood_recent.py delete mode 100644 server/dash/dashboards/overview.py delete mode 100644 server/dash/dashboards/recent.py delete mode 100644 server/dash/dashboards/types_map.py diff --git a/server/dash/dashboards/neighborhood.py b/server/dash/dashboards/neighborhood.py deleted file mode 100644 index 1a1233cad..000000000 --- a/server/dash/dashboards/neighborhood.py +++ /dev/null @@ -1,127 +0,0 @@ -import textwrap - -import pandas as pd -import plotly.express as px -from dash import callback, dcc, html -from dash.dependencies import Input, Output - -from config import API_HOST -from design import CONFIG_OPTIONS, DISCRETE_COLORS, DISCRETE_COLORS_MAP, LABELS, apply_figure_style - -# TITLE -title = "NEIGHBORHOODS" - -# DATA -print(" * Downloading data for dataframe") -query_string = '/reports?field=type_name&field=council_name&field=created_date' -df = pd.read_json(API_HOST + query_string) -print(" * Dataframe has been loaded") - -fig = px.line() -apply_figure_style(fig) - - -def populate_options(): - values = [] - for i in df.sort_values('council_name').council_name.unique(): - values.append( - { - 'label': i, - 'value': i - } - ) - return values - - -layout = html.Div([ - html.H1(title), - dcc.Dropdown( - id='council_list', - clearable=False, - value="Arleta", - placeholder="Select a neighborhood", - options=populate_options() - ), - dcc.Graph( - id='ncAvgCompLineChart', - figure=fig, - config=CONFIG_OPTIONS - ), - dcc.Graph( - id='ncNumReqTypeLineChart', - figure=fig, - config=CONFIG_OPTIONS - ) -]) - - - - - -# Define callback to update graph -@callback( - Output('ncAvgCompLineChart', 'figure'), - [Input("council_list", "value")] -) -def update_figure(selected_council): - neighborhood_sum_df = df[df.council_name == selected_council].groupby(['created_date']).agg('sum').reset_index() # noqa - total_sum_df = df.groupby(['created_date']).agg('sum').reset_index() - total_sum_df["nc_avg"] = total_sum_df["counts"] / 99 - merged_df = neighborhood_sum_df.merge(total_sum_df["nc_avg"].to_frame(), left_index=True, right_index=True) # noqa - - fig = px.line( - merged_df, - x="created_date", - y=['counts', 'nc_avg'], - color_discrete_sequence=DISCRETE_COLORS, - labels=LABELS, - title= "Number of " + selected_council + " Requests compare with the average of all Neighborhood Councils requests" - ) - - fig.update_xaxes( - tickformat="%a\n%m/%d", - ) - - fig.update_traces( - mode='markers+lines' - ) # add markers to lines - - apply_figure_style(fig) - - return fig - - -# Define callback to update graph -@callback( - Output('ncNumReqTypeLineChart', 'figure'), - [Input("council_list", "value")] -) -def update_council_figure(selected_council): - - report_df = df[df.council_name == selected_council].groupby(['created_date', 'type_name']).agg('sum').reset_index() # noqa - report_df.type_name = report_df.type_name.map(lambda x: '
'.join(textwrap.wrap(x, width=16))) # noqa - - fig = px.line( - report_df, - x="created_date", - y="counts", - color="type_name", - color_discrete_sequence=DISCRETE_COLORS, - color_discrete_map=DISCRETE_COLORS_MAP, - labels=LABELS, - title="Number of Different Requests Types for " + selected_council - ) - - fig.update_xaxes( - tickformat="%a\n%m/%d", - ) - - fig.update_traces( - mode='markers+lines' - ) # add markers to lines - - apply_figure_style(fig) - - return fig - - diff --git a/server/dash/dashboards/neighborhood_recent.py b/server/dash/dashboards/neighborhood_recent.py deleted file mode 100644 index 36e627047..000000000 --- a/server/dash/dashboards/neighborhood_recent.py +++ /dev/null @@ -1,269 +0,0 @@ -import datetime -import json -import pandas as pd -import plotly.express as px -import textwrap -import urllib - -from config import API_HOST -from dash import dash_table, dcc, html ,callback -from dash.dependencies import Input, Output -from design import CONFIG_OPTIONS, DISCRETE_COLORS, DISCRETE_COLORS_MAP, LABELS, apply_figure_style -from flask import request - -pretty_columns = { - 'srnumber': "SR Number", - 'createdDate': "Created Date", - 'closedDate': "Closed Date", - 'typeName': "Type Name", - 'agencyName': "Agency Name", - 'sourceName': "Source Name", - 'address': "Address" -} - -start_date = datetime.date.today() - datetime.timedelta(days=7) -end_date = datetime.date.today() - datetime.timedelta(days=1) - -# TITLE -title = "NEIGHBORHOOD WEEKLY REPORT" - -# DATA -df_path = f"/requests/updated?start_date={start_date}&end_date={end_date}" -print(" * Downloading data for dataframe") - -BATCH_SIZE = 10000 - -def batch_get_data(url): - '''Requesting data from specified API endpoint by loading data in chunks. - - The function concatenates the BATCH_SIZE argument to the specified API endpoint (url), - loads the data in individual chunks, and returns a pandas datafrane. In other words, - the function will first store row 1 to row BATCH_SIZE in the first iteration, then - loads row BATCH_SIZE+1 to 2*BATCH_SIZE and so on until all data loaded. - By default, BATCH_SIZE is 10000. - - Typical Usage Example: - - API_HOST = "https://dev-api.311-data.org" - df_path = f"/requests/updated?start_date={'2022-05-21'}&end_date={'2022-06-21'}" - request_df = batch_get_data(API_HOST + df_path) - - ''' - if '?' in url: - batch_url = f"{url}&limit={BATCH_SIZE}" - else: - batch_url = f"{url}?limit={BATCH_SIZE}" - - response_size = BATCH_SIZE - result_list = [] - skip = 0 - - # loop through query results and add to a list (better performance!) - while (response_size == BATCH_SIZE): - batch = json.loads(urllib.request.urlopen(f"{batch_url}&skip={skip}").read()) - result_list.extend(batch) - response_size = len(batch) - skip = skip + BATCH_SIZE - - # convert JSON object list to dataframe - df = pd.DataFrame.from_records(result_list) - - return df - - -df = batch_get_data(API_HOST + df_path) -df['createdDate'] = pd.to_datetime( - df['createdDate'], errors='coerce').dt.strftime('%Y-%m-%d') -df['closedDate'] = pd.to_datetime( - df['closedDate'], errors='coerce').dt.strftime('%Y-%m-%d') -print(" * Dataframe has been loaded") - -try: - selected_council = request.args.get('councilName') or 'Arleta' -except (RuntimeError): - selected_council = 'Arleta' - -table_df = df.query(f"councilName == '{selected_council}'")[['srnumber', 'createdDate', 'closedDate', 'typeName', 'agencyName', 'sourceName', 'address']] # noqa -figure_df = df.query(f"councilName == '{selected_council}' and createdDate >= '{start_date}'").groupby(['createdDate', 'typeName'])['srnumber'].count().reset_index() # noqa - - -# Populate the neighborhood dropdown -def populate_options(): - council_df_path = '/councils' - council_df = pd.read_json(API_HOST + council_df_path) - values = [] - for i in council_df.sort_values('councilName').councilName.unique(): - values.append({ - 'label': i, - 'value': i - }) - return values - - -fig = px.line( - figure_df, - x="createdDate", - y="srnumber", - color="typeName", - color_discrete_sequence=DISCRETE_COLORS, - color_discrete_map=DISCRETE_COLORS_MAP, - labels=LABELS, -) - -fig.update_xaxes( - tickformat="%a\n%m/%d", -) - -fig.update_traces( - mode='markers+lines' -) # add markers to lines - -apply_figure_style(fig) - -pie_fig = px.pie( - figure_df, - names="typeName", - values="srnumber", - color="typeName", - color_discrete_sequence=DISCRETE_COLORS, - color_discrete_map=DISCRETE_COLORS_MAP, - labels=LABELS, - hole=.3, -) -apply_figure_style(pie_fig) - - -# Layout -layout = html.Div([ - html.H1(title), - dcc.Dropdown( - id='council_list', - clearable=False, - value=selected_council, - placeholder="Select a neighborhood", - options=populate_options() - ), - html.Div(f"Weekly report ({start_date.strftime('%b %d')} to {end_date.strftime('%b %d')})"), # noqa - html.Div([ - html.Div( - [html.H2(id="numNewRequests"), html.Label("New Requests")], - className="stats-label" - ), - html.Div( - [html.H2(id="numClosedRequests"), html.Label("Closed Requests")], - className="stats-label" - ), - html.Div( - [html.H2(id="closeMinusNew"), html.Label("Net Change")], - className="stats-label" - ), - ], className="graph-row"), - html.Div([ - html.Div( - dcc.Graph(id='newRequestsLineChart', figure=fig, config=CONFIG_OPTIONS), - className="half-graph" - ), - html.Div( - dcc.Graph(id='neighRecentReqTypePieChart', figure=pie_fig, config=CONFIG_OPTIONS), - className="half-graph" - ) - ]), - html.Div( - dash_table.DataTable( - id='neighCouncilTbl', - columns=[ - {"name": pretty_columns[i], "id": i} for i in table_df.columns - ], - style_as_list_view=True, - style_cell={ - 'padding': '5px', - 'textAlign': 'left', - 'fontFamily': 'Roboto, Arial', - 'fontSize': 12, - 'color': '#333333', - 'backgroundColor': '#EEEEEE', - }, - style_header={ - 'backgroundColor': 'white', - 'fontWeight': 'bold' - }, - sort_action='native', - # filter_action='native', - page_size=20, - ) - ) -]) - - -# Define callback to update graph -@callback( - Output("neighCouncilTbl", "data"), - Input("council_list", "value") -) -def update_table(selected_council): - table_df = df.query(f"councilName == '{selected_council}'")[['srnumber', 'createdDate', 'closedDate', 'typeName', 'agencyName', 'sourceName', 'address']] # noqa - return table_df.to_dict('records') - - -@callback( - [ - Output("numNewRequests", "children"), - Output("numClosedRequests", "children"), - Output("closeMinusNew", "children"), - ], - Input("council_list", "value") -) -def update_text(selected_council): - create_count = df.query(f"councilName == '{selected_council}' and createdDate >= '{start_date}'")['srnumber'].count() # noqa - close_count = df.query(f"councilName == '{selected_council}' and closedDate >= '{start_date}'")['srnumber'].count() # noqa - return create_count, close_count, create_count - close_count - - -@callback( - Output("newRequestsLineChart", "figure"), - Input("council_list", "value") -) -def update_figure(selected_council): - figure_df = df.query(f"councilName == '{selected_council}' and createdDate >= '{start_date}'").groupby(['createdDate', 'typeName'])['srnumber'].count().reset_index() # noqa - figure_df.typeName = figure_df.typeName.map(lambda x: '
'.join(textwrap.wrap(x, width=16))) # noqa - - fig = px.line( - figure_df, - x="createdDate", - y="srnumber", - color="typeName", - color_discrete_sequence=DISCRETE_COLORS, - labels=LABELS, - title="Number of new " + selected_council + " Requests" - ) - fig.update_xaxes( - tickformat="%a\n%m/%d", - ) - fig.update_traces( - mode='markers+lines' - ) # add markers to lines - - apply_figure_style(fig) - - return fig - - -@callback( - Output("neighRecentReqTypePieChart", "pie_fig"), - Input("council_list", "value") -) -def update_council_figure(selected_council): - pie_df = df.query(f"councilName == '{selected_council}' and createdDate >= '{start_date}'").groupby(['typeName']).agg('count').reset_index() # noqa - - pie_fig = px.pie( - pie_df, - names="typeName", - values="srnumber", - color_discrete_sequence=DISCRETE_COLORS, - labels=LABELS, - title="Share of new requests types for " + selected_council - ) - - apply_figure_style(pie_fig) - - return pie_fig diff --git a/server/dash/dashboards/overview.py b/server/dash/dashboards/overview.py deleted file mode 100644 index 6ac1f4687..000000000 --- a/server/dash/dashboards/overview.py +++ /dev/null @@ -1,183 +0,0 @@ -import pandas as pd -import plotly.express as px -import plotly.graph_objects as go -import textwrap - -from dash import dcc, html -from config import API_HOST -from design import CONFIG_OPTIONS, DISCRETE_COLORS, DISCRETE_COLORS_MAP, LABELS, apply_figure_style - -# TITLE -title = "311 DATA OVERVIEW" - -# FIGURES -# council figure -print(" * Downloading data for dataframe") -query_string = "/reports?field=council_name&filter=created_date>=2016-01-01" -df1 = pd.read_json(API_HOST + query_string) -df1 = df1.groupby(['council_name'])['counts'].sum().sort_values().to_frame() -reqByNcBarChart = px.bar( - df1, - x=df1.index, - y='counts', - color_discrete_sequence=DISCRETE_COLORS, - labels=LABELS, - title="Total Requests by Neighborhood Councils", -) - -# year totals figure -print(" * Downloading data for dataframe") -query_string = "/reports?field=created_year&filter=created_date>=2016-01-01" -df2 = pd.read_json(API_HOST + query_string) -df2 = df2.groupby(['created_year'])['counts'].sum().to_frame() -numReqByYearBarChart = px.bar( - df2, - x=df2.index, - y='counts', - color_discrete_sequence=['#1D6996'], - labels=LABELS, - title="Total Requestes by Year", -) - -# agency figure -print(" * Downloading data for dataframe") -query_string = "/reports?field=agency_name&filter=created_date>=2016-01-01" -df5 = pd.read_json(API_HOST + query_string) -df5 = df5.groupby(['agency_name'])['counts'].sum().to_frame() -df5.sort_values('counts', ascending=False, inplace=True) -df5.loc['Others'] = df5[4:].sum() -df5.sort_values('counts', ascending=False, inplace=True) -df5 = df5[:5] -df5.index = df5.index.map(lambda x: '
'.join(textwrap.wrap(x, width=16))) - -shareReqByAgencyPieChart = px.pie( - df5, - names=df5.index, - values='counts', - color_discrete_sequence=DISCRETE_COLORS, - labels=LABELS, - hole=.3, - title="Total Requests by Agency", -) - -# source figure -print(" * Downloading data for dataframe") -query_string = "/reports?field=source_name&filter=created_date>=2016-01-01" -df6 = pd.read_json(API_HOST + query_string) -df6 = df6.groupby(['source_name'])['counts'].sum().to_frame() -df6.sort_values('counts', ascending=False, inplace=True) -df6.loc['Others'] = df6[4:].sum() -df6.sort_values('counts', ascending=False, inplace=True) -df6 = df6[:5] -reqSourceBarchart = px.bar( - df6, - x=df6.index, - y='counts', - color_discrete_sequence=['#1D6996'], - labels=LABELS, - title="Total Requests by Source", -) - -# types figure -print(" * Downloading data for dataframe") -query_string = "/reports?field=type_name&filter=created_date>=2016-01-01" -df3 = pd.read_json(API_HOST + query_string) -df3 = df3.groupby(['type_name'], as_index=False)['counts'].sum() -reqTypeSharePieChart = px.pie( - df3, - names="type_name", - values="counts", - color_discrete_sequence=DISCRETE_COLORS, - color="type_name", - color_discrete_map=DISCRETE_COLORS_MAP, - labels=LABELS, - hole=.3, - title="Total Requests by Type", -) -# fig3.update_layout(showlegend=False) - -stas_df = pd.read_json(API_HOST + '/types/stats') -stas_df = stas_df.sort_values('median', ascending=False) - -medDaysToCloseBoxPlot = go.Figure() - -medDaysToCloseBoxPlot.add_trace( - go.Box( - y=stas_df.type_name, - q1=stas_df['q1'], - median=stas_df['median'], - q3=stas_df['q3'], - marker_color='#29404F', - fillcolor='#E17C05', - ) -) - -medDaysToCloseBoxPlot.update_xaxes( - - dtick=5 -) - -medDaysToCloseBoxPlot.update_layout( - title="Total Median Days to Close by Type", -) - -# apply shared styles -apply_figure_style(reqByNcBarChart) -apply_figure_style(numReqByYearBarChart) -apply_figure_style(reqTypeSharePieChart) -apply_figure_style(medDaysToCloseBoxPlot) -apply_figure_style(shareReqByAgencyPieChart) -apply_figure_style(reqSourceBarchart) - -# LAYOUT -layout = html.Div([ - html.H1(title), - html.P("The figures below represent the total number of 311 requests made across LA County from 2016-2021. In 2020, we saw an all-time high with more than 1.4 million requests.", style={'padding':'20px', 'font-size':'18px', 'font-style':'italic'}), - html.Div([ - html.Div([html.H2(f"{df2['counts'].sum():,}"), html.Label("Total Requests")], className="stats-label"), # noqa - html.Div([html.H2(df1.shape[0] - 1), html.Label("Neighborhoods")], className="stats-label"), # noqa - html.Div([html.H2(df3.shape[0]), html.Label("Request Types")], className="stats-label"), # noqa - ], className="graph-row"), - html.Div([ - dcc.Graph( - id='numReqByYearBarChart', - figure=numReqByYearBarChart, - config=CONFIG_OPTIONS, - className="half-graph" - ), - dcc.Graph( - id='reqTypeSharePieChart', - figure=reqTypeSharePieChart, - config=CONFIG_OPTIONS, - className="half-graph" - )], - className="graph-row"), - dcc.Graph( - id='medDaysToCloseBoxPlot', - figure=medDaysToCloseBoxPlot, - config=CONFIG_OPTIONS, - responsive=True, - ), - html.Div([ - dcc.Graph( - id='shareReqByAgencyPieChart', - figure=shareReqByAgencyPieChart, - config=CONFIG_OPTIONS, - className="half-graph", - responsive=True, - ), - dcc.Graph( - id='reqSourceBarchart', - figure=reqSourceBarchart, - config=CONFIG_OPTIONS, - className="half-graph", - responsive=True, - )], - className="graph-row"), - dcc.Graph( - id='reqByNcBarChart', - figure=reqByNcBarChart, - config=CONFIG_OPTIONS, - responsive=True, - ), -]) diff --git a/server/dash/dashboards/recent.py b/server/dash/dashboards/recent.py deleted file mode 100644 index 5b69bd759..000000000 --- a/server/dash/dashboards/recent.py +++ /dev/null @@ -1,87 +0,0 @@ -import datetime -import pandas as pd -import plotly.express as px -import textwrap - -from dash import dcc, html -from config import API_HOST -from design import CONFIG_OPTIONS, DISCRETE_COLORS, DISCRETE_COLORS_MAP, LABELS, apply_figure_style - -# TITLE -title = "RECENT 311 REQUESTS" - -# DATA -start_date = datetime.date.today() - datetime.timedelta(days=30) -end_date = datetime.date.today() - datetime.timedelta(days=1) - -query_string = f"/reports?filter=created_date>={start_date}&filter=created_date<={end_date}" # noqa -print(" * Downloading data for dataframe") -df = pd.read_json(API_HOST + query_string) -print(" * Dataframe has been loaded") - -# FIGURES -report_df = df.groupby(['created_date', 'type_name']).agg('sum').reset_index() -report_df.type_name = report_df.type_name.map(lambda x: '
'.join(textwrap.wrap(x, width=16))) # noqa - -# Line Graph -numCreatedReqLineChart = px.line( - report_df, - x="created_date", - y="counts", - color="type_name", - color_discrete_sequence=DISCRETE_COLORS, - color_discrete_map=DISCRETE_COLORS_MAP, - labels=LABELS, -) - -numCreatedReqLineChart.update_xaxes( - tickformat="%a\n%m/%d", -) - -numCreatedReqLineChart.update_traces( - mode='markers+lines' -) # add markers to lines - -# Pie Chart -pie_df = df.groupby(['type_name']).agg('sum').reset_index() -recentReqTypeSharePieChart = px.pie( - pie_df, - names="type_name", - values="counts", - color="type_name", - color_discrete_sequence=DISCRETE_COLORS, - color_discrete_map=DISCRETE_COLORS_MAP, - labels=LABELS, - hole=.3, -) - -df['created_date'] = pd.to_datetime(df['created_date']) -dow_df = df.groupby(['created_date']).agg('sum').reset_index() -dow_df['day_of_week'] = dow_df['created_date'].dt.day_name() -numReqByDayOfWeekBarChart = px.bar( - dow_df, - x="day_of_week", - y="counts", - color_discrete_sequence=DISCRETE_COLORS, - labels=LABELS, -) - -apply_figure_style(numCreatedReqLineChart) -apply_figure_style(recentReqTypeSharePieChart) -apply_figure_style(numReqByDayOfWeekBarChart) - -# LAYOUT -layout = html.Div([ - html.H1(title), - html.P("The figures below represent the total number of 311 requests made across LA County over the past 30 days.", style={'padding':'20px', 'font-size':'18px', 'font-style':'italic'}), - html.Div([ - html.Div([html.H2(f"{report_df['counts'].sum():,}"), html.Label("Total Requests")], className="stats-label"), # noqa - html.Div([html.H2(f"{start_date.strftime('%b %d')}"), html.Label("Report Start Date")], className="stats-label"), # noqa - html.Div([html.H2(f"{end_date.strftime('%b %d')}"), html.Label("Report End Date")], className="stats-label"), # noqa - ], className="graph-row", style={'color':'white'}), - dcc.Graph(id='graph', figure=numCreatedReqLineChart, config=CONFIG_OPTIONS), - html.Div([ - dcc.Graph(id='numReqByDayOfWeekBarChart', figure=numReqByDayOfWeekBarChart, config=CONFIG_OPTIONS, className="half-graph"), # noqa - dcc.Graph(id='recentReqTypeSharePieChart', figure=recentReqTypeSharePieChart, config=CONFIG_OPTIONS, className="half-graph"), # noqa - ], className="graph-row"), -]) diff --git a/server/dash/dashboards/types_map.py b/server/dash/dashboards/types_map.py deleted file mode 100644 index 408382831..000000000 --- a/server/dash/dashboards/types_map.py +++ /dev/null @@ -1,86 +0,0 @@ -import datetime -import pandas as pd -import plotly.express as px -import json - -from dash import dcc, html, callback -from dash.dependencies import Input, Output -from config import API_HOST -from design import CONFIG_OPTIONS, LABELS, apply_figure_style -from urllib.request import urlopen - - -# TITLE -title = "REQUEST TYPE MAP" - -# DATA -start_date = datetime.date.today() - datetime.timedelta(days=365) -print(" * Downloading data for dataframe") -report_df = pd.read_json(f"{API_HOST}/reports?field=council_name&field=type_name&filter=created_date>={start_date}") # noqa - -print(" * Downloading geojson") -with urlopen(f"{API_HOST}/geojson") as response: - nc_geojson = json.load(response) - - -# REPORT -def populate_options(): - values = [] - for i in report_df.type_name.sort_values().unique(): - values.append({ - 'label': i, - 'value': i - }) - return values - - -layout = html.Div([ - html.H1(title), - dcc.Dropdown( - id="typesMapReqtypes", - clearable=False, - value="Illegal Dumping", - placeholder="Select a type", - searchable=False, - options=populate_options(), - ), - dcc.Graph(id="reqTypeChoroplethGraph", config=CONFIG_OPTIONS, className="hidden-graph"), -]) - - -@callback( - Output("reqTypeChoroplethGraph", "figure"), - [Input("typesMapReqtypes", "value")] -) -def display_choropleth(selected_value): - fig = px.choropleth( - report_df.query(f"type_name == '{selected_value}'"), - geojson=nc_geojson, - locations="council_name", - featureidkey="properties.council_name", - color="counts", - color_continuous_scale=px.colors.sequential.YlOrRd, - labels=LABELS, - projection="mercator", - ) - # show only the relevant map - fig.update_geos( - fitbounds="locations", - visible=False, - ) - # crops/zooms it - fig.update_layout( - autosize=True, - margin={"r": 0, "t": 0, "l": 0, "b": 0}, - ) - # move the scale to bottom left - fig.update_coloraxes( - # showscale=False - colorbar_x=0.225, - colorbar_y=0.225, - # colorbar_ypad=10, - colorbar_lenmode="fraction", - colorbar_len=0.3, - ) - apply_figure_style(fig) - return fig From 43e4d25df008bd9bb774f0973e4ddf7f7bf4f7ab Mon Sep 17 00:00:00 2001 From: Yi Heng Joshua Wu Date: Tue, 19 Jul 2022 18:11:20 -0700 Subject: [PATCH 13/53] Documentation for ncSUmComp --- server/dash/dashboards/ncSumComp.py | 334 ++++++++++++++++------------ 1 file changed, 193 insertions(+), 141 deletions(-) diff --git a/server/dash/dashboards/ncSumComp.py b/server/dash/dashboards/ncSumComp.py index 74fe2c03c..e54fa3331 100644 --- a/server/dash/dashboards/ncSumComp.py +++ b/server/dash/dashboards/ncSumComp.py @@ -7,148 +7,156 @@ import requests as re from config import API_HOST -from design import DISCRETE_COLORS, LABELS, apply_figure_style +from design import DISCRETE_COLORS, LABELS, apply_figure_style from dash import dcc, html, callback from dash.dependencies import Input, Output from dash.exceptions import PreventUpdate # Setting 1 week worth of data +# TODO: reset back to 1 week worth of data start_date = datetime.date.today() - datetime.timedelta(days=300) end_date = datetime.date.today() - datetime.timedelta(days=200) - + +# Loading the dataframe with the 311 Data API df_path = f"/requests/updated?start_date={start_date}&end_date={end_date}" results = re.get(API_HOST + df_path) data_json = results.json() data_2020 = pd.json_normalize(data_json) - layout = html.Div([ html.Div(children=[ - ## Neighborhood Council Dashboard + # Neighborhood Council Dashboard html.Div(children=[ - html.H2("LA 311 Requests - Neighborhood Council Summary Dashboard", style={'vertical-align':'middle'} ), - html.Div([daq.ToggleSwitch(id='dataQualitySwitch', value=True, style={'height':'vh'}, size = 35), - html.Div(id='dataQualityOutput')], style={'font-family':'Open Sans'}) - ], style={'display':'flex', "justify-content": "space-between", 'vertical-align':'middle' , 'height':'5vh', 'width':'97.5vw'}), - - ## Summary Dropdown + html.H2("LA 311 Requests - Neighborhood Council Summary Dashboard", + style={'vertical-align': 'middle'}), + html.Div([daq.ToggleSwitch(id='dataQualitySwitch', value=True, style={'height': 'vh'}, size=35), + html.Div(id='dataQualityOutput')], style={'font-family': 'Open Sans'}) + ], style={'display': 'flex', "justify-content": "space-between", 'vertical-align': 'middle', 'height': '5vh', 'width': '97.5vw'}), + + # Summary Dropdown html.Div(children=[ - html.Div(dcc.Dropdown(sorted([n for n in set(data_2020['councilName'])]), ' ', id='nc_dropdown', placeholder="Select a Neighborhood Council..."), style={'display': 'inline-block', 'width':'48.5vw'}), - html.Div(dcc.Dropdown(id='nc_dropdown_filter', multi=True, placeholder="Select a Request Type..."), style={'display': 'inline-block', 'width':'48.5vw'}) - ], style={'display':'flex' , "justify-content": "space-between", "width": "97.5vw", "height": "10vh"}), + html.Div(dcc.Dropdown(sorted([n for n in set(data_2020['councilName'])]), ' ', id='nc_dropdown', + placeholder="Select a Neighborhood Council..."), style={'display': 'inline-block', 'width': '48.5vw'}), + html.Div(dcc.Dropdown(id='nc_dropdown_filter', multi=True, placeholder="Select a Request Type..."), style={ + 'display': 'inline-block', 'width': '48.5vw'}) + ], style={'display': 'flex', "justify-content": "space-between", "width": "97.5vw", "height": "10vh"}), - html.Div(html.Br(), style={"height":"0.5vh"}), + html.Div(html.Br(), style={"height": "0.5vh"}), # Line Chart for Number of Request throughout the day - html.Div(dcc.Graph(id='ncAvgCompLineChart', style={"height":"40vh", 'width':'97.4vw'}), style={"border":"0.5px black solid", "height":"40vh", 'width':'97.4vw'}), - - html.Div(html.Br(), style={"height":"1vh"}), - + html.Div(dcc.Graph(id='ncAvgCompLineChart', style={"height": "40vh", 'width': '97.4vw'}), style={ + "border": "0.5px black solid", "height": "40vh", 'width': '97.4vw'}), + + html.Div(html.Br(), style={"height": "1vh"}), + html.Div(children=[ html.Div( # Pie Chart for the share of request type dcc.Graph( - id="reqTypePieChart", style={"height":"40vh", 'width':'48.5vw'} - ), style={'display': 'inline-block', 'width':'48.5vw', "border":"0.5px black solid", "height":"40vh"}), # for border-radius , add stuff later + id="reqTypePieChart", style={"height": "40vh", 'width': '48.5vw'} + ), style={'display': 'inline-block', 'width': '48.5vw', "border": "0.5px black solid", "height": "40vh"}), # for border-radius , add stuff later html.Div( # Histogram for the request timeToClose - dcc.Graph( - id="timeCloseHist", style={"height":"40vh", 'width':'48vw'} - ), style={'display': 'inline-block', 'width':'48vw', "border":"0.5px black solid", "height":"40vh"}) - ], style={'display':'flex', "justify-content": "space-between", "width":'97.5vw'}) - + dcc.Graph( + id="timeCloseHist", style={"height": "40vh", 'width': '48vw'} + ), style={'display': 'inline-block', 'width': '48vw', "border": "0.5px black solid", "height": "40vh"}) + ], style={'display': 'flex', "justify-content": "space-between", "width": '97.5vw'}) + ]), - html.Div(html.Br(), style={"height":"vh"}), - + html.Div(html.Br(), style={"height": "vh"}), + # Neighborhood Council Summarization Dashboard html.Div(children=[ html.H2("LA 311 Requests - Neighborhood Council Comparison Dashboard") - ], style={'textAlign':'center', 'height':'5vh'}), + ], style={'textAlign': 'center', 'height': '5vh'}), - ## Comparison Dropdowns + # Comparison Dropdowns html.Div(children=[ - html.Div(dcc.Dropdown(sorted([n for n in set(data_2020['councilName'])]), ' ', id='nc_comp_dropdown', placeholder="Select a Neighborhood Council..."), style={'display': 'inline-block', 'width':'48.5vw'}), - html.Div(dcc.Dropdown(sorted([n for n in set(data_2020['councilName'])]), ' ', id='nc_comp_dropdown2', placeholder="Select a Neighborhood Council..."), style={'display': 'inline-block', 'width':'48.5vw'}), - ], style={'display':'flex' , "justify-content": "space-between", "width": "97.5vw", "height": "12vh"}), + html.Div(dcc.Dropdown(sorted([n for n in set(data_2020['councilName'])]), ' ', id='nc_comp_dropdown', + placeholder="Select a Neighborhood Council..."), style={'display': 'inline-block', 'width': '48.5vw'}), + html.Div(dcc.Dropdown(sorted([n for n in set(data_2020['councilName'])]), ' ', id='nc_comp_dropdown2', + placeholder="Select a Neighborhood Council..."), style={'display': 'inline-block', 'width': '48.5vw'}), + ], style={'display': 'flex', "justify-content": "space-between", "width": "97.5vw", "height": "12vh"}), - - html.Div(html.Br(), style={"height":"1vh"}), + html.Div(html.Br(), style={"height": "1vh"}), - ## NC Comparison - Indicator Visuals - html.Div(children = [ - html.Div(children = [ - + # NC Comparison - Indicator Visuals + html.Div(children=[ + html.Div(children=[ - # Indicator Visuals for Total number of requests and the number of days the data spans across + # Indicator Visuals for Total number of requests and the number of days the data spans across html.Div([ - html.H6("Total Number of Requests", style={"text-align":'center'}), - html.H1(id='totalReqCard', style={"text-align":'center'})], - style={'display': 'inline-block', 'width':'24vw', 'height':'16vh', "border":"0.5px black solid"}), + html.H6("Total Number of Requests", style={"text-align": 'center'}), + html.H1(id='totalReqCard', style={"text-align": 'center'})], + style={'display': 'inline-block', 'width': '24vw', 'height': '16vh', "border": "0.5px black solid"}), html.Div([ - html.H6("Number of Days", style={"text-align":'center'}), - html.H1(id='numDaysCard', style={"text-align":'center'})], - style={'display': 'inline-block', 'width':'24vw', 'height':'16vh', "border":"0.5px black solid"}) - - - ], style={'display':'flex' , "justify-content": "space-between", 'width':'48.5vw'}), - - + html.H6("Number of Days", style={"text-align": 'center'}), + html.H1(id='numDaysCard', style={"text-align": 'center'})], + style={'display': 'inline-block', 'width': '24vw', 'height': '16vh', "border": "0.5px black solid"}) + + ], style={'display': 'flex', "justify-content": "space-between", 'width': '48.5vw'}), + # Indicator Visuals for Total number of requests and the number of days the data spans across - html.Div(children=[ + html.Div(children=[ html.Div([ - html.H6("Total Number of Requests", style={"text-align":'center'}), - html.H1(id='totalReqCard2', style={"text-align":'center'})], - style={'display': 'inline-block', 'width':'24vw', 'height':'16vh', "border":"0.5px black solid"}), + html.H6("Total Number of Requests", style={"text-align": 'center'}), + html.H1(id='totalReqCard2', style={"text-align": 'center'})], + style={'display': 'inline-block', 'width': '24vw', 'height': '16vh', "border": "0.5px black solid"}), html.Div([ - html.H6("Number of Days", style={"text-align":'center'}), - html.H1(id='numDaysCard2', style={"text-align":'center'})], - style={'display': 'inline-block', 'width':'24vw', 'height':'16vh', "border":"0.5px black solid"}) - - - ], style={'display':'flex' , "justify-content": "space-between", 'width':'48.5vw'}) - ] , style={'display':'flex' , "justify-content": "space-between", "width": "97.5vw"}), - - - html.Div(html.Br(), style={"height":"1vh"}), - - ## NC Comparison - Request Source Bar Charts - html.Div(children = [ - html.Div(dcc.Graph(id='reqSourceBarChart', style={"height":"30vh"}), style={'display': 'inline-block', 'width':'48.5vw', "border":"0.5px black solid", "height":"30vh"}), - html.Div(dcc.Graph(id='reqSourceBarChart2', style={"height":"30vh"}), style={'display': 'inline-block', 'width':'48.5vw', "border":"0.5px black solid", "margin-left": "10px", "height":"30vh"}) - ] , style={'display':'flex' , "justify-content": "space-between", "width": "97.5vw"}), - - html.Div(html.Br(), style={"height":"1vh"}), - - ## NC Comparison - Number of Requests per day Overlapping line chart - html.Div(dcc.Graph(id='overlayReqTimeLineChart', style={"height":"32vh", "width":"97.5vw"}), style={"border":"0.5px black solid", "height":"32vh", "width":"97.5vw"}) -]) + html.H6("Number of Days", style={"text-align": 'center'}), + html.H1(id='numDaysCard2', style={"text-align": 'center'})], + style={'display': 'inline-block', 'width': '24vw', 'height': '16vh', "border": "0.5px black solid"}) + + ], style={'display': 'flex', "justify-content": "space-between", 'width': '48.5vw'}) + ], style={'display': 'flex', "justify-content": "space-between", "width": "97.5vw"}), + html.Div(html.Br(), style={"height": "1vh"}), + # NC Comparison - Request Source Bar Charts + html.Div(children=[ + html.Div(dcc.Graph(id='reqSourceBarChart', style={"height": "30vh"}), style={ + 'display': 'inline-block', 'width': '48.5vw', "border": "0.5px black solid", "height": "30vh"}), + html.Div(dcc.Graph(id='reqSourceBarChart2', style={"height": "30vh"}), style={ + 'display': 'inline-block', 'width': '48.5vw', "border": "0.5px black solid", "margin-left": "10px", "height": "30vh"}) + ], style={'display': 'flex', "justify-content": "space-between", "width": "97.5vw"}), + + html.Div(html.Br(), style={"height": "1vh"}), + + # NC Comparison - Number of Requests per day Overlapping line chart + html.Div(dcc.Graph(id='overlayReqTimeLineChart', style={"height": "32vh", "width": "97.5vw"}), style={ + "border": "0.5px black solid", "height": "32vh", "width": "97.5vw"}) +]) -# Callback Function to generate Dynamic Filter Selection - -# Removing request types that doesn't exist in NC @callback( [Output('nc_dropdown_filter', 'options'), Output('nc_dropdown_filter', 'value')], - Input('nc_dropdown', 'value') - + Input('nc_dropdown', 'value') ) def generate_dynamic_filter(nc_dropdown): + """Enables the dashboard to show dynamic filters. + + This function takes the selected neighborhood council (nc) value from the "nc_dropdown" dropdown and output a a list of available request types + from that neigbhorhood council. + + Args: + nc_dropdown: A string argument automatically detected by Dash callback function when "nc_dropdown" element is selected in the layout. + + Returns: + nc_dropdown_filter: a list of request types available from the selected neigbhorhood council. + nc_dropdown_filter: placeholder value for the dropdown when nothing is selected. + """ + # If no neighborhood council is selected, use all data. if not nc_dropdown: df = data_2020 else: df = data_2020[data_2020['councilName'] == nc_dropdown] - + rTypes = sorted([n for n in set(df['typeName'])]) return rTypes, ' ' - -## Generate the charts and filters for NC Summary Dashboard, including: -## Share of Request Type Pie Chart, Distribution of request timeToClose, -## Number of Request throughout the day line chart, @callback( Output('reqTypePieChart', 'figure'), Output('timeCloseHist', 'figure'), @@ -159,6 +167,23 @@ def generate_dynamic_filter(nc_dropdown): Input('dataQualitySwitch', 'value') ) def generate_nc_summary_charts(nc_dropdown, nc_dropdown_filter=None, dataQualitySwitch=True): + """Generates the summary visualizations for LA 311 requests data based on selected neighborhood conucil, request types, and data quality switch. + + This function takes the selected neighborhood council (nc) value from the "nc_dropdown" dropdown, selected request type from "nc_dropdown_filter" + dropdown, and the status on the "dataQualitySwitch" toggle to output a 4 main visualizations to provide an overview. + + Args: + nc_dropdown: A string argument automatically detected by Dash callback function when "nc_dropdown" element is selected in the layout. + nc_dropdown_filter: A list of strings automatically detected by Dash callback function when "nc_dropdown_filter" element is selected in the layout, default None. + dataQualitySwitch: A boolean for data quality filter automatically detected by Dash callback function when "dataQualitySwitch" element is selected in the layout, default True. + + Returns: + reqTypePieChart: pie chart that shows the share of request types out of all requests. + timeCloseHist: histogram showing the distribution of time it takes for requests to close. + numReqLineChart: line chart showing the number of requests throughout the day. + dataQualityOutput: A string stating the status of the data quality filter ("Quality Filter: On" or "Quality Filter: Off"). + """ + # NC Dropdown if not nc_dropdown: df = data_2020 @@ -169,20 +194,24 @@ def generate_nc_summary_charts(nc_dropdown, nc_dropdown_filter=None, dataQuality df = data_2020 elif nc_dropdown_filter: df = df[df['typeName'].isin(nc_dropdown_filter)] - - # Pie Chart for the distribution of Request Types + + # Pie Chart for the distribution of Request Types rtype = pd.DataFrame(df['typeName'].value_counts()) rtype = rtype.reset_index() - reqTypePieChart = px.pie(rtype, values="typeName", names="index", title="Share of each Request Type") - reqTypePieChart.update_layout(margin=dict(l=50, r=50, b=50, t=50), legend_title = dict(font = dict(size = 10)), font=dict(size=9)) + reqTypePieChart = px.pie(rtype, values="typeName", names="index", + title="Share of each Request Type") + reqTypePieChart.update_layout(margin=dict(l=50, r=50, b=50, t=50), + legend_title=dict(font=dict(size=10)), font=dict(size=9)) # Distribution of Time to Close Date of each request - ## Calculate the Time to Closed - df.loc[:, 'createDateDT'] = pd.to_datetime(df.loc[:, 'createdDate'].str[:-4].str.split("T").str.join(" ")) - df.loc[:, 'closeDateDT'] = pd.to_datetime(df.loc[:, 'closedDate'].str[:-4].str.split("T").str.join(" ")) + # Calculate the Time to Closed + df.loc[:, 'createDateDT'] = pd.to_datetime( + df.loc[:, 'createdDate'].str[:-4].str.split("T").str.join(" ")) + df.loc[:, 'closeDateDT'] = pd.to_datetime( + df.loc[:, 'closedDate'].str[:-4].str.split("T").str.join(" ")) df.loc[:, 'timeToClose'] = (df.loc[:, 'closeDateDT'] - df.loc[:, 'createDateDT']).dt.days - ## Calculate the Optimal number of bins based on Freedman-Diaconis Rule + # Calculate the Optimal number of bins based on Freedman-Diaconis Rule # Replace empty rows with 0.0000001 To avoid log(0) error later df.loc[:, "timeToClose"] = df.loc[:, "timeToClose"].fillna(0.0000001) @@ -190,26 +219,26 @@ def generate_nc_summary_charts(nc_dropdown, nc_dropdown_filter=None, dataQuality # Replace negative values # TODO: figure out what to do when there is no data avaialble df = df[df['timeToClose'] > 0] - print(df.shape) if df.shape[0] == 0: raise PreventUpdate() else: - q3, q1 = np.percentile(df.loc[:, "timeToClose"].astype(int), [75 ,25]) + q3, q1 = np.percentile(df.loc[:, "timeToClose"].astype(int), [75, 25]) iqr = q3 - q1 if not iqr: numBins = 100 else: - numBins = int((2*iqr)/(df.shape[0]**(1/3))) + numBins = int((2 * iqr) / (df.shape[0]**(1 / 3))) # Log Transform, Compute IQR, then exclude outliers df.loc[:, "logTimeToClose"] = np.log(df.loc[:, "timeToClose"]) - log_q3, log_q1 = np.percentile(df.loc[:, "logTimeToClose"], [75 ,25]) + log_q3, log_q1 = np.percentile(df.loc[:, "logTimeToClose"], [75, 25]) log_iqr = log_q3 - log_q1 # Data Quality switch to remove outliers as defined by Median +- 1.5*IQR if dataQualitySwitch: # TODO: figure out what happens when the filtering mechanism output no data at all - temp = df[(df.loc[:, "logTimeToClose"] > 1.5*log_iqr - np.median(df.loc[:, "logTimeToClose"])) & (df.loc[:, "logTimeToClose"] < 1.5*log_iqr + np.median(df.loc[:, "logTimeToClose"]))] + temp = df[(df.loc[:, "logTimeToClose"] > 1.5 * log_iqr - np.median(df.loc[:, "logTimeToClose"])) & + (df.loc[:, "logTimeToClose"] < 1.5 * log_iqr + np.median(df.loc[:, "logTimeToClose"]))] if temp.shape[0] > 0: df = temp dataQualityOutput = "Quality Filter: On" @@ -217,25 +246,18 @@ def generate_nc_summary_charts(nc_dropdown, nc_dropdown_filter=None, dataQuality dataQualityOutput = "Quality Filter: Off" # Distribution for the total number of requests - timeCloseHist = px.histogram(df, x="timeToClose", title="Distribution of Time to Close Request", nbins= numBins, range_x=[min(df.loc[:, 'timeToClose']), max(df.loc[:, 'timeToClose'])], labels={"timeToClose":"Request Duration", "count":"Frequency"}) + timeCloseHist = px.histogram(df, x="timeToClose", title="Distribution of Time to Close Request", nbins=numBins, range_x=[min( + df.loc[:, 'timeToClose']), max(df.loc[:, 'timeToClose'])], labels={"timeToClose": "Request Duration", "count": "Frequency"}) timeCloseHist.update_layout(margin=dict(l=50, r=50, b=50, t=50), font=dict(size=9)) - ## Time Series for the Total Number of Requests - - # Implementation with Datetime + # Time Series for the Total Number of Requests rtime = pd.DataFrame(df.groupby('createDateDT', as_index=False)['srnumber'].count()) - numReqLineChart = px.line(rtime, x="createDateDT", y = 'srnumber', title="Total Number of 311 Requests Overtime", labels={"createDateDT":"DateTime", "srnumber":"Frequency"} ) + numReqLineChart = px.line(rtime, x="createDateDT", y='srnumber', title="Total Number of 311 Requests Overtime", labels={ + "createDateDT": "DateTime", "srnumber": "Frequency"}) numReqLineChart.update_layout(margin=dict(l=25, r=25, b=25, t=50), font=dict(size=9)) - - ## Potentially applying stylistic changes according to previous version - apply_figure_style(reqTypePieChart) - apply_figure_style(timeCloseHist) - apply_figure_style(numReqLineChart) - return reqTypePieChart, timeCloseHist, numReqLineChart, dataQualityOutput -## Generate the charts and filters for NC Comparison Dashboards, including the following @callback( Output('reqSourceBarChart', 'figure'), Output('reqSourceBarChart2', 'figure'), @@ -245,77 +267,107 @@ def generate_nc_summary_charts(nc_dropdown, nc_dropdown_filter=None, dataQuality Output('numDaysCard2', 'children'), Output('overlayReqTimeLineChart', 'figure'), Input('nc_comp_dropdown', 'value'), - Input('nc_comp_dropdown2', 'value'), + Input('nc_comp_dropdown2', 'value'), prevent_initial_call=True ) def generate_nc_comparison_charts(nc_comp_dropdown, nc_comp_dropdown2): + """Generates the comparison visualizations for LA 311 requests data based on the two selected neighborhood conucils. + + This function takes the first selected neighborhood council (nc) value from the "nc_comp_dropdown" dropdown and second selected neighborhood council value from "nc_comp_dropdown2" + dropdown and output 3 sets of comparison visuals and 1 overlapping visual. + + Args: + nc_comp_dropdown: A string argument automatically detected by Dash callback function when "nc_comp_dropdown" element is selected in the layout. + nc_comp_dropdown2: A string argument automatically detected by Dash callback function when "nc_comp_dropdown2" element is selected in the layout. + + Returns: + reqSourceBarChart: bar chart showing the number of request from each source for the first neighborhood council (e.g. mobile, app, self-report...etc). + reqSourceBarChart2: bar chart showing the number of request from each source for the second neighborhood council (e.g. mobile, app, self-report...etc). + totalReqCard: integer for the the total number of request in first selected neigborhood council. + totalReqCard2: integer for the total number of requests in the second selected neighborhood council. + numDaysCard: integer for the total number of days the data available in first selected neighborhood council span. + numDaysCard2: integer for the total number of days the data available in second selected neighborhood council span. + overlayReqTimeLineChart: line chart showing the number of requests throughout the day for both first and second selected neighborhood council. + """ + # Check if the neighborhood council dropdown is selected or not, else use all data if not nc_comp_dropdown: df_nc1 = data_2020 else: df_nc1 = data_2020[data_2020['councilName'] == nc_comp_dropdown] + + # Check if the second neighborhood council dropdown is selected or not, else use all data if not nc_comp_dropdown2: df_nc2 = data_2020 else: df_nc2 = data_2020[data_2020['councilName'] == nc_comp_dropdown2] - - print(df_nc1.shape) - df_nc1.loc[:, 'createDateDT'] = pd.to_datetime(df_nc1.loc[:, 'createdDate'].str[:-4].str.split("T").str.join(" ")) - df_nc2.loc[:, 'createDateDT'] = pd.to_datetime(df_nc2.loc[:, 'createdDate'].str[:-4].str.split("T").str.join(" ")) - + # Convert the strings into datetime + df_nc1.loc[:, 'createDateDT'] = pd.to_datetime( + df_nc1.loc[:, 'createdDate'].str[:-4].str.split("T").str.join(" ")) + df_nc2.loc[:, 'createDateDT'] = pd.to_datetime( + df_nc2.loc[:, 'createdDate'].str[:-4].str.split("T").str.join(" ")) - # Bar chart of different Request Type Sources + # Bar chart of different Request Type Sources for first selected neigbhorhood council rSource = pd.DataFrame(df_nc1['sourceName'].value_counts()) rSource = rSource.reset_index() - reqSourceBarChart = px.bar(rSource, x="sourceName", y="index", orientation='h', title='Number of Requests by Source', labels={"index":"Request Source", "sourceName":"Frequency"}) + reqSourceBarChart = px.bar(rSource, x="sourceName", y="index", orientation='h', title='Number of Requests by Source', labels={ + "index": "Request Source", "sourceName": "Frequency"}) reqSourceBarChart.update_layout(margin=dict(l=25, r=25, b=25, t=50), font=dict(size=9)) + # Bar chart of different Request Type Sources for second selected neigbhorhood council rSource2 = pd.DataFrame(df_nc2['sourceName'].value_counts()) rSource2 = rSource2.reset_index() - reqSourceBarChart2 = px.bar(rSource2, x="sourceName", y="index", orientation='h', title='Number of Requests by Source', labels={"index":"Request Source", "sourceName":"Frequency"}) - reqSourceBarChart2.update_layout(margin=dict(l=25, r=25, b=25, t=50) , font=dict(size=9)) + reqSourceBarChart2 = px.bar(rSource2, x="sourceName", y="index", orientation='h', title='Number of Requests by Source', labels={ + "index": "Request Source", "sourceName": "Frequency"}) + reqSourceBarChart2.update_layout(margin=dict(l=25, r=25, b=25, t=50), font=dict(size=9)) - # Indicator Variables for Total Requests 1 + # Total number of requests for first neigbhorhood council totalReqCard = df_nc1.shape[0] - # Indicator Variables for Total Requests 2 + # Total number of requests for second neigbhorhood council totalReqCard2 = df_nc2.shape[0] - # Indicator Visuals for the Number of Days the dataset spans 1 - + # Total number of days the available requests in first neigbhorhood council span numDaysCard = np.max(df_nc1['createDateDT'].dt.day) - np.min(df_nc1['createDateDT'].dt.day) + 1 - # Indicator Visuals for the Number of Days the dataset spans 2 + # Total number of days the available requests in second neigbhorhood council span numDaysCard2 = np.max(df_nc2['createDateDT'].dt.day) - np.min(df_nc2['createDateDT'].dt.day) + 1 - - # Overlapping line chart for number of request throughout the day + # Overlapping line chart for number of request throughout the day for both first and second neighborhood council rTime = pd.DataFrame(df_nc1.groupby('createDateDT', as_index=False)['srnumber'].count()) rTime2 = pd.DataFrame(df_nc2.groupby('createDateDT', as_index=False)['srnumber'].count()) overlayReqTimeLineChart = go.Figure() - overlayReqTimeLineChart.add_trace(go.Scatter(x=rTime['createDateDT'], y=rTime['srnumber'], mode='lines', name='NC1')) - overlayReqTimeLineChart.add_trace(go.Scatter(x=rTime2['createDateDT'], y=rTime2['srnumber'], mode='lines', name='NC2')) - - overlayReqTimeLineChart.update_layout(title='Number of Request Throughout the Day', margin=dict(l=25, r=25, b=35, t=50), xaxis_range=[min(min(rTime['createDateDT']), min(rTime2['createDateDT'])), max(max(rTime['createDateDT']), max(rTime2['createDateDT']))], font=dict(size=9)) - - apply_figure_style(reqSourceBarChart) - apply_figure_style(reqSourceBarChart2) - apply_figure_style(overlayReqTimeLineChart) - + overlayReqTimeLineChart.add_trace(go.Scatter( + x=rTime['createDateDT'], y=rTime['srnumber'], mode='lines', name='NC1')) + overlayReqTimeLineChart.add_trace(go.Scatter( + x=rTime2['createDateDT'], y=rTime2['srnumber'], mode='lines', name='NC2')) + overlayReqTimeLineChart.update_layout(title='Number of Request Throughout the Day', margin=dict(l=25, r=25, b=35, t=50), xaxis_range=[min( + min(rTime['createDateDT']), min(rTime2['createDateDT'])), max(max(rTime['createDateDT']), max(rTime2['createDateDT']))], font=dict(size=9)) return reqSourceBarChart, reqSourceBarChart2, totalReqCard, totalReqCard2, numDaysCard, numDaysCard2, overlayReqTimeLineChart - - -# Define callback to update graph @callback( Output('ncAvgCompLineChart', 'figure'), [Input("nc_dropdown", "value")] ) def update_figure(nc_dropdown): + """Generates a line chart visualizations for LA 311 requests data based on the two selected neighborhood conucils. + + This function takes the selected neighborhood council (nc) value from the "nc_dropdown" dropdown and output a line chart showing + the number of requests throughout the day and the average number of requests throughout the day (total number of requests / all 99 neighborhood councils). + + Args: + nc_dropdown: A string argument automatically detected by Dash callback function when "nc_dropdown" element is selected in the layout. + + Returns: + ncAvgCompLineChart: line chart showing the number of requests throughout the day for the selected neighborhood council and average + """ + # If dropdown value is empty, use all data available if not nc_dropdown: df = data_2020 + + # Calculating the average number of requests throughout the day neighborhood_sum_df = df[df.council_name == nc_dropdown].groupby(['created_date']).agg('sum').reset_index() # noqa total_sum_df = df.groupby(['created_date']).agg('sum').reset_index() total_sum_df["nc_avg"] = total_sum_df["counts"] / 99 @@ -327,7 +379,7 @@ def update_figure(nc_dropdown): y=['counts', 'nc_avg'], color_discrete_sequence=DISCRETE_COLORS, labels=LABELS, - title= "Number of " + nc_dropdown + " Requests compare with the average of all Neighborhood Councils requests" + title="Number of " + nc_dropdown + " Requests compare with the average of all Neighborhood Councils requests" ) fig.update_xaxes( @@ -340,4 +392,4 @@ def update_figure(nc_dropdown): apply_figure_style(fig) - return fig \ No newline at end of file + return fig From 952a1c08ec716926db715da5616e0b691ba8bf0a Mon Sep 17 00:00:00 2001 From: Yi Heng Joshua Wu Date: Tue, 19 Jul 2022 20:37:27 -0700 Subject: [PATCH 14/53] Add print status logs --- server/dash/app.py | 4 ++-- server/dash/dashboards/ncSumComp.py | 7 ++++++- server/dash/index.py | 1 - 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/server/dash/app.py b/server/dash/app.py index c1b02a80d..78017af77 100644 --- a/server/dash/app.py +++ b/server/dash/app.py @@ -2,8 +2,8 @@ from dash import Dash, dcc, html external_stylesheets = ['/static/reports.css'] -server = flask.Flask(__name__) -app = Dash(__name__, external_stylesheets=external_stylesheets, suppress_callback_exceptions=True, server=server) +# server = flask.Flask(__name__) +app = Dash(__name__, external_stylesheets=external_stylesheets, suppress_callback_exceptions=True) # set up default layout app.layout = html.Div([ diff --git a/server/dash/dashboards/ncSumComp.py b/server/dash/dashboards/ncSumComp.py index e54fa3331..dd4f0e008 100644 --- a/server/dash/dashboards/ncSumComp.py +++ b/server/dash/dashboards/ncSumComp.py @@ -18,10 +18,12 @@ end_date = datetime.date.today() - datetime.timedelta(days=200) # Loading the dataframe with the 311 Data API +print(" * Downloading data for dataframe") df_path = f"/requests/updated?start_date={start_date}&end_date={end_date}" results = re.get(API_HOST + df_path) data_json = results.json() data_2020 = pd.json_normalize(data_json) +print(" * Loading complete dataframe") layout = html.Div([ @@ -183,7 +185,7 @@ def generate_nc_summary_charts(nc_dropdown, nc_dropdown_filter=None, dataQuality numReqLineChart: line chart showing the number of requests throughout the day. dataQualityOutput: A string stating the status of the data quality filter ("Quality Filter: On" or "Quality Filter: Off"). """ - + print(" * Generating summary visualizations") # NC Dropdown if not nc_dropdown: df = data_2020 @@ -196,6 +198,7 @@ def generate_nc_summary_charts(nc_dropdown, nc_dropdown_filter=None, dataQuality df = df[df['typeName'].isin(nc_dropdown_filter)] # Pie Chart for the distribution of Request Types + print(" * Generating requests types pie chart") rtype = pd.DataFrame(df['typeName'].value_counts()) rtype = rtype.reset_index() reqTypePieChart = px.pie(rtype, values="typeName", names="index", @@ -246,11 +249,13 @@ def generate_nc_summary_charts(nc_dropdown, nc_dropdown_filter=None, dataQuality dataQualityOutput = "Quality Filter: Off" # Distribution for the total number of requests + print(" * Generating requests time to close histogram") timeCloseHist = px.histogram(df, x="timeToClose", title="Distribution of Time to Close Request", nbins=numBins, range_x=[min( df.loc[:, 'timeToClose']), max(df.loc[:, 'timeToClose'])], labels={"timeToClose": "Request Duration", "count": "Frequency"}) timeCloseHist.update_layout(margin=dict(l=50, r=50, b=50, t=50), font=dict(size=9)) # Time Series for the Total Number of Requests + print(" * Generating number of requests line chart") rtime = pd.DataFrame(df.groupby('createDateDT', as_index=False)['srnumber'].count()) numReqLineChart = px.line(rtime, x="createDateDT", y='srnumber', title="Total Number of 311 Requests Overtime", labels={ "createDateDT": "DateTime", "srnumber": "Frequency"}) diff --git a/server/dash/index.py b/server/dash/index.py index 006040726..e3e1a2a5e 100644 --- a/server/dash/index.py +++ b/server/dash/index.py @@ -39,7 +39,6 @@ def display_page(pathname): last_part = re.search("([\w]*)$", pathname).group(1) # noqa - print(last_part) # run the dashboard if last_part in available_dashboards: logger.log(logging.INFO, f"Running dashboard: {last_part}") From 2b2307371d54eb736e4e27ad9e41f9dab049809b Mon Sep 17 00:00:00 2001 From: Yi Heng Joshua Wu Date: Sun, 24 Jul 2022 12:48:11 -0700 Subject: [PATCH 15/53] Split into two pr --- server/dash/dashboards/ncSumComp.py | 400 ---------------------------- 1 file changed, 400 deletions(-) delete mode 100644 server/dash/dashboards/ncSumComp.py diff --git a/server/dash/dashboards/ncSumComp.py b/server/dash/dashboards/ncSumComp.py deleted file mode 100644 index dd4f0e008..000000000 --- a/server/dash/dashboards/ncSumComp.py +++ /dev/null @@ -1,400 +0,0 @@ -import datetime -import dash_daq as daq -import numpy as np -import pandas as pd -import plotly.express as px -import plotly.graph_objects as go -import requests as re - -from config import API_HOST -from design import DISCRETE_COLORS, LABELS, apply_figure_style -from dash import dcc, html, callback -from dash.dependencies import Input, Output -from dash.exceptions import PreventUpdate - -# Setting 1 week worth of data -# TODO: reset back to 1 week worth of data -start_date = datetime.date.today() - datetime.timedelta(days=300) -end_date = datetime.date.today() - datetime.timedelta(days=200) - -# Loading the dataframe with the 311 Data API -print(" * Downloading data for dataframe") -df_path = f"/requests/updated?start_date={start_date}&end_date={end_date}" -results = re.get(API_HOST + df_path) -data_json = results.json() -data_2020 = pd.json_normalize(data_json) -print(" * Loading complete dataframe") - -layout = html.Div([ - - html.Div(children=[ - - # Neighborhood Council Dashboard - html.Div(children=[ - html.H2("LA 311 Requests - Neighborhood Council Summary Dashboard", - style={'vertical-align': 'middle'}), - html.Div([daq.ToggleSwitch(id='dataQualitySwitch', value=True, style={'height': 'vh'}, size=35), - html.Div(id='dataQualityOutput')], style={'font-family': 'Open Sans'}) - ], style={'display': 'flex', "justify-content": "space-between", 'vertical-align': 'middle', 'height': '5vh', 'width': '97.5vw'}), - - # Summary Dropdown - html.Div(children=[ - html.Div(dcc.Dropdown(sorted([n for n in set(data_2020['councilName'])]), ' ', id='nc_dropdown', - placeholder="Select a Neighborhood Council..."), style={'display': 'inline-block', 'width': '48.5vw'}), - html.Div(dcc.Dropdown(id='nc_dropdown_filter', multi=True, placeholder="Select a Request Type..."), style={ - 'display': 'inline-block', 'width': '48.5vw'}) - ], style={'display': 'flex', "justify-content": "space-between", "width": "97.5vw", "height": "10vh"}), - - html.Div(html.Br(), style={"height": "0.5vh"}), - - # Line Chart for Number of Request throughout the day - html.Div(dcc.Graph(id='ncAvgCompLineChart', style={"height": "40vh", 'width': '97.4vw'}), style={ - "border": "0.5px black solid", "height": "40vh", 'width': '97.4vw'}), - - html.Div(html.Br(), style={"height": "1vh"}), - - html.Div(children=[ - html.Div( - # Pie Chart for the share of request type - dcc.Graph( - id="reqTypePieChart", style={"height": "40vh", 'width': '48.5vw'} - ), style={'display': 'inline-block', 'width': '48.5vw', "border": "0.5px black solid", "height": "40vh"}), # for border-radius , add stuff later - html.Div( - # Histogram for the request timeToClose - dcc.Graph( - id="timeCloseHist", style={"height": "40vh", 'width': '48vw'} - ), style={'display': 'inline-block', 'width': '48vw', "border": "0.5px black solid", "height": "40vh"}) - ], style={'display': 'flex', "justify-content": "space-between", "width": '97.5vw'}) - - ]), - - html.Div(html.Br(), style={"height": "vh"}), - - # Neighborhood Council Summarization Dashboard - html.Div(children=[ - html.H2("LA 311 Requests - Neighborhood Council Comparison Dashboard") - ], style={'textAlign': 'center', 'height': '5vh'}), - - # Comparison Dropdowns - html.Div(children=[ - html.Div(dcc.Dropdown(sorted([n for n in set(data_2020['councilName'])]), ' ', id='nc_comp_dropdown', - placeholder="Select a Neighborhood Council..."), style={'display': 'inline-block', 'width': '48.5vw'}), - html.Div(dcc.Dropdown(sorted([n for n in set(data_2020['councilName'])]), ' ', id='nc_comp_dropdown2', - placeholder="Select a Neighborhood Council..."), style={'display': 'inline-block', 'width': '48.5vw'}), - ], style={'display': 'flex', "justify-content": "space-between", "width": "97.5vw", "height": "12vh"}), - - html.Div(html.Br(), style={"height": "1vh"}), - - # NC Comparison - Indicator Visuals - html.Div(children=[ - html.Div(children=[ - - # Indicator Visuals for Total number of requests and the number of days the data spans across - html.Div([ - html.H6("Total Number of Requests", style={"text-align": 'center'}), - html.H1(id='totalReqCard', style={"text-align": 'center'})], - style={'display': 'inline-block', 'width': '24vw', 'height': '16vh', "border": "0.5px black solid"}), - html.Div([ - html.H6("Number of Days", style={"text-align": 'center'}), - html.H1(id='numDaysCard', style={"text-align": 'center'})], - style={'display': 'inline-block', 'width': '24vw', 'height': '16vh', "border": "0.5px black solid"}) - - ], style={'display': 'flex', "justify-content": "space-between", 'width': '48.5vw'}), - - # Indicator Visuals for Total number of requests and the number of days the data spans across - html.Div(children=[ - html.Div([ - html.H6("Total Number of Requests", style={"text-align": 'center'}), - html.H1(id='totalReqCard2', style={"text-align": 'center'})], - style={'display': 'inline-block', 'width': '24vw', 'height': '16vh', "border": "0.5px black solid"}), - html.Div([ - html.H6("Number of Days", style={"text-align": 'center'}), - html.H1(id='numDaysCard2', style={"text-align": 'center'})], - style={'display': 'inline-block', 'width': '24vw', 'height': '16vh', "border": "0.5px black solid"}) - - ], style={'display': 'flex', "justify-content": "space-between", 'width': '48.5vw'}) - ], style={'display': 'flex', "justify-content": "space-between", "width": "97.5vw"}), - - html.Div(html.Br(), style={"height": "1vh"}), - - # NC Comparison - Request Source Bar Charts - html.Div(children=[ - html.Div(dcc.Graph(id='reqSourceBarChart', style={"height": "30vh"}), style={ - 'display': 'inline-block', 'width': '48.5vw', "border": "0.5px black solid", "height": "30vh"}), - html.Div(dcc.Graph(id='reqSourceBarChart2', style={"height": "30vh"}), style={ - 'display': 'inline-block', 'width': '48.5vw', "border": "0.5px black solid", "margin-left": "10px", "height": "30vh"}) - ], style={'display': 'flex', "justify-content": "space-between", "width": "97.5vw"}), - - html.Div(html.Br(), style={"height": "1vh"}), - - # NC Comparison - Number of Requests per day Overlapping line chart - html.Div(dcc.Graph(id='overlayReqTimeLineChart', style={"height": "32vh", "width": "97.5vw"}), style={ - "border": "0.5px black solid", "height": "32vh", "width": "97.5vw"}) -]) - -@callback( - [Output('nc_dropdown_filter', 'options'), - Output('nc_dropdown_filter', 'value')], - Input('nc_dropdown', 'value') -) -def generate_dynamic_filter(nc_dropdown): - """Enables the dashboard to show dynamic filters. - - This function takes the selected neighborhood council (nc) value from the "nc_dropdown" dropdown and output a a list of available request types - from that neigbhorhood council. - - Args: - nc_dropdown: A string argument automatically detected by Dash callback function when "nc_dropdown" element is selected in the layout. - - Returns: - nc_dropdown_filter: a list of request types available from the selected neigbhorhood council. - nc_dropdown_filter: placeholder value for the dropdown when nothing is selected. - """ - # If no neighborhood council is selected, use all data. - if not nc_dropdown: - df = data_2020 - else: - df = data_2020[data_2020['councilName'] == nc_dropdown] - - rTypes = sorted([n for n in set(df['typeName'])]) - return rTypes, ' ' - -@callback( - Output('reqTypePieChart', 'figure'), - Output('timeCloseHist', 'figure'), - Output('numReqLineChart', 'figure'), - Output('dataQualityOutput', 'children'), - Input('nc_dropdown', 'value'), - [Input('nc_dropdown_filter', 'value')], - Input('dataQualitySwitch', 'value') -) -def generate_nc_summary_charts(nc_dropdown, nc_dropdown_filter=None, dataQualitySwitch=True): - """Generates the summary visualizations for LA 311 requests data based on selected neighborhood conucil, request types, and data quality switch. - - This function takes the selected neighborhood council (nc) value from the "nc_dropdown" dropdown, selected request type from "nc_dropdown_filter" - dropdown, and the status on the "dataQualitySwitch" toggle to output a 4 main visualizations to provide an overview. - - Args: - nc_dropdown: A string argument automatically detected by Dash callback function when "nc_dropdown" element is selected in the layout. - nc_dropdown_filter: A list of strings automatically detected by Dash callback function when "nc_dropdown_filter" element is selected in the layout, default None. - dataQualitySwitch: A boolean for data quality filter automatically detected by Dash callback function when "dataQualitySwitch" element is selected in the layout, default True. - - Returns: - reqTypePieChart: pie chart that shows the share of request types out of all requests. - timeCloseHist: histogram showing the distribution of time it takes for requests to close. - numReqLineChart: line chart showing the number of requests throughout the day. - dataQualityOutput: A string stating the status of the data quality filter ("Quality Filter: On" or "Quality Filter: Off"). - """ - print(" * Generating summary visualizations") - # NC Dropdown - if not nc_dropdown: - df = data_2020 - else: - df = data_2020[data_2020['councilName'] == nc_dropdown] - # Filter as per selection on Reqquest Type Dropdown - if not nc_dropdown_filter and not nc_dropdown: - df = data_2020 - elif nc_dropdown_filter: - df = df[df['typeName'].isin(nc_dropdown_filter)] - - # Pie Chart for the distribution of Request Types - print(" * Generating requests types pie chart") - rtype = pd.DataFrame(df['typeName'].value_counts()) - rtype = rtype.reset_index() - reqTypePieChart = px.pie(rtype, values="typeName", names="index", - title="Share of each Request Type") - reqTypePieChart.update_layout(margin=dict(l=50, r=50, b=50, t=50), - legend_title=dict(font=dict(size=10)), font=dict(size=9)) - - # Distribution of Time to Close Date of each request - # Calculate the Time to Closed - df.loc[:, 'createDateDT'] = pd.to_datetime( - df.loc[:, 'createdDate'].str[:-4].str.split("T").str.join(" ")) - df.loc[:, 'closeDateDT'] = pd.to_datetime( - df.loc[:, 'closedDate'].str[:-4].str.split("T").str.join(" ")) - df.loc[:, 'timeToClose'] = (df.loc[:, 'closeDateDT'] - df.loc[:, 'createDateDT']).dt.days - - # Calculate the Optimal number of bins based on Freedman-Diaconis Rule - - # Replace empty rows with 0.0000001 To avoid log(0) error later - df.loc[:, "timeToClose"] = df.loc[:, "timeToClose"].fillna(0.0000001) - - # Replace negative values - # TODO: figure out what to do when there is no data avaialble - df = df[df['timeToClose'] > 0] - if df.shape[0] == 0: - raise PreventUpdate() - else: - q3, q1 = np.percentile(df.loc[:, "timeToClose"].astype(int), [75, 25]) - iqr = q3 - q1 - if not iqr: - numBins = 100 - else: - numBins = int((2 * iqr) / (df.shape[0]**(1 / 3))) - - # Log Transform, Compute IQR, then exclude outliers - df.loc[:, "logTimeToClose"] = np.log(df.loc[:, "timeToClose"]) - log_q3, log_q1 = np.percentile(df.loc[:, "logTimeToClose"], [75, 25]) - log_iqr = log_q3 - log_q1 - - # Data Quality switch to remove outliers as defined by Median +- 1.5*IQR - if dataQualitySwitch: - # TODO: figure out what happens when the filtering mechanism output no data at all - temp = df[(df.loc[:, "logTimeToClose"] > 1.5 * log_iqr - np.median(df.loc[:, "logTimeToClose"])) & - (df.loc[:, "logTimeToClose"] < 1.5 * log_iqr + np.median(df.loc[:, "logTimeToClose"]))] - if temp.shape[0] > 0: - df = temp - dataQualityOutput = "Quality Filter: On" - else: - dataQualityOutput = "Quality Filter: Off" - - # Distribution for the total number of requests - print(" * Generating requests time to close histogram") - timeCloseHist = px.histogram(df, x="timeToClose", title="Distribution of Time to Close Request", nbins=numBins, range_x=[min( - df.loc[:, 'timeToClose']), max(df.loc[:, 'timeToClose'])], labels={"timeToClose": "Request Duration", "count": "Frequency"}) - timeCloseHist.update_layout(margin=dict(l=50, r=50, b=50, t=50), font=dict(size=9)) - - # Time Series for the Total Number of Requests - print(" * Generating number of requests line chart") - rtime = pd.DataFrame(df.groupby('createDateDT', as_index=False)['srnumber'].count()) - numReqLineChart = px.line(rtime, x="createDateDT", y='srnumber', title="Total Number of 311 Requests Overtime", labels={ - "createDateDT": "DateTime", "srnumber": "Frequency"}) - numReqLineChart.update_layout(margin=dict(l=25, r=25, b=25, t=50), font=dict(size=9)) - - return reqTypePieChart, timeCloseHist, numReqLineChart, dataQualityOutput - -@callback( - Output('reqSourceBarChart', 'figure'), - Output('reqSourceBarChart2', 'figure'), - Output('totalReqCard', 'children'), - Output('totalReqCard2', 'children'), - Output('numDaysCard', 'children'), - Output('numDaysCard2', 'children'), - Output('overlayReqTimeLineChart', 'figure'), - Input('nc_comp_dropdown', 'value'), - Input('nc_comp_dropdown2', 'value'), - prevent_initial_call=True -) -def generate_nc_comparison_charts(nc_comp_dropdown, nc_comp_dropdown2): - """Generates the comparison visualizations for LA 311 requests data based on the two selected neighborhood conucils. - - This function takes the first selected neighborhood council (nc) value from the "nc_comp_dropdown" dropdown and second selected neighborhood council value from "nc_comp_dropdown2" - dropdown and output 3 sets of comparison visuals and 1 overlapping visual. - - Args: - nc_comp_dropdown: A string argument automatically detected by Dash callback function when "nc_comp_dropdown" element is selected in the layout. - nc_comp_dropdown2: A string argument automatically detected by Dash callback function when "nc_comp_dropdown2" element is selected in the layout. - - Returns: - reqSourceBarChart: bar chart showing the number of request from each source for the first neighborhood council (e.g. mobile, app, self-report...etc). - reqSourceBarChart2: bar chart showing the number of request from each source for the second neighborhood council (e.g. mobile, app, self-report...etc). - totalReqCard: integer for the the total number of request in first selected neigborhood council. - totalReqCard2: integer for the total number of requests in the second selected neighborhood council. - numDaysCard: integer for the total number of days the data available in first selected neighborhood council span. - numDaysCard2: integer for the total number of days the data available in second selected neighborhood council span. - overlayReqTimeLineChart: line chart showing the number of requests throughout the day for both first and second selected neighborhood council. - """ - # Check if the neighborhood council dropdown is selected or not, else use all data - if not nc_comp_dropdown: - df_nc1 = data_2020 - else: - df_nc1 = data_2020[data_2020['councilName'] == nc_comp_dropdown] - - # Check if the second neighborhood council dropdown is selected or not, else use all data - if not nc_comp_dropdown2: - df_nc2 = data_2020 - else: - df_nc2 = data_2020[data_2020['councilName'] == nc_comp_dropdown2] - - # Convert the strings into datetime - df_nc1.loc[:, 'createDateDT'] = pd.to_datetime( - df_nc1.loc[:, 'createdDate'].str[:-4].str.split("T").str.join(" ")) - df_nc2.loc[:, 'createDateDT'] = pd.to_datetime( - df_nc2.loc[:, 'createdDate'].str[:-4].str.split("T").str.join(" ")) - - # Bar chart of different Request Type Sources for first selected neigbhorhood council - rSource = pd.DataFrame(df_nc1['sourceName'].value_counts()) - rSource = rSource.reset_index() - reqSourceBarChart = px.bar(rSource, x="sourceName", y="index", orientation='h', title='Number of Requests by Source', labels={ - "index": "Request Source", "sourceName": "Frequency"}) - reqSourceBarChart.update_layout(margin=dict(l=25, r=25, b=25, t=50), font=dict(size=9)) - - # Bar chart of different Request Type Sources for second selected neigbhorhood council - rSource2 = pd.DataFrame(df_nc2['sourceName'].value_counts()) - rSource2 = rSource2.reset_index() - reqSourceBarChart2 = px.bar(rSource2, x="sourceName", y="index", orientation='h', title='Number of Requests by Source', labels={ - "index": "Request Source", "sourceName": "Frequency"}) - reqSourceBarChart2.update_layout(margin=dict(l=25, r=25, b=25, t=50), font=dict(size=9)) - - # Total number of requests for first neigbhorhood council - totalReqCard = df_nc1.shape[0] - - # Total number of requests for second neigbhorhood council - totalReqCard2 = df_nc2.shape[0] - - # Total number of days the available requests in first neigbhorhood council span - numDaysCard = np.max(df_nc1['createDateDT'].dt.day) - np.min(df_nc1['createDateDT'].dt.day) + 1 - - # Total number of days the available requests in second neigbhorhood council span - numDaysCard2 = np.max(df_nc2['createDateDT'].dt.day) - np.min(df_nc2['createDateDT'].dt.day) + 1 - - # Overlapping line chart for number of request throughout the day for both first and second neighborhood council - rTime = pd.DataFrame(df_nc1.groupby('createDateDT', as_index=False)['srnumber'].count()) - rTime2 = pd.DataFrame(df_nc2.groupby('createDateDT', as_index=False)['srnumber'].count()) - overlayReqTimeLineChart = go.Figure() - overlayReqTimeLineChart.add_trace(go.Scatter( - x=rTime['createDateDT'], y=rTime['srnumber'], mode='lines', name='NC1')) - overlayReqTimeLineChart.add_trace(go.Scatter( - x=rTime2['createDateDT'], y=rTime2['srnumber'], mode='lines', name='NC2')) - - overlayReqTimeLineChart.update_layout(title='Number of Request Throughout the Day', margin=dict(l=25, r=25, b=35, t=50), xaxis_range=[min( - min(rTime['createDateDT']), min(rTime2['createDateDT'])), max(max(rTime['createDateDT']), max(rTime2['createDateDT']))], font=dict(size=9)) - - return reqSourceBarChart, reqSourceBarChart2, totalReqCard, totalReqCard2, numDaysCard, numDaysCard2, overlayReqTimeLineChart - -@callback( - Output('ncAvgCompLineChart', 'figure'), - [Input("nc_dropdown", "value")] -) -def update_figure(nc_dropdown): - """Generates a line chart visualizations for LA 311 requests data based on the two selected neighborhood conucils. - - This function takes the selected neighborhood council (nc) value from the "nc_dropdown" dropdown and output a line chart showing - the number of requests throughout the day and the average number of requests throughout the day (total number of requests / all 99 neighborhood councils). - - Args: - nc_dropdown: A string argument automatically detected by Dash callback function when "nc_dropdown" element is selected in the layout. - - Returns: - ncAvgCompLineChart: line chart showing the number of requests throughout the day for the selected neighborhood council and average - """ - # If dropdown value is empty, use all data available - if not nc_dropdown: - df = data_2020 - - # Calculating the average number of requests throughout the day - neighborhood_sum_df = df[df.council_name == nc_dropdown].groupby(['created_date']).agg('sum').reset_index() # noqa - total_sum_df = df.groupby(['created_date']).agg('sum').reset_index() - total_sum_df["nc_avg"] = total_sum_df["counts"] / 99 - merged_df = neighborhood_sum_df.merge(total_sum_df["nc_avg"].to_frame(), left_index=True, right_index=True) # noqa - - fig = px.line( - merged_df, - x="created_date", - y=['counts', 'nc_avg'], - color_discrete_sequence=DISCRETE_COLORS, - labels=LABELS, - title="Number of " + nc_dropdown + " Requests compare with the average of all Neighborhood Councils requests" - ) - - fig.update_xaxes( - tickformat="%a\n%m/%d", - ) - - fig.update_traces( - mode='markers+lines' - ) # add markers to lines - - apply_figure_style(fig) - - return fig From d6a4b69118d2f55879c4c24ef7dc8238faed120d Mon Sep 17 00:00:00 2001 From: Yi Heng Joshua Wu Date: Tue, 26 Jul 2022 20:51:55 -0700 Subject: [PATCH 16/53] Update app.py comment --- server/dash/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/dash/app.py b/server/dash/app.py index 78017af77..46c851d1f 100644 --- a/server/dash/app.py +++ b/server/dash/app.py @@ -2,7 +2,7 @@ from dash import Dash, dcc, html external_stylesheets = ['/static/reports.css'] -# server = flask.Flask(__name__) +# Suppress Callback Exceptions due to multi-page Dashboard layout, some callback id may not exist initially app = Dash(__name__, external_stylesheets=external_stylesheets, suppress_callback_exceptions=True) # set up default layout From b9a47b49602ad546fb6c544f44152d324581a413 Mon Sep 17 00:00:00 2001 From: Yi Heng Joshua Wu Date: Tue, 26 Jul 2022 20:54:36 -0700 Subject: [PATCH 17/53] Capitalize title constant --- server/dash/dashboards/overviewComb.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/server/dash/dashboards/overviewComb.py b/server/dash/dashboards/overviewComb.py index b3fb267c6..a74c7e378 100644 --- a/server/dash/dashboards/overviewComb.py +++ b/server/dash/dashboards/overviewComb.py @@ -9,7 +9,7 @@ from config import API_HOST from design import CONFIG_OPTIONS, DISCRETE_COLORS, DISCRETE_COLORS_MAP, LABELS, apply_figure_style -title = "OVERVIEW COMBINED DASHBOARD" +TITLE = "OVERVIEW COMBINED DASHBOARD" # DATA print(" * Downloading data for dataframe") @@ -63,7 +63,7 @@ values='counts', labels=LABELS, hole=.3, - title="Total Requests by Agency", + TITLE="Total Requests by Agency", ) shareReqByAgencyPieChart.update_layout(margin=dict(l=100, r=100, b=100, t=100)) @@ -80,7 +80,7 @@ df6, y=df6.index, x='counts', - title="Total Requests by Source", + TITLE="Total Requests by Source", orientation='h' ) @@ -103,7 +103,7 @@ dtick=5 ) medDaysToCloseBoxPlot.update_layout( - title="Total Median Days to Close by Type", + TITLE="Total Median Days to Close by Type", ) # Day of Week Bar Chart: @@ -133,14 +133,14 @@ x=df1.index, y='counts', labels=LABELS, - title="Total Requests by Neighborhood Councils", + TITLE="Total Requests by Neighborhood Councils", ) reqByNcBarChart.update_layout(font=dict(size=12)) # LAYOUT layout = html.Div([ # Page 2 with Cards + Stuf - html.H1(title + " Pt. 1"), + html.H1(TITLE + " Pt. 1"), html.P("The figures below represent the total number of 311 requests made across LA County from 2016-2021. In 2020, we saw an all-time high with more than 1.4 million requests.", style={'font-size': '18px', 'font-style': 'italic'}), @@ -165,7 +165,7 @@ responsive=True, style={"width": "35vw", "height": "60vh"}), style={"border": "0.5px black solid"}) ], className="graph-row", style={'display': 'flex', "justify-content": "space-between"}), html.Div(html.Br(), style={"height": "2vh"}), - html.H1(title + " Pt. 2"), + html.H1(TITLE + " Pt. 2"), html.Div([ html.Div(dcc.Graph(id='numReqByDayOfWeekBarChart', figure=numReqByDayOfWeekBarChart, className="half-graph", style={"width": "48vw", "height": "40vh"}), style={"border": "0.5px black solid"}), # noqa html.Div(dcc.Graph(id='reqSourceBarchart', figure=reqSourceBarchart, className="half-graph", From 50dc742cec8a1884781ffdd596fbeeae9f24d4cc Mon Sep 17 00:00:00 2001 From: Yi Heng Joshua Wu Date: Tue, 26 Jul 2022 21:22:30 -0700 Subject: [PATCH 18/53] Update api path var names --- server/dash/dashboards/overviewComb.py | 47 ++++++++++++++------------ 1 file changed, 26 insertions(+), 21 deletions(-) diff --git a/server/dash/dashboards/overviewComb.py b/server/dash/dashboards/overviewComb.py index a74c7e378..47a10274d 100644 --- a/server/dash/dashboards/overviewComb.py +++ b/server/dash/dashboards/overviewComb.py @@ -3,48 +3,53 @@ import pandas as pd import plotly.express as px import plotly.graph_objects as go -from dash import callback, dcc, html -from dash.dependencies import Input, Output +from dash import dcc, html from config import API_HOST -from design import CONFIG_OPTIONS, DISCRETE_COLORS, DISCRETE_COLORS_MAP, LABELS, apply_figure_style +from design import LABELS TITLE = "OVERVIEW COMBINED DASHBOARD" # DATA -print(" * Downloading data for dataframe") -query_string = '/reports?field=type_name&field=council_name&field=created_date' -df = pd.read_json(API_HOST + query_string) -print(" * Dataframe has been loaded") +NC_REQ_TYPE_DATA_API_PATH = '/reports?field=type_name&field=council_name&field=created_date' +print(" * Downloading data from API path: " + NC_REQ_TYPE_DATA_API_PATH) +df = pd.read_json(API_HOST + NC_REQ_TYPE_DATA_API_PATH) +print(" * Dataframe has been loaded from API path: " + NC_REQ_TYPE_DATA_API_PATH) # Loading the dataframe for the NCs and correspoding requests -print(" * Downloading data for dataframe") -query_string = "/reports?field=council_name&filter=created_date>=2016-01-01" -df1 = pd.read_json(API_HOST + query_string) +NC_REQ_COUNT_DATA_API_PATH = "/reports?field=council_name&filter=created_date>=2016-01-01" +print(" * Downloading data from API path: " + NC_REQ_COUNT_DATA_API_PATH) +df1 = pd.read_json(API_HOST + NC_REQ_COUNT_DATA_API_PATH) df1 = df1.groupby(['council_name'])['counts'].sum().sort_values().to_frame() +print(" * Dataframe has been loaded from API path: " + NC_REQ_COUNT_DATA_API_PATH) # Loading the data for the number of new requests -print(" * Downloading data for dataframe") -query_string = "/reports?field=created_year&filter=created_date>=2016-01-01" -df2 = pd.read_json(API_HOST + query_string) +NEW_REQ_COUNT_DATA_API_PATH = "/reports?field=created_year&filter=created_date>=2016-01-01" +print(" * Downloading data from API path: " + NEW_REQ_COUNT_DATA_API_PATH) +df2 = pd.read_json(API_HOST + NEW_REQ_COUNT_DATA_API_PATH) df2 = df2.groupby(['created_year'])['counts'].sum().to_frame() +print(" * Dataframe has been loaded from API path: " + NEW_REQ_COUNT_DATA_API_PATH) # Loading the count of each request types overall -print(" * Downloading data for dataframe") -query_string = "/reports?field=type_name&filter=created_date>=2016-01-01" -df3 = pd.read_json(API_HOST + query_string) +REQ_COUNT_DATA_API_PATH = "/reports?field=type_name&filter=created_date>=2016-01-01" +print(" * Downloading data from API path: " + REQ_COUNT_DATA_API_PATH) +df3 = pd.read_json(API_HOST + REQ_COUNT_DATA_API_PATH) df3 = df3.groupby(['type_name'], as_index=False)['counts'].sum() +print(" * Dataframe has been loaded from API path: " + REQ_COUNT_DATA_API_PATH) # Loading the total number of request source -print(" * Downloading data for dataframe") -query_string = "/reports?field=source_name&filter=created_date>=2016-01-01" -df6 = pd.read_json(API_HOST + query_string) +REQ_SOURCE_COUNT_DATA_API_PATH = "/reports?field=source_name&filter=created_date>=2016-01-01" +print(" * Downloading data from API path: " + REQ_SOURCE_COUNT_DATA_API_PATH) +df6 = pd.read_json(API_HOST + REQ_SOURCE_COUNT_DATA_API_PATH) reqSourceLab = df6.groupby(['source_name'])['counts'].sum() +print(" * Dataframe has been loaded from API path: " + REQ_SOURCE_COUNT_DATA_API_PATH) # Loading the number of Request Agencies -query_string = "/reports?field=agency_name&filter=created_date>=2016-01-01" -df5 = pd.read_json(API_HOST + query_string) +REQ_AGENCY_COUNT_DATA_API_PATH = "/reports?field=agency_name&filter=created_date>=2016-01-01" +print(" * Downloading data from API path: " + REQ_AGENCY_COUNT_DATA_API_PATH) +df5 = pd.read_json(API_HOST + REQ_AGENCY_COUNT_DATA_API_PATH) reqAgency = df5.groupby(['agency_name'])['counts'].sum() +print(" * Dataframe has been loaded from API path: " + REQ_AGENCY_COUNT_DATA_API_PATH) # Request Share by Agency Pie Chart print(" * Downloading data for dataframe") From ccd1d53e397711edd1da7d3e9754e3fb58f3d92c Mon Sep 17 00:00:00 2001 From: Yi Heng Joshua Wu Date: Tue, 26 Jul 2022 21:32:14 -0700 Subject: [PATCH 19/53] Remove duplicate code --- server/dash/dashboards/overviewComb.py | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/server/dash/dashboards/overviewComb.py b/server/dash/dashboards/overviewComb.py index 47a10274d..3f1d214c5 100644 --- a/server/dash/dashboards/overviewComb.py +++ b/server/dash/dashboards/overviewComb.py @@ -47,24 +47,21 @@ # Loading the number of Request Agencies REQ_AGENCY_COUNT_DATA_API_PATH = "/reports?field=agency_name&filter=created_date>=2016-01-01" print(" * Downloading data from API path: " + REQ_AGENCY_COUNT_DATA_API_PATH) -df5 = pd.read_json(API_HOST + REQ_AGENCY_COUNT_DATA_API_PATH) -reqAgency = df5.groupby(['agency_name'])['counts'].sum() +req_agency_count_df = pd.read_json(API_HOST + REQ_AGENCY_COUNT_DATA_API_PATH) +agency_count = req_agency_count_df.groupby(['agency_name'])['counts'].sum() print(" * Dataframe has been loaded from API path: " + REQ_AGENCY_COUNT_DATA_API_PATH) # Request Share by Agency Pie Chart -print(" * Downloading data for dataframe") -query_string = "/reports?field=agency_name&filter=created_date>=2016-01-01" -df5 = pd.read_json(API_HOST + query_string) -df5 = df5.groupby(['agency_name'])['counts'].sum().to_frame() -df5.sort_values('counts', ascending=False, inplace=True) -df5.loc['Others'] = df5[4:].sum() -df5.sort_values('counts', ascending=False, inplace=True) -df5 = df5[:5] -df5.index = df5.index.map(lambda x: '
'.join(textwrap.wrap(x, width=16))) +req_agency_count_df = agency_count.to_frame() +req_agency_count_df.sort_values('counts', ascending=False, inplace=True) +req_agency_count_df.loc['Others'] = req_agency_count_df[4:].sum() +req_agency_count_df.sort_values('counts', ascending=False, inplace=True) +req_agency_count_df = req_agency_count_df[:5] +req_agency_count_df.index = req_agency_count_df.index.map(lambda x: '
'.join(textwrap.wrap(x, width=16))) shareReqByAgencyPieChart = px.pie( - df5, - names=df5.index, + req_agency_count_df, + names=req_agency_count_df.index, values='counts', labels=LABELS, hole=.3, @@ -158,7 +155,7 @@ "text-align": 'center', "border": "0.5px black solid", 'width': '18vw', 'display': 'inline-block'}), html.Div([html.H2(reqSourceLab.shape[0]), html.Label("Request Source")], style={ "text-align": 'center', "border": "0.5px black solid", 'width': '18vw', 'display': 'inline-block'}), - html.Div([html.H2(reqAgency.shape[0]), html.Label("Request Agency")], style={ + html.Div([html.H2(agency_count.shape[0]), html.Label("Request Agency")], style={ "text-align": 'center', "border": "0.5px black solid", 'width': '18vw', 'display': 'inline-block'}) ], style={'display': 'flex', "justify-content": "space-between"}), From 17f637b927fa21dbbd62005ad656151ddc89cba1 Mon Sep 17 00:00:00 2001 From: Yi Heng Joshua Wu Date: Tue, 26 Jul 2022 21:35:13 -0700 Subject: [PATCH 20/53] Remove duplicate req source code --- server/dash/dashboards/overviewComb.py | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/server/dash/dashboards/overviewComb.py b/server/dash/dashboards/overviewComb.py index 3f1d214c5..eade94ed7 100644 --- a/server/dash/dashboards/overviewComb.py +++ b/server/dash/dashboards/overviewComb.py @@ -40,8 +40,8 @@ # Loading the total number of request source REQ_SOURCE_COUNT_DATA_API_PATH = "/reports?field=source_name&filter=created_date>=2016-01-01" print(" * Downloading data from API path: " + REQ_SOURCE_COUNT_DATA_API_PATH) -df6 = pd.read_json(API_HOST + REQ_SOURCE_COUNT_DATA_API_PATH) -reqSourceLab = df6.groupby(['source_name'])['counts'].sum() +req_source_count_df = pd.read_json(API_HOST + REQ_SOURCE_COUNT_DATA_API_PATH) +req_source_count = req_source_count_df.groupby(['source_name'])['counts'].sum() print(" * Dataframe has been loaded from API path: " + REQ_SOURCE_COUNT_DATA_API_PATH) # Loading the number of Request Agencies @@ -70,17 +70,15 @@ shareReqByAgencyPieChart.update_layout(margin=dict(l=100, r=100, b=100, t=100)) # Request Type by Source Bar Chart -print(" * Downloading data for dataframe") -query_string = "/reports?field=source_name&filter=created_date>=2016-01-01" -df6 = pd.read_json(API_HOST + query_string) -df6 = df6.groupby(['source_name'])['counts'].sum().to_frame() -df6.sort_values('counts', ascending=False, inplace=True) -df6.loc['Others'] = df6[4:].sum() -df6.sort_values('counts', ascending=False, inplace=True) -df6 = df6[:5] +req_source_count_df = req_source_count.to_frame() +req_source_count_df.sort_values('counts', ascending=False, inplace=True) +req_source_count_df.loc['Others'] = req_source_count_df[4:].sum() +req_source_count_df.sort_values('counts', ascending=False, inplace=True) +req_source_count_df = req_source_count_df[:5] + reqSourceBarchart = px.bar( - df6, - y=df6.index, + req_source_count_df, + y=req_source_count_df.index, x='counts', TITLE="Total Requests by Source", orientation='h' @@ -153,7 +151,7 @@ style={"text-align": 'center', "border": "0.5px black solid", 'width': '18vw', 'display': 'inline-block'}), html.Div([html.H2(df3.shape[0]), html.Label("Request Types")], style={ "text-align": 'center', "border": "0.5px black solid", 'width': '18vw', 'display': 'inline-block'}), - html.Div([html.H2(reqSourceLab.shape[0]), html.Label("Request Source")], style={ + html.Div([html.H2(req_source_count.shape[0]), html.Label("Request Source")], style={ "text-align": 'center', "border": "0.5px black solid", 'width': '18vw', 'display': 'inline-block'}), html.Div([html.H2(agency_count.shape[0]), html.Label("Request Agency")], style={ "text-align": 'center', "border": "0.5px black solid", 'width': '18vw', 'display': 'inline-block'}) From 62f741c236d04ec3a0c1930e34b62e49806d75f9 Mon Sep 17 00:00:00 2001 From: Yi Heng Joshua Wu Date: Tue, 26 Jul 2022 21:36:30 -0700 Subject: [PATCH 21/53] Update stas_df var name --- server/dash/dashboards/overviewComb.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/server/dash/dashboards/overviewComb.py b/server/dash/dashboards/overviewComb.py index eade94ed7..8b99d69de 100644 --- a/server/dash/dashboards/overviewComb.py +++ b/server/dash/dashboards/overviewComb.py @@ -85,21 +85,20 @@ ) # Median Request Days to Close Box Plot -stas_df = pd.read_json(API_HOST + '/types/stats') -stas_df = stas_df.sort_values('median', ascending=False) +stats_df = pd.read_json(API_HOST + '/types/stats') +stats_df = stats_df.sort_values('median', ascending=False) medDaysToCloseBoxPlot = go.Figure() medDaysToCloseBoxPlot.add_trace( go.Box( - y=stas_df.type_name, - q1=stas_df['q1'], - median=stas_df['median'], - q3=stas_df['q3'], + y=stats_df.type_name, + q1=stats_df['q1'], + median=stats_df['median'], + q3=stats_df['q3'], marker_color='#29404F', fillcolor='#E17C05', ) ) medDaysToCloseBoxPlot.update_xaxes( - dtick=5 ) medDaysToCloseBoxPlot.update_layout( From 3a3a6bcdd118c7cecc0b2626fc6819372cf151d9 Mon Sep 17 00:00:00 2001 From: Yi Heng Joshua Wu Date: Tue, 26 Jul 2022 21:39:07 -0700 Subject: [PATCH 22/53] Update req_count var name --- server/dash/dashboards/overviewComb.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/server/dash/dashboards/overviewComb.py b/server/dash/dashboards/overviewComb.py index 8b99d69de..11307116c 100644 --- a/server/dash/dashboards/overviewComb.py +++ b/server/dash/dashboards/overviewComb.py @@ -19,8 +19,8 @@ # Loading the dataframe for the NCs and correspoding requests NC_REQ_COUNT_DATA_API_PATH = "/reports?field=council_name&filter=created_date>=2016-01-01" print(" * Downloading data from API path: " + NC_REQ_COUNT_DATA_API_PATH) -df1 = pd.read_json(API_HOST + NC_REQ_COUNT_DATA_API_PATH) -df1 = df1.groupby(['council_name'])['counts'].sum().sort_values().to_frame() +nc_req_count_df = pd.read_json(API_HOST + NC_REQ_COUNT_DATA_API_PATH) +nc_req_count_df = nc_req_count_df.groupby(['council_name'])['counts'].sum().sort_values().to_frame() print(" * Dataframe has been loaded from API path: " + NC_REQ_COUNT_DATA_API_PATH) # Loading the data for the number of new requests @@ -123,13 +123,9 @@ numReqByDayOfWeekBarChart.update_xaxes(categoryorder='array', categoryarray= ["Sunday", "Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday"]) # Total Request by NC -print(" * Downloading data for dataframe") -query_string = "/reports?field=council_name&filter=created_date>=2016-01-01" -df1 = pd.read_json(API_HOST + query_string) -df1 = df1.groupby(['council_name'])['counts'].sum().sort_values().to_frame() reqByNcBarChart = px.bar( - df1, - x=df1.index, + nc_req_count_df, + x=nc_req_count_df.index, y='counts', labels=LABELS, TITLE="Total Requests by Neighborhood Councils", @@ -146,7 +142,7 @@ html.Div([ html.Div([html.H2(f"{df2['counts'].sum():,}"), html.Label( "Total Requests")], style={"text-align": 'center', "border": "0.5px black solid", 'width': '18vw', 'display': 'inline-block'}), - html.Div([html.H2(df1.shape[0] - 1), html.Label("Neighborhoods")], + html.Div([html.H2(nc_req_count_df.shape[0] - 1), html.Label("Neighborhoods")], style={"text-align": 'center', "border": "0.5px black solid", 'width': '18vw', 'display': 'inline-block'}), html.Div([html.H2(df3.shape[0]), html.Label("Request Types")], style={ "text-align": 'center', "border": "0.5px black solid", 'width': '18vw', 'display': 'inline-block'}), From 154dabdfeecd4b5108b2a8077b69e56e80ffc7f9 Mon Sep 17 00:00:00 2001 From: Yi Heng Joshua Wu Date: Tue, 26 Jul 2022 21:43:34 -0700 Subject: [PATCH 23/53] update date range api path var --- server/dash/dashboards/overviewComb.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/server/dash/dashboards/overviewComb.py b/server/dash/dashboards/overviewComb.py index 11307116c..57fd5223f 100644 --- a/server/dash/dashboards/overviewComb.py +++ b/server/dash/dashboards/overviewComb.py @@ -57,7 +57,8 @@ req_agency_count_df.loc['Others'] = req_agency_count_df[4:].sum() req_agency_count_df.sort_values('counts', ascending=False, inplace=True) req_agency_count_df = req_agency_count_df[:5] -req_agency_count_df.index = req_agency_count_df.index.map(lambda x: '
'.join(textwrap.wrap(x, width=16))) +req_agency_count_df.index = req_agency_count_df.index.map( + lambda x: '
'.join(textwrap.wrap(x, width=16))) shareReqByAgencyPieChart = px.pie( req_agency_count_df, @@ -87,6 +88,7 @@ # Median Request Days to Close Box Plot stats_df = pd.read_json(API_HOST + '/types/stats') stats_df = stats_df.sort_values('median', ascending=False) + medDaysToCloseBoxPlot = go.Figure() medDaysToCloseBoxPlot.add_trace( go.Box( @@ -108,19 +110,22 @@ # Day of Week Bar Chart: start_date = datetime.date.today() - datetime.timedelta(days=30) end_date = datetime.date.today() - datetime.timedelta(days=1) -query_string = f"/reports?filter=created_date>={start_date}&filter=created_date<={end_date}" # noqa -print(" * Downloading data for dataframe") -df = pd.read_json(API_HOST + query_string) +DATE_RANGE_REQ_DATA_API_PATH = f"/reports?filter=created_date>={start_date}&filter=created_date<={end_date}" # noqa +print(" * Downloading data for dataframe from API path: " + DATE_RANGE_REQ_DATA_API_PATH) +df = pd.read_json(API_HOST + DATE_RANGE_REQ_DATA_API_PATH) +print(" * Dataframe has been loaded from API path: " + DATE_RANGE_REQ_DATA_API_PATH) df['created_date'] = pd.to_datetime(df['created_date']) dow_df = df.groupby(['created_date']).agg('sum').reset_index() dow_df['day_of_week'] = dow_df['created_date'].dt.day_name() + numReqByDayOfWeekBarChart = px.bar( dow_df, x="day_of_week", y="counts", labels=LABELS, ) -numReqByDayOfWeekBarChart.update_xaxes(categoryorder='array', categoryarray= ["Sunday", "Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday"]) +numReqByDayOfWeekBarChart.update_xaxes(categoryorder='array', categoryarray=[ + "Sunday", "Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday"]) # Total Request by NC reqByNcBarChart = px.bar( From d109f28f635d044af33d25882ab396902f5d4af5 Mon Sep 17 00:00:00 2001 From: Yi Heng Joshua Wu Date: Tue, 26 Jul 2022 21:55:05 -0700 Subject: [PATCH 24/53] Update var to snake case --- server/dash/dashboards/overviewComb.py | 44 +++++++++++++------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/server/dash/dashboards/overviewComb.py b/server/dash/dashboards/overviewComb.py index 57fd5223f..62192b710 100644 --- a/server/dash/dashboards/overviewComb.py +++ b/server/dash/dashboards/overviewComb.py @@ -26,15 +26,15 @@ # Loading the data for the number of new requests NEW_REQ_COUNT_DATA_API_PATH = "/reports?field=created_year&filter=created_date>=2016-01-01" print(" * Downloading data from API path: " + NEW_REQ_COUNT_DATA_API_PATH) -df2 = pd.read_json(API_HOST + NEW_REQ_COUNT_DATA_API_PATH) -df2 = df2.groupby(['created_year'])['counts'].sum().to_frame() +new_req_count_df = pd.read_json(API_HOST + NEW_REQ_COUNT_DATA_API_PATH) +new_req_count_df = new_req_count_df.groupby(['created_year'])['counts'].sum().to_frame() print(" * Dataframe has been loaded from API path: " + NEW_REQ_COUNT_DATA_API_PATH) # Loading the count of each request types overall REQ_COUNT_DATA_API_PATH = "/reports?field=type_name&filter=created_date>=2016-01-01" print(" * Downloading data from API path: " + REQ_COUNT_DATA_API_PATH) -df3 = pd.read_json(API_HOST + REQ_COUNT_DATA_API_PATH) -df3 = df3.groupby(['type_name'], as_index=False)['counts'].sum() +req_count_df = pd.read_json(API_HOST + REQ_COUNT_DATA_API_PATH) +req_count = req_count_df.groupby(['type_name'], as_index=False)['counts'].sum() print(" * Dataframe has been loaded from API path: " + REQ_COUNT_DATA_API_PATH) # Loading the total number of request source @@ -60,7 +60,7 @@ req_agency_count_df.index = req_agency_count_df.index.map( lambda x: '
'.join(textwrap.wrap(x, width=16))) -shareReqByAgencyPieChart = px.pie( +req_share_by_agency_pie_chart = px.pie( req_agency_count_df, names=req_agency_count_df.index, values='counts', @@ -68,7 +68,7 @@ hole=.3, TITLE="Total Requests by Agency", ) -shareReqByAgencyPieChart.update_layout(margin=dict(l=100, r=100, b=100, t=100)) +req_share_by_agency_pie_chart.update_layout(margin=dict(l=100, r=100, b=100, t=100)) # Request Type by Source Bar Chart req_source_count_df = req_source_count.to_frame() @@ -77,7 +77,7 @@ req_source_count_df.sort_values('counts', ascending=False, inplace=True) req_source_count_df = req_source_count_df[:5] -reqSourceBarchart = px.bar( +req_source_bar_chart = px.bar( req_source_count_df, y=req_source_count_df.index, x='counts', @@ -89,8 +89,8 @@ stats_df = pd.read_json(API_HOST + '/types/stats') stats_df = stats_df.sort_values('median', ascending=False) -medDaysToCloseBoxPlot = go.Figure() -medDaysToCloseBoxPlot.add_trace( +med_days_to_close_box_plot = go.Figure() +med_days_to_close_box_plot.add_trace( go.Box( y=stats_df.type_name, q1=stats_df['q1'], @@ -100,10 +100,10 @@ fillcolor='#E17C05', ) ) -medDaysToCloseBoxPlot.update_xaxes( +med_days_to_close_box_plot.update_xaxes( dtick=5 ) -medDaysToCloseBoxPlot.update_layout( +med_days_to_close_box_plot.update_layout( TITLE="Total Median Days to Close by Type", ) @@ -118,24 +118,24 @@ dow_df = df.groupby(['created_date']).agg('sum').reset_index() dow_df['day_of_week'] = dow_df['created_date'].dt.day_name() -numReqByDayOfWeekBarChart = px.bar( +num_req_by_day_bar_chart = px.bar( dow_df, x="day_of_week", y="counts", labels=LABELS, ) -numReqByDayOfWeekBarChart.update_xaxes(categoryorder='array', categoryarray=[ +num_req_by_day_bar_chart.update_xaxes(categoryorder='array', categoryarray=[ "Sunday", "Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday"]) # Total Request by NC -reqByNcBarChart = px.bar( +req_by_nc_bar_chart = px.bar( nc_req_count_df, x=nc_req_count_df.index, y='counts', labels=LABELS, TITLE="Total Requests by Neighborhood Councils", ) -reqByNcBarChart.update_layout(font=dict(size=12)) +req_by_nc_bar_chart.update_layout(font=dict(size=12)) # LAYOUT layout = html.Div([ @@ -145,11 +145,11 @@ style={'font-size': '18px', 'font-style': 'italic'}), html.Div([ - html.Div([html.H2(f"{df2['counts'].sum():,}"), html.Label( + html.Div([html.H2(f"{new_req_count_df['counts'].sum():,}"), html.Label( "Total Requests")], style={"text-align": 'center', "border": "0.5px black solid", 'width': '18vw', 'display': 'inline-block'}), html.Div([html.H2(nc_req_count_df.shape[0] - 1), html.Label("Neighborhoods")], style={"text-align": 'center', "border": "0.5px black solid", 'width': '18vw', 'display': 'inline-block'}), - html.Div([html.H2(df3.shape[0]), html.Label("Request Types")], style={ + html.Div([html.H2(req_count_df.shape[0]), html.Label("Request Types")], style={ "text-align": 'center', "border": "0.5px black solid", 'width': '18vw', 'display': 'inline-block'}), html.Div([html.H2(req_source_count.shape[0]), html.Label("Request Source")], style={ "text-align": 'center', "border": "0.5px black solid", 'width': '18vw', 'display': 'inline-block'}), @@ -159,20 +159,20 @@ html.Div(html.Br(), style={"height": "3vh"}), html.Div([ - html.Div(dcc.Graph(id='medDaysToCloseBoxPlot', figure=medDaysToCloseBoxPlot, responsive=True, style={ + html.Div(dcc.Graph(id='med_days_to_close_box_plot', figure=med_days_to_close_box_plot, responsive=True, style={ "width": "60vw", "height": "60vh"}), style={"border": "0.5px black solid"}), - html.Div(dcc.Graph(id='shareReqByAgencyPieChart', figure=shareReqByAgencyPieChart, className="half-graph", + html.Div(dcc.Graph(id='req_share_by_agency_pie_chart', figure=req_share_by_agency_pie_chart, className="half-graph", responsive=True, style={"width": "35vw", "height": "60vh"}), style={"border": "0.5px black solid"}) ], className="graph-row", style={'display': 'flex', "justify-content": "space-between"}), html.Div(html.Br(), style={"height": "2vh"}), html.H1(TITLE + " Pt. 2"), html.Div([ - html.Div(dcc.Graph(id='numReqByDayOfWeekBarChart', figure=numReqByDayOfWeekBarChart, className="half-graph", style={"width": "48vw", "height": "40vh"}), style={"border": "0.5px black solid"}), # noqa - html.Div(dcc.Graph(id='reqSourceBarchart', figure=reqSourceBarchart, className="half-graph", + html.Div(dcc.Graph(id='num_req_by_day_bar_chart', figure=num_req_by_day_bar_chart, className="half-graph", style={"width": "48vw", "height": "40vh"}), style={"border": "0.5px black solid"}), # noqa + html.Div(dcc.Graph(id='req_source_bar_chart', figure=req_source_bar_chart, className="half-graph", responsive=True, style={"width": "48vw", "height": "40vh"}), style={"border": "0.5px black solid"}) ], className="graph-row", style={'display': 'flex', "justify-content": "space-between"}), html.Div(html.Br(), style={"height": "2vh"}), - html.Div(dcc.Graph(id='reqByNcBarChart', figure=reqByNcBarChart, responsive=True, + html.Div(dcc.Graph(id='req_by_nc_bar_chart', figure=req_by_nc_bar_chart, responsive=True, style={"height": "45vh"}), style={"border": "0.5px black solid"}) ]) From c3e92bade90c2eafab389618f2d5769c9344ef1a Mon Sep 17 00:00:00 2001 From: Yi Heng Joshua Wu Date: Tue, 26 Jul 2022 22:00:08 -0700 Subject: [PATCH 25/53] Update indicator style literals --- server/dash/dashboards/overviewComb.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/server/dash/dashboards/overviewComb.py b/server/dash/dashboards/overviewComb.py index 62192b710..a17b09ddd 100644 --- a/server/dash/dashboards/overviewComb.py +++ b/server/dash/dashboards/overviewComb.py @@ -125,7 +125,7 @@ labels=LABELS, ) num_req_by_day_bar_chart.update_xaxes(categoryorder='array', categoryarray=[ - "Sunday", "Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday"]) + "Sunday", "Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday"]) # Total Request by NC req_by_nc_bar_chart = px.bar( @@ -137,6 +137,10 @@ ) req_by_nc_bar_chart.update_layout(font=dict(size=12)) +# LAYOUT VARIABLES +INDICATOR_CARD_STYLE = {"text-align": 'center', + "border": "0.5px black solid", 'width': '18vw', 'display': 'inline-block'} + # LAYOUT layout = html.Div([ # Page 2 with Cards + Stuf @@ -144,17 +148,18 @@ html.P("The figures below represent the total number of 311 requests made across LA County from 2016-2021. In 2020, we saw an all-time high with more than 1.4 million requests.", style={'font-size': '18px', 'font-style': 'italic'}), + html.Div([ html.Div([html.H2(f"{new_req_count_df['counts'].sum():,}"), html.Label( - "Total Requests")], style={"text-align": 'center', "border": "0.5px black solid", 'width': '18vw', 'display': 'inline-block'}), + "Total Requests")], style=INDICATOR_CARD_STYLE), html.Div([html.H2(nc_req_count_df.shape[0] - 1), html.Label("Neighborhoods")], - style={"text-align": 'center', "border": "0.5px black solid", 'width': '18vw', 'display': 'inline-block'}), - html.Div([html.H2(req_count_df.shape[0]), html.Label("Request Types")], style={ - "text-align": 'center', "border": "0.5px black solid", 'width': '18vw', 'display': 'inline-block'}), - html.Div([html.H2(req_source_count.shape[0]), html.Label("Request Source")], style={ - "text-align": 'center', "border": "0.5px black solid", 'width': '18vw', 'display': 'inline-block'}), - html.Div([html.H2(agency_count.shape[0]), html.Label("Request Agency")], style={ - "text-align": 'center', "border": "0.5px black solid", 'width': '18vw', 'display': 'inline-block'}) + style=INDICATOR_CARD_STYLE), + html.Div([html.H2(req_count_df.shape[0]), html.Label( + "Request Types")], style=INDICATOR_CARD_STYLE), + html.Div([html.H2(req_source_count.shape[0]), html.Label( + "Request Source")], style=INDICATOR_CARD_STYLE), + html.Div([html.H2(agency_count.shape[0]), html.Label( + "Request Agency")], style=INDICATOR_CARD_STYLE) ], style={'display': 'flex', "justify-content": "space-between"}), html.Div(html.Br(), style={"height": "3vh"}), From 10483dfe23231bb49fa965a60c5a0339f40bf560 Mon Sep 17 00:00:00 2001 From: Yi Heng Joshua Wu Date: Tue, 26 Jul 2022 22:14:50 -0700 Subject: [PATCH 26/53] Border style --- server/dash/dashboards/overviewComb.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/server/dash/dashboards/overviewComb.py b/server/dash/dashboards/overviewComb.py index a17b09ddd..b468f6629 100644 --- a/server/dash/dashboards/overviewComb.py +++ b/server/dash/dashboards/overviewComb.py @@ -140,6 +140,7 @@ # LAYOUT VARIABLES INDICATOR_CARD_STYLE = {"text-align": 'center', "border": "0.5px black solid", 'width': '18vw', 'display': 'inline-block'} +BORDER_STYLE = {"border": "0.5px black solid"} # LAYOUT layout = html.Div([ @@ -165,19 +166,19 @@ html.Div(html.Br(), style={"height": "3vh"}), html.Div([ html.Div(dcc.Graph(id='med_days_to_close_box_plot', figure=med_days_to_close_box_plot, responsive=True, style={ - "width": "60vw", "height": "60vh"}), style={"border": "0.5px black solid"}), + "width": "60vw", "height": "60vh"}), style=BORDER_STYLE), html.Div(dcc.Graph(id='req_share_by_agency_pie_chart', figure=req_share_by_agency_pie_chart, className="half-graph", - responsive=True, style={"width": "35vw", "height": "60vh"}), style={"border": "0.5px black solid"}) + responsive=True, style={"width": "35vw", "height": "60vh"}), style=BORDER_STYLE) ], className="graph-row", style={'display': 'flex', "justify-content": "space-between"}), html.Div(html.Br(), style={"height": "2vh"}), html.H1(TITLE + " Pt. 2"), html.Div([ - html.Div(dcc.Graph(id='num_req_by_day_bar_chart', figure=num_req_by_day_bar_chart, className="half-graph", style={"width": "48vw", "height": "40vh"}), style={"border": "0.5px black solid"}), # noqa + html.Div(dcc.Graph(id='num_req_by_day_bar_chart', figure=num_req_by_day_bar_chart, className="half-graph", style={"width": "48vw", "height": "40vh"}), style=BORDER_STYLE), # noqa html.Div(dcc.Graph(id='req_source_bar_chart', figure=req_source_bar_chart, className="half-graph", - responsive=True, style={"width": "48vw", "height": "40vh"}), style={"border": "0.5px black solid"}) + responsive=True, style={"width": "48vw", "height": "40vh"}), style=BORDER_STYLE) ], className="graph-row", style={'display': 'flex', "justify-content": "space-between"}), html.Div(html.Br(), style={"height": "2vh"}), html.Div(dcc.Graph(id='req_by_nc_bar_chart', figure=req_by_nc_bar_chart, responsive=True, - style={"height": "45vh"}), style={"border": "0.5px black solid"}) + style={"height": "45vh"}), style=BORDER_STYLE) ]) From 090f732d9873de4eb04f9ea68531e88f1120def5 Mon Sep 17 00:00:00 2001 From: Yi Heng Joshua Wu Date: Tue, 26 Jul 2022 22:16:02 -0700 Subject: [PATCH 27/53] Equal space box style --- server/dash/dashboards/overviewComb.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/server/dash/dashboards/overviewComb.py b/server/dash/dashboards/overviewComb.py index b468f6629..8e6850d35 100644 --- a/server/dash/dashboards/overviewComb.py +++ b/server/dash/dashboards/overviewComb.py @@ -140,7 +140,8 @@ # LAYOUT VARIABLES INDICATOR_CARD_STYLE = {"text-align": 'center', "border": "0.5px black solid", 'width': '18vw', 'display': 'inline-block'} -BORDER_STYLE = {"border": "0.5px black solid"} +BORDER_STYLE = {"border": "0.5px black solid"} +EQUAL_SPACE_BOX_STYLE = {'display': 'flex', "justify-content": "space-between"} # LAYOUT layout = html.Div([ @@ -161,7 +162,7 @@ "Request Source")], style=INDICATOR_CARD_STYLE), html.Div([html.H2(agency_count.shape[0]), html.Label( "Request Agency")], style=INDICATOR_CARD_STYLE) - ], style={'display': 'flex', "justify-content": "space-between"}), + ], style=EQUAL_SPACE_BOX_STYLE), html.Div(html.Br(), style={"height": "3vh"}), html.Div([ @@ -169,14 +170,14 @@ "width": "60vw", "height": "60vh"}), style=BORDER_STYLE), html.Div(dcc.Graph(id='req_share_by_agency_pie_chart', figure=req_share_by_agency_pie_chart, className="half-graph", responsive=True, style={"width": "35vw", "height": "60vh"}), style=BORDER_STYLE) - ], className="graph-row", style={'display': 'flex', "justify-content": "space-between"}), + ], className="graph-row", style=EQUAL_SPACE_BOX_STYLE), html.Div(html.Br(), style={"height": "2vh"}), html.H1(TITLE + " Pt. 2"), html.Div([ html.Div(dcc.Graph(id='num_req_by_day_bar_chart', figure=num_req_by_day_bar_chart, className="half-graph", style={"width": "48vw", "height": "40vh"}), style=BORDER_STYLE), # noqa html.Div(dcc.Graph(id='req_source_bar_chart', figure=req_source_bar_chart, className="half-graph", responsive=True, style={"width": "48vw", "height": "40vh"}), style=BORDER_STYLE) - ], className="graph-row", style={'display': 'flex', "justify-content": "space-between"}), + ], className="graph-row", style=EQUAL_SPACE_BOX_STYLE), html.Div(html.Br(), style={"height": "2vh"}), html.Div(dcc.Graph(id='req_by_nc_bar_chart', figure=req_by_nc_bar_chart, responsive=True, style={"height": "45vh"}), style=BORDER_STYLE) From 17d141d643ea5a12893bdfd09a34becfed57571a Mon Sep 17 00:00:00 2001 From: Yi Heng Joshua Wu Date: Tue, 26 Jul 2022 22:19:32 -0700 Subject: [PATCH 28/53] Dashboard outline --- server/dash/dashboards/overviewComb.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/server/dash/dashboards/overviewComb.py b/server/dash/dashboards/overviewComb.py index 8e6850d35..ce207ee1a 100644 --- a/server/dash/dashboards/overviewComb.py +++ b/server/dash/dashboards/overviewComb.py @@ -141,15 +141,15 @@ INDICATOR_CARD_STYLE = {"text-align": 'center', "border": "0.5px black solid", 'width': '18vw', 'display': 'inline-block'} BORDER_STYLE = {"border": "0.5px black solid"} -EQUAL_SPACE_BOX_STYLE = {'display': 'flex', "justify-content": "space-between"} +EQUAL_SPACE_BOX_STYLE = {'display': 'flex', "justify-content": "space-between"} +DASHBOARD_OUTLINE = "The figures below represent the total number of 311 requests made across LA County from 2016-2021. In 2020, we saw an all-time high with more than 1.4 million requests." # LAYOUT layout = html.Div([ - # Page 2 with Cards + Stuf - html.H1(TITLE + " Pt. 1"), - html.P("The figures below represent the total number of 311 requests made across LA County from 2016-2021. In 2020, we saw an all-time high with more than 1.4 million requests.", - style={'font-size': '18px', 'font-style': 'italic'}), + # Part 1 of dashboard + html.H1(TITLE + " Pt. 1"), + html.P(DASHBOARD_OUTLINE, style={'font-size': '18px', 'font-style': 'italic'}), html.Div([ html.Div([html.H2(f"{new_req_count_df['counts'].sum():,}"), html.Label( @@ -165,20 +165,27 @@ ], style=EQUAL_SPACE_BOX_STYLE), html.Div(html.Br(), style={"height": "3vh"}), + html.Div([ html.Div(dcc.Graph(id='med_days_to_close_box_plot', figure=med_days_to_close_box_plot, responsive=True, style={ "width": "60vw", "height": "60vh"}), style=BORDER_STYLE), html.Div(dcc.Graph(id='req_share_by_agency_pie_chart', figure=req_share_by_agency_pie_chart, className="half-graph", responsive=True, style={"width": "35vw", "height": "60vh"}), style=BORDER_STYLE) ], className="graph-row", style=EQUAL_SPACE_BOX_STYLE), + html.Div(html.Br(), style={"height": "2vh"}), + + # Part 2 of dashboard html.H1(TITLE + " Pt. 2"), html.Div([ - html.Div(dcc.Graph(id='num_req_by_day_bar_chart', figure=num_req_by_day_bar_chart, className="half-graph", style={"width": "48vw", "height": "40vh"}), style=BORDER_STYLE), # noqa + html.Div(dcc.Graph(id='num_req_by_day_bar_chart', figure=num_req_by_day_bar_chart, className="half-graph", + style={"width": "48vw", "height": "40vh"}), style=BORDER_STYLE), # noqa html.Div(dcc.Graph(id='req_source_bar_chart', figure=req_source_bar_chart, className="half-graph", responsive=True, style={"width": "48vw", "height": "40vh"}), style=BORDER_STYLE) ], className="graph-row", style=EQUAL_SPACE_BOX_STYLE), + html.Div(html.Br(), style={"height": "2vh"}), + html.Div(dcc.Graph(id='req_by_nc_bar_chart', figure=req_by_nc_bar_chart, responsive=True, style={"height": "45vh"}), style=BORDER_STYLE) From 7f3d39ebb20625ac8843820f6ee277a4647fec62 Mon Sep 17 00:00:00 2001 From: Yi Heng Joshua Wu Date: Tue, 26 Jul 2022 22:21:35 -0700 Subject: [PATCH 29/53] two chart_style --- server/dash/dashboards/overviewComb.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/server/dash/dashboards/overviewComb.py b/server/dash/dashboards/overviewComb.py index ce207ee1a..69bf2de59 100644 --- a/server/dash/dashboards/overviewComb.py +++ b/server/dash/dashboards/overviewComb.py @@ -142,8 +142,10 @@ "border": "0.5px black solid", 'width': '18vw', 'display': 'inline-block'} BORDER_STYLE = {"border": "0.5px black solid"} EQUAL_SPACE_BOX_STYLE = {'display': 'flex', "justify-content": "space-between"} +TWO_CHART_STYLE = {"width": "48vw", "height": "40vh"} DASHBOARD_OUTLINE = "The figures below represent the total number of 311 requests made across LA County from 2016-2021. In 2020, we saw an all-time high with more than 1.4 million requests." + # LAYOUT layout = html.Div([ @@ -179,9 +181,9 @@ html.H1(TITLE + " Pt. 2"), html.Div([ html.Div(dcc.Graph(id='num_req_by_day_bar_chart', figure=num_req_by_day_bar_chart, className="half-graph", - style={"width": "48vw", "height": "40vh"}), style=BORDER_STYLE), # noqa + style=TWO_CHART_STYLE), style=BORDER_STYLE), # noqa html.Div(dcc.Graph(id='req_source_bar_chart', figure=req_source_bar_chart, className="half-graph", - responsive=True, style={"width": "48vw", "height": "40vh"}), style=BORDER_STYLE) + responsive=True, style=TWO_CHART_STYLE), style=BORDER_STYLE) ], className="graph-row", style=EQUAL_SPACE_BOX_STYLE), html.Div(html.Br(), style={"height": "2vh"}), From 8d4b82fca4ee738cd635d2973418bd179366f06d Mon Sep 17 00:00:00 2001 From: Yi Heng Joshua Wu Date: Tue, 26 Jul 2022 22:31:32 -0700 Subject: [PATCH 30/53] Remove unused code --- server/dash/dashboards/overviewComb.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/server/dash/dashboards/overviewComb.py b/server/dash/dashboards/overviewComb.py index 69bf2de59..439be1bdb 100644 --- a/server/dash/dashboards/overviewComb.py +++ b/server/dash/dashboards/overviewComb.py @@ -11,10 +11,6 @@ TITLE = "OVERVIEW COMBINED DASHBOARD" # DATA -NC_REQ_TYPE_DATA_API_PATH = '/reports?field=type_name&field=council_name&field=created_date' -print(" * Downloading data from API path: " + NC_REQ_TYPE_DATA_API_PATH) -df = pd.read_json(API_HOST + NC_REQ_TYPE_DATA_API_PATH) -print(" * Dataframe has been loaded from API path: " + NC_REQ_TYPE_DATA_API_PATH) # Loading the dataframe for the NCs and correspoding requests NC_REQ_COUNT_DATA_API_PATH = "/reports?field=council_name&filter=created_date>=2016-01-01" @@ -112,14 +108,14 @@ end_date = datetime.date.today() - datetime.timedelta(days=1) DATE_RANGE_REQ_DATA_API_PATH = f"/reports?filter=created_date>={start_date}&filter=created_date<={end_date}" # noqa print(" * Downloading data for dataframe from API path: " + DATE_RANGE_REQ_DATA_API_PATH) -df = pd.read_json(API_HOST + DATE_RANGE_REQ_DATA_API_PATH) +date_range_req_df = pd.read_json(API_HOST + DATE_RANGE_REQ_DATA_API_PATH) print(" * Dataframe has been loaded from API path: " + DATE_RANGE_REQ_DATA_API_PATH) -df['created_date'] = pd.to_datetime(df['created_date']) -dow_df = df.groupby(['created_date']).agg('sum').reset_index() -dow_df['day_of_week'] = dow_df['created_date'].dt.day_name() +date_range_req_df['created_date'] = pd.to_datetime(date_range_req_df['created_date']) +day_of_week_df = date_range_req_df.groupby(['created_date']).agg('sum').reset_index() +day_of_week_df['day_of_week'] = day_of_week_df['created_date'].dt.day_name() num_req_by_day_bar_chart = px.bar( - dow_df, + day_of_week_df, x="day_of_week", y="counts", labels=LABELS, @@ -145,7 +141,6 @@ TWO_CHART_STYLE = {"width": "48vw", "height": "40vh"} DASHBOARD_OUTLINE = "The figures below represent the total number of 311 requests made across LA County from 2016-2021. In 2020, we saw an all-time high with more than 1.4 million requests." - # LAYOUT layout = html.Div([ From 3de735f83ee74ad805390004010c34550015abce Mon Sep 17 00:00:00 2001 From: Yi Heng Joshua Wu Date: Tue, 26 Jul 2022 23:14:31 -0700 Subject: [PATCH 31/53] Update api path with urlencode --- server/dash/dashboards/overviewComb.py | 26 ++++++++++++++++++-------- server/dash/requirements.txt | 3 ++- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/server/dash/dashboards/overviewComb.py b/server/dash/dashboards/overviewComb.py index 439be1bdb..d75983c76 100644 --- a/server/dash/dashboards/overviewComb.py +++ b/server/dash/dashboards/overviewComb.py @@ -1,5 +1,6 @@ import textwrap import datetime +import urllib.parse import pandas as pd import plotly.express as px import plotly.graph_objects as go @@ -9,39 +10,44 @@ from design import LABELS TITLE = "OVERVIEW COMBINED DASHBOARD" - +REPORT_API_PATH_ROOT = "/reports?" # DATA # Loading the dataframe for the NCs and correspoding requests -NC_REQ_COUNT_DATA_API_PATH = "/reports?field=council_name&filter=created_date>=2016-01-01" +nc_req_count_api_params = {'field':'council_name', 'filter':'created_date>=2016-01-01'} +NC_REQ_COUNT_DATA_API_PATH = REPORT_API_PATH_ROOT + urllib.parse.urlencode(nc_req_count_api_params) print(" * Downloading data from API path: " + NC_REQ_COUNT_DATA_API_PATH) nc_req_count_df = pd.read_json(API_HOST + NC_REQ_COUNT_DATA_API_PATH) nc_req_count_df = nc_req_count_df.groupby(['council_name'])['counts'].sum().sort_values().to_frame() print(" * Dataframe has been loaded from API path: " + NC_REQ_COUNT_DATA_API_PATH) # Loading the data for the number of new requests -NEW_REQ_COUNT_DATA_API_PATH = "/reports?field=created_year&filter=created_date>=2016-01-01" +new_req_count_api_params = {'field':'created_year', 'filter':'created_date>=2016-01-01'} +NEW_REQ_COUNT_DATA_API_PATH = REPORT_API_PATH_ROOT + urllib.parse.urlencode(new_req_count_api_params) print(" * Downloading data from API path: " + NEW_REQ_COUNT_DATA_API_PATH) new_req_count_df = pd.read_json(API_HOST + NEW_REQ_COUNT_DATA_API_PATH) new_req_count_df = new_req_count_df.groupby(['created_year'])['counts'].sum().to_frame() print(" * Dataframe has been loaded from API path: " + NEW_REQ_COUNT_DATA_API_PATH) # Loading the count of each request types overall -REQ_COUNT_DATA_API_PATH = "/reports?field=type_name&filter=created_date>=2016-01-01" +req_count_api_params = {'field':'type_name', 'filter':'created_date>=2016-01-01'} +REQ_COUNT_DATA_API_PATH = REPORT_API_PATH_ROOT + urllib.parse.urlencode(req_count_api_params) print(" * Downloading data from API path: " + REQ_COUNT_DATA_API_PATH) req_count_df = pd.read_json(API_HOST + REQ_COUNT_DATA_API_PATH) req_count = req_count_df.groupby(['type_name'], as_index=False)['counts'].sum() print(" * Dataframe has been loaded from API path: " + REQ_COUNT_DATA_API_PATH) # Loading the total number of request source -REQ_SOURCE_COUNT_DATA_API_PATH = "/reports?field=source_name&filter=created_date>=2016-01-01" +req_source_count_api_params = {'field':'source_name', 'filter':'created_date>=2016-01-01'} +REQ_SOURCE_COUNT_DATA_API_PATH = REPORT_API_PATH_ROOT + urllib.parse.urlencode(req_source_count_api_params) print(" * Downloading data from API path: " + REQ_SOURCE_COUNT_DATA_API_PATH) req_source_count_df = pd.read_json(API_HOST + REQ_SOURCE_COUNT_DATA_API_PATH) req_source_count = req_source_count_df.groupby(['source_name'])['counts'].sum() print(" * Dataframe has been loaded from API path: " + REQ_SOURCE_COUNT_DATA_API_PATH) # Loading the number of Request Agencies -REQ_AGENCY_COUNT_DATA_API_PATH = "/reports?field=agency_name&filter=created_date>=2016-01-01" +req_agency_count_api_params = {'field':'agency_name', 'filter':'created_date>=2016-01-01'} +REQ_AGENCY_COUNT_DATA_API_PATH = REPORT_API_PATH_ROOT + urllib.parse.urlencode(req_agency_count_api_params) print(" * Downloading data from API path: " + REQ_AGENCY_COUNT_DATA_API_PATH) req_agency_count_df = pd.read_json(API_HOST + REQ_AGENCY_COUNT_DATA_API_PATH) agency_count = req_agency_count_df.groupby(['agency_name'])['counts'].sum() @@ -82,8 +88,11 @@ ) # Median Request Days to Close Box Plot -stats_df = pd.read_json(API_HOST + '/types/stats') +REQ_TYPE_STATS_API_PATH = '/types/stats' +print(" * Downloading data from API path: " + REQ_TYPE_STATS_API_PATH) +stats_df = pd.read_json(API_HOST + REQ_TYPE_STATS_API_PATH) stats_df = stats_df.sort_values('median', ascending=False) +print(" * Dataframe has been loaded from API path: " + REQ_TYPE_STATS_API_PATH) med_days_to_close_box_plot = go.Figure() med_days_to_close_box_plot.add_trace( @@ -106,7 +115,8 @@ # Day of Week Bar Chart: start_date = datetime.date.today() - datetime.timedelta(days=30) end_date = datetime.date.today() - datetime.timedelta(days=1) -DATE_RANGE_REQ_DATA_API_PATH = f"/reports?filter=created_date>={start_date}&filter=created_date<={end_date}" # noqa +date_range_req_data_params = {'filter':f"created_date>={start_date}", 'filter':f"created_date<={end_date}"} +DATE_RANGE_REQ_DATA_API_PATH = REPORT_API_PATH_ROOT + + urllib.parse.urlencode(date_range_req_data_params) print(" * Downloading data for dataframe from API path: " + DATE_RANGE_REQ_DATA_API_PATH) date_range_req_df = pd.read_json(API_HOST + DATE_RANGE_REQ_DATA_API_PATH) print(" * Dataframe has been loaded from API path: " + DATE_RANGE_REQ_DATA_API_PATH) diff --git a/server/dash/requirements.txt b/server/dash/requirements.txt index 4c00e8f6e..cbef48629 100644 --- a/server/dash/requirements.txt +++ b/server/dash/requirements.txt @@ -35,4 +35,5 @@ certifi==2021.10.8 charset-normalizer==2.0.12 dash-daq==0.5.0 dash-bootstrap-components==1.1.0 -flask \ No newline at end of file +flask +urllib3 \ No newline at end of file From c97b6c7db240c380968ccce91bbc43d840f725a6 Mon Sep 17 00:00:00 2001 From: Yi Heng Joshua Wu Date: Tue, 26 Jul 2022 23:44:52 -0700 Subject: [PATCH 32/53] Update API path names --- server/dash/dashboards/overviewComb.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/server/dash/dashboards/overviewComb.py b/server/dash/dashboards/overviewComb.py index d75983c76..6ecffe057 100644 --- a/server/dash/dashboards/overviewComb.py +++ b/server/dash/dashboards/overviewComb.py @@ -68,7 +68,7 @@ values='counts', labels=LABELS, hole=.3, - TITLE="Total Requests by Agency", + title="Total Requests by Agency", ) req_share_by_agency_pie_chart.update_layout(margin=dict(l=100, r=100, b=100, t=100)) @@ -83,7 +83,7 @@ req_source_count_df, y=req_source_count_df.index, x='counts', - TITLE="Total Requests by Source", + title="Total Requests by Source", orientation='h' ) @@ -109,14 +109,14 @@ dtick=5 ) med_days_to_close_box_plot.update_layout( - TITLE="Total Median Days to Close by Type", + title="Total Median Days to Close by Type", ) # Day of Week Bar Chart: -start_date = datetime.date.today() - datetime.timedelta(days=30) +start_date = datetime.date.today() - datetime.timedelta(days=365) end_date = datetime.date.today() - datetime.timedelta(days=1) date_range_req_data_params = {'filter':f"created_date>={start_date}", 'filter':f"created_date<={end_date}"} -DATE_RANGE_REQ_DATA_API_PATH = REPORT_API_PATH_ROOT + + urllib.parse.urlencode(date_range_req_data_params) +DATE_RANGE_REQ_DATA_API_PATH = REPORT_API_PATH_ROOT + urllib.parse.urlencode(date_range_req_data_params) print(" * Downloading data for dataframe from API path: " + DATE_RANGE_REQ_DATA_API_PATH) date_range_req_df = pd.read_json(API_HOST + DATE_RANGE_REQ_DATA_API_PATH) print(" * Dataframe has been loaded from API path: " + DATE_RANGE_REQ_DATA_API_PATH) @@ -139,7 +139,7 @@ x=nc_req_count_df.index, y='counts', labels=LABELS, - TITLE="Total Requests by Neighborhood Councils", + title="Total Requests by Neighborhood Councils", ) req_by_nc_bar_chart.update_layout(font=dict(size=12)) From c513549e984667d1c226a936bf2114956cd7fb5f Mon Sep 17 00:00:00 2001 From: Yi Heng Joshua Wu Date: Thu, 28 Jul 2022 20:00:39 -0700 Subject: [PATCH 33/53] Remove flask from app.py --- server/dash/app.py | 1 - 1 file changed, 1 deletion(-) diff --git a/server/dash/app.py b/server/dash/app.py index 46c851d1f..cd0b7e5cc 100644 --- a/server/dash/app.py +++ b/server/dash/app.py @@ -1,4 +1,3 @@ -import flask from dash import Dash, dcc, html external_stylesheets = ['/static/reports.css'] From d98a844bab610486a3e6be0c99cd852fc98c7811 Mon Sep 17 00:00:00 2001 From: Yi Heng Joshua Wu Date: Thu, 28 Jul 2022 20:01:31 -0700 Subject: [PATCH 34/53] Adding period at end of comment --- server/dash/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/dash/app.py b/server/dash/app.py index cd0b7e5cc..16628322a 100644 --- a/server/dash/app.py +++ b/server/dash/app.py @@ -1,7 +1,7 @@ from dash import Dash, dcc, html external_stylesheets = ['/static/reports.css'] -# Suppress Callback Exceptions due to multi-page Dashboard layout, some callback id may not exist initially +# Suppress Callback Exceptions due to multi-page Dashboard layout, some callback id may not exist initially. app = Dash(__name__, external_stylesheets=external_stylesheets, suppress_callback_exceptions=True) # set up default layout From 4cb5e710b46b3cad53531bea4cd35ad830735ed0 Mon Sep 17 00:00:00 2001 From: Yi Heng Joshua Wu Date: Thu, 28 Jul 2022 20:05:24 -0700 Subject: [PATCH 35/53] Update comments --- server/dash/dashboards/overviewComb.py | 28 +++++++++++++------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/server/dash/dashboards/overviewComb.py b/server/dash/dashboards/overviewComb.py index 6ecffe057..4b9ea04f1 100644 --- a/server/dash/dashboards/overviewComb.py +++ b/server/dash/dashboards/overviewComb.py @@ -13,7 +13,7 @@ REPORT_API_PATH_ROOT = "/reports?" # DATA -# Loading the dataframe for the NCs and correspoding requests +# Loading the dataframe for the NCs and corresponding requests. nc_req_count_api_params = {'field':'council_name', 'filter':'created_date>=2016-01-01'} NC_REQ_COUNT_DATA_API_PATH = REPORT_API_PATH_ROOT + urllib.parse.urlencode(nc_req_count_api_params) print(" * Downloading data from API path: " + NC_REQ_COUNT_DATA_API_PATH) @@ -21,7 +21,7 @@ nc_req_count_df = nc_req_count_df.groupby(['council_name'])['counts'].sum().sort_values().to_frame() print(" * Dataframe has been loaded from API path: " + NC_REQ_COUNT_DATA_API_PATH) -# Loading the data for the number of new requests +# Loading the data for the number of new requests. new_req_count_api_params = {'field':'created_year', 'filter':'created_date>=2016-01-01'} NEW_REQ_COUNT_DATA_API_PATH = REPORT_API_PATH_ROOT + urllib.parse.urlencode(new_req_count_api_params) print(" * Downloading data from API path: " + NEW_REQ_COUNT_DATA_API_PATH) @@ -29,7 +29,7 @@ new_req_count_df = new_req_count_df.groupby(['created_year'])['counts'].sum().to_frame() print(" * Dataframe has been loaded from API path: " + NEW_REQ_COUNT_DATA_API_PATH) -# Loading the count of each request types overall +# Loading the count of each request types overall. req_count_api_params = {'field':'type_name', 'filter':'created_date>=2016-01-01'} REQ_COUNT_DATA_API_PATH = REPORT_API_PATH_ROOT + urllib.parse.urlencode(req_count_api_params) print(" * Downloading data from API path: " + REQ_COUNT_DATA_API_PATH) @@ -37,7 +37,7 @@ req_count = req_count_df.groupby(['type_name'], as_index=False)['counts'].sum() print(" * Dataframe has been loaded from API path: " + REQ_COUNT_DATA_API_PATH) -# Loading the total number of request source +# Loading the total number of request source. req_source_count_api_params = {'field':'source_name', 'filter':'created_date>=2016-01-01'} REQ_SOURCE_COUNT_DATA_API_PATH = REPORT_API_PATH_ROOT + urllib.parse.urlencode(req_source_count_api_params) print(" * Downloading data from API path: " + REQ_SOURCE_COUNT_DATA_API_PATH) @@ -45,7 +45,7 @@ req_source_count = req_source_count_df.groupby(['source_name'])['counts'].sum() print(" * Dataframe has been loaded from API path: " + REQ_SOURCE_COUNT_DATA_API_PATH) -# Loading the number of Request Agencies +# Loading the number of Request Agencies. req_agency_count_api_params = {'field':'agency_name', 'filter':'created_date>=2016-01-01'} REQ_AGENCY_COUNT_DATA_API_PATH = REPORT_API_PATH_ROOT + urllib.parse.urlencode(req_agency_count_api_params) print(" * Downloading data from API path: " + REQ_AGENCY_COUNT_DATA_API_PATH) @@ -53,7 +53,7 @@ agency_count = req_agency_count_df.groupby(['agency_name'])['counts'].sum() print(" * Dataframe has been loaded from API path: " + REQ_AGENCY_COUNT_DATA_API_PATH) -# Request Share by Agency Pie Chart +# Request Share by Agency Pie Chart. req_agency_count_df = agency_count.to_frame() req_agency_count_df.sort_values('counts', ascending=False, inplace=True) req_agency_count_df.loc['Others'] = req_agency_count_df[4:].sum() @@ -72,7 +72,7 @@ ) req_share_by_agency_pie_chart.update_layout(margin=dict(l=100, r=100, b=100, t=100)) -# Request Type by Source Bar Chart +# Request Type by Source Bar Chart. req_source_count_df = req_source_count.to_frame() req_source_count_df.sort_values('counts', ascending=False, inplace=True) req_source_count_df.loc['Others'] = req_source_count_df[4:].sum() @@ -87,7 +87,7 @@ orientation='h' ) -# Median Request Days to Close Box Plot +# Median Request Days to Close Box Plot. REQ_TYPE_STATS_API_PATH = '/types/stats' print(" * Downloading data from API path: " + REQ_TYPE_STATS_API_PATH) stats_df = pd.read_json(API_HOST + REQ_TYPE_STATS_API_PATH) @@ -112,7 +112,7 @@ title="Total Median Days to Close by Type", ) -# Day of Week Bar Chart: +# Day of Week Bar Chart. start_date = datetime.date.today() - datetime.timedelta(days=365) end_date = datetime.date.today() - datetime.timedelta(days=1) date_range_req_data_params = {'filter':f"created_date>={start_date}", 'filter':f"created_date<={end_date}"} @@ -133,7 +133,7 @@ num_req_by_day_bar_chart.update_xaxes(categoryorder='array', categoryarray=[ "Sunday", "Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday"]) -# Total Request by NC +# Total Request by NC. req_by_nc_bar_chart = px.bar( nc_req_count_df, x=nc_req_count_df.index, @@ -143,7 +143,7 @@ ) req_by_nc_bar_chart.update_layout(font=dict(size=12)) -# LAYOUT VARIABLES +# LAYOUT VARIABLES. INDICATOR_CARD_STYLE = {"text-align": 'center', "border": "0.5px black solid", 'width': '18vw', 'display': 'inline-block'} BORDER_STYLE = {"border": "0.5px black solid"} @@ -151,10 +151,10 @@ TWO_CHART_STYLE = {"width": "48vw", "height": "40vh"} DASHBOARD_OUTLINE = "The figures below represent the total number of 311 requests made across LA County from 2016-2021. In 2020, we saw an all-time high with more than 1.4 million requests." -# LAYOUT +# LAYOUT. layout = html.Div([ - # Part 1 of dashboard + # Dashboard showing summary statistics, median time to close box plot, and request share by agency. html.H1(TITLE + " Pt. 1"), html.P(DASHBOARD_OUTLINE, style={'font-size': '18px', 'font-style': 'italic'}), @@ -182,7 +182,7 @@ html.Div(html.Br(), style={"height": "2vh"}), - # Part 2 of dashboard + # Bar charts showing number of requests by day, number of requests by source, and number of requests by NCs. html.H1(TITLE + " Pt. 2"), html.Div([ html.Div(dcc.Graph(id='num_req_by_day_bar_chart', figure=num_req_by_day_bar_chart, className="half-graph", From fab34ce40c36f5ab0e4e8bec2022a4955e0f5bdf Mon Sep 17 00:00:00 2001 From: Yi Heng Joshua Wu Date: Thu, 28 Jul 2022 20:07:02 -0700 Subject: [PATCH 36/53] Renamed file to overview_combined --- .../dash/dashboards/{overviewComb.py => overview_combined.py.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename server/dash/dashboards/{overviewComb.py => overview_combined.py.py} (100%) diff --git a/server/dash/dashboards/overviewComb.py b/server/dash/dashboards/overview_combined.py.py similarity index 100% rename from server/dash/dashboards/overviewComb.py rename to server/dash/dashboards/overview_combined.py.py From 363fff3d291e1c0bdd702ecc4635ed3eee34b9b0 Mon Sep 17 00:00:00 2001 From: Yi Heng Joshua Wu Date: Thu, 28 Jul 2022 20:09:50 -0700 Subject: [PATCH 37/53] make string two lines --- server/dash/dashboards/overview_combined.py.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/dash/dashboards/overview_combined.py.py b/server/dash/dashboards/overview_combined.py.py index 4b9ea04f1..a14276fa5 100644 --- a/server/dash/dashboards/overview_combined.py.py +++ b/server/dash/dashboards/overview_combined.py.py @@ -149,7 +149,8 @@ BORDER_STYLE = {"border": "0.5px black solid"} EQUAL_SPACE_BOX_STYLE = {'display': 'flex', "justify-content": "space-between"} TWO_CHART_STYLE = {"width": "48vw", "height": "40vh"} -DASHBOARD_OUTLINE = "The figures below represent the total number of 311 requests made across LA County from 2016-2021. In 2020, we saw an all-time high with more than 1.4 million requests." +DASHBOARD_OUTLINE = """The figures below represent the total number of 311 requests made +across LA County from 2016-2021. In 2020, we saw an all-time high with more than 1.4 million requests.""" # LAYOUT. layout = html.Div([ From b17f5e7c8665a4b8d64d73a3fe4da0ac16549ae1 Mon Sep 17 00:00:00 2001 From: Yi Heng Joshua Wu Date: Thu, 28 Jul 2022 20:14:11 -0700 Subject: [PATCH 38/53] Fixed duplicate file extension --- .../dashboards/{overview_combined.py.py => overview_combined.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename server/dash/dashboards/{overview_combined.py.py => overview_combined.py} (100%) diff --git a/server/dash/dashboards/overview_combined.py.py b/server/dash/dashboards/overview_combined.py similarity index 100% rename from server/dash/dashboards/overview_combined.py.py rename to server/dash/dashboards/overview_combined.py From 73c52fdb311aedd08fbb85557fb5aba99438831a Mon Sep 17 00:00:00 2001 From: Yi Heng Joshua Wu Date: Thu, 28 Jul 2022 20:16:53 -0700 Subject: [PATCH 39/53] Add created date filter --- server/dash/dashboards/overview_combined.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/server/dash/dashboards/overview_combined.py b/server/dash/dashboards/overview_combined.py index a14276fa5..15c5c165e 100644 --- a/server/dash/dashboards/overview_combined.py +++ b/server/dash/dashboards/overview_combined.py @@ -11,10 +11,11 @@ TITLE = "OVERVIEW COMBINED DASHBOARD" REPORT_API_PATH_ROOT = "/reports?" +CREATED_DATE_FILTER = 'created_date>=2016-01-01' # DATA # Loading the dataframe for the NCs and corresponding requests. -nc_req_count_api_params = {'field':'council_name', 'filter':'created_date>=2016-01-01'} +nc_req_count_api_params = {'field':'council_name', 'filter':CREATED_DATE_FILTER} NC_REQ_COUNT_DATA_API_PATH = REPORT_API_PATH_ROOT + urllib.parse.urlencode(nc_req_count_api_params) print(" * Downloading data from API path: " + NC_REQ_COUNT_DATA_API_PATH) nc_req_count_df = pd.read_json(API_HOST + NC_REQ_COUNT_DATA_API_PATH) @@ -22,7 +23,7 @@ print(" * Dataframe has been loaded from API path: " + NC_REQ_COUNT_DATA_API_PATH) # Loading the data for the number of new requests. -new_req_count_api_params = {'field':'created_year', 'filter':'created_date>=2016-01-01'} +new_req_count_api_params = {'field':'created_year', 'filter':CREATED_DATE_FILTER} NEW_REQ_COUNT_DATA_API_PATH = REPORT_API_PATH_ROOT + urllib.parse.urlencode(new_req_count_api_params) print(" * Downloading data from API path: " + NEW_REQ_COUNT_DATA_API_PATH) new_req_count_df = pd.read_json(API_HOST + NEW_REQ_COUNT_DATA_API_PATH) @@ -30,7 +31,7 @@ print(" * Dataframe has been loaded from API path: " + NEW_REQ_COUNT_DATA_API_PATH) # Loading the count of each request types overall. -req_count_api_params = {'field':'type_name', 'filter':'created_date>=2016-01-01'} +req_count_api_params = {'field':'type_name', 'filter':CREATED_DATE_FILTER} REQ_COUNT_DATA_API_PATH = REPORT_API_PATH_ROOT + urllib.parse.urlencode(req_count_api_params) print(" * Downloading data from API path: " + REQ_COUNT_DATA_API_PATH) req_count_df = pd.read_json(API_HOST + REQ_COUNT_DATA_API_PATH) @@ -38,7 +39,7 @@ print(" * Dataframe has been loaded from API path: " + REQ_COUNT_DATA_API_PATH) # Loading the total number of request source. -req_source_count_api_params = {'field':'source_name', 'filter':'created_date>=2016-01-01'} +req_source_count_api_params = {'field':'source_name', 'filter':CREATED_DATE_FILTER} REQ_SOURCE_COUNT_DATA_API_PATH = REPORT_API_PATH_ROOT + urllib.parse.urlencode(req_source_count_api_params) print(" * Downloading data from API path: " + REQ_SOURCE_COUNT_DATA_API_PATH) req_source_count_df = pd.read_json(API_HOST + REQ_SOURCE_COUNT_DATA_API_PATH) @@ -46,7 +47,7 @@ print(" * Dataframe has been loaded from API path: " + REQ_SOURCE_COUNT_DATA_API_PATH) # Loading the number of Request Agencies. -req_agency_count_api_params = {'field':'agency_name', 'filter':'created_date>=2016-01-01'} +req_agency_count_api_params = {'field':'agency_name', 'filter':CREATED_DATE_FILTER} REQ_AGENCY_COUNT_DATA_API_PATH = REPORT_API_PATH_ROOT + urllib.parse.urlencode(req_agency_count_api_params) print(" * Downloading data from API path: " + REQ_AGENCY_COUNT_DATA_API_PATH) req_agency_count_df = pd.read_json(API_HOST + REQ_AGENCY_COUNT_DATA_API_PATH) From 9d5423772652a263e1bfe5454db3380af10caa07 Mon Sep 17 00:00:00 2001 From: Yi Heng Joshua Wu Date: Thu, 28 Jul 2022 20:33:03 -0700 Subject: [PATCH 40/53] Added helper function --- server/dash/dashboards/overview_combined.py | 82 +++++++++++++-------- 1 file changed, 50 insertions(+), 32 deletions(-) diff --git a/server/dash/dashboards/overview_combined.py b/server/dash/dashboards/overview_combined.py index 15c5c165e..3962cc70d 100644 --- a/server/dash/dashboards/overview_combined.py +++ b/server/dash/dashboards/overview_combined.py @@ -14,48 +14,66 @@ CREATED_DATE_FILTER = 'created_date>=2016-01-01' # DATA +def generate_dataframe_from_api(api_params, group_by_col, sort_values=False): + """ + + """ + DATA_API_PATH = REPORT_API_PATH_ROOT + urllib.parse.urlencode(api_params) + print(" * Downloading data from API path: " + DATA_API_PATH) + result_df = pd.read_json(API_HOST + DATA_API_PATH) + result_df_gb = result_df.groupby([group_by_col])['counts'].sum().sort_values().to_frame() + print(" * Dataframe has been loaded from API path: " + DATA_API_PATH) + return result_df, result_df_gb + + + # Loading the dataframe for the NCs and corresponding requests. -nc_req_count_api_params = {'field':'council_name', 'filter':CREATED_DATE_FILTER} -NC_REQ_COUNT_DATA_API_PATH = REPORT_API_PATH_ROOT + urllib.parse.urlencode(nc_req_count_api_params) -print(" * Downloading data from API path: " + NC_REQ_COUNT_DATA_API_PATH) -nc_req_count_df = pd.read_json(API_HOST + NC_REQ_COUNT_DATA_API_PATH) -nc_req_count_df = nc_req_count_df.groupby(['council_name'])['counts'].sum().sort_values().to_frame() -print(" * Dataframe has been loaded from API path: " + NC_REQ_COUNT_DATA_API_PATH) +nc_req_count_api_params = {'field': 'council_name', 'filter': CREATED_DATE_FILTER} +_, nc_req_count_df = generate_dataframe_from_api(nc_req_count_api_params, nc_req_count_api_params['field']) +# NC_REQ_COUNT_DATA_API_PATH = REPORT_API_PATH_ROOT + urllib.parse.urlencode(nc_req_count_api_params) +# print(" * Downloading data from API path: " + NC_REQ_COUNT_DATA_API_PATH) +# nc_req_count_df = pd.read_json(API_HOST + NC_REQ_COUNT_DATA_API_PATH) +# nc_req_count_df = nc_req_count_df.groupby(['council_name'])['counts'].sum().sort_values().to_frame() +# print(" * Dataframe has been loaded from API path: " + NC_REQ_COUNT_DATA_API_PATH) # Loading the data for the number of new requests. -new_req_count_api_params = {'field':'created_year', 'filter':CREATED_DATE_FILTER} -NEW_REQ_COUNT_DATA_API_PATH = REPORT_API_PATH_ROOT + urllib.parse.urlencode(new_req_count_api_params) -print(" * Downloading data from API path: " + NEW_REQ_COUNT_DATA_API_PATH) -new_req_count_df = pd.read_json(API_HOST + NEW_REQ_COUNT_DATA_API_PATH) -new_req_count_df = new_req_count_df.groupby(['created_year'])['counts'].sum().to_frame() -print(" * Dataframe has been loaded from API path: " + NEW_REQ_COUNT_DATA_API_PATH) +new_req_count_api_params = {'field': 'created_year', 'filter': CREATED_DATE_FILTER} +_, new_req_count_df = generate_dataframe_from_api(new_req_count_api_params, new_req_count_api_params['field']) +# NEW_REQ_COUNT_DATA_API_PATH = REPORT_API_PATH_ROOT + urllib.parse.urlencode(new_req_count_api_params) +# print(" * Downloading data from API path: " + NEW_REQ_COUNT_DATA_API_PATH) +# new_req_count_df = pd.read_json(API_HOST + NEW_REQ_COUNT_DATA_API_PATH) +# new_req_count_df = new_req_count_df.groupby(['created_year'])['counts'].sum().to_frame() +# print(" * Dataframe has been loaded from API path: " + NEW_REQ_COUNT_DATA_API_PATH) # Loading the count of each request types overall. -req_count_api_params = {'field':'type_name', 'filter':CREATED_DATE_FILTER} -REQ_COUNT_DATA_API_PATH = REPORT_API_PATH_ROOT + urllib.parse.urlencode(req_count_api_params) -print(" * Downloading data from API path: " + REQ_COUNT_DATA_API_PATH) -req_count_df = pd.read_json(API_HOST + REQ_COUNT_DATA_API_PATH) -req_count = req_count_df.groupby(['type_name'], as_index=False)['counts'].sum() -print(" * Dataframe has been loaded from API path: " + REQ_COUNT_DATA_API_PATH) +req_count_api_params = {'field': 'type_name', 'filter': CREATED_DATE_FILTER} +req_count_df, req_count = generate_dataframe_from_api(new_req_count_api_params, new_req_count_api_params['field']) +# REQ_COUNT_DATA_API_PATH = REPORT_API_PATH_ROOT + urllib.parse.urlencode(req_count_api_params) +# print(" * Downloading data from API path: " + REQ_COUNT_DATA_API_PATH) +# req_count_df = pd.read_json(API_HOST + REQ_COUNT_DATA_API_PATH) +# req_count = req_count_df.groupby(['type_name'], as_index=False)['counts'].sum() +# print(" * Dataframe has been loaded from API path: " + REQ_COUNT_DATA_API_PATH) # Loading the total number of request source. -req_source_count_api_params = {'field':'source_name', 'filter':CREATED_DATE_FILTER} -REQ_SOURCE_COUNT_DATA_API_PATH = REPORT_API_PATH_ROOT + urllib.parse.urlencode(req_source_count_api_params) -print(" * Downloading data from API path: " + REQ_SOURCE_COUNT_DATA_API_PATH) -req_source_count_df = pd.read_json(API_HOST + REQ_SOURCE_COUNT_DATA_API_PATH) -req_source_count = req_source_count_df.groupby(['source_name'])['counts'].sum() -print(" * Dataframe has been loaded from API path: " + REQ_SOURCE_COUNT_DATA_API_PATH) +req_source_count_api_params = {'field': 'source_name', 'filter': CREATED_DATE_FILTER} +req_source_count_df, req_source_count = generate_dataframe_from_api(req_source_count_api_params, req_source_count_api_params['field']) +# REQ_SOURCE_COUNT_DATA_API_PATH = REPORT_API_PATH_ROOT + urllib.parse.urlencode(req_source_count_api_params) +# print(" * Downloading data from API path: " + REQ_SOURCE_COUNT_DATA_API_PATH) +# req_source_count_df = pd.read_json(API_HOST + REQ_SOURCE_COUNT_DATA_API_PATH) +# req_source_count = req_source_count_df.groupby(['source_name'])['counts'].sum() +# print(" * Dataframe has been loaded from API path: " + REQ_SOURCE_COUNT_DATA_API_PATH) # Loading the number of Request Agencies. -req_agency_count_api_params = {'field':'agency_name', 'filter':CREATED_DATE_FILTER} -REQ_AGENCY_COUNT_DATA_API_PATH = REPORT_API_PATH_ROOT + urllib.parse.urlencode(req_agency_count_api_params) -print(" * Downloading data from API path: " + REQ_AGENCY_COUNT_DATA_API_PATH) -req_agency_count_df = pd.read_json(API_HOST + REQ_AGENCY_COUNT_DATA_API_PATH) -agency_count = req_agency_count_df.groupby(['agency_name'])['counts'].sum() -print(" * Dataframe has been loaded from API path: " + REQ_AGENCY_COUNT_DATA_API_PATH) +req_agency_count_api_params = {'field': 'agency_name', 'filter': CREATED_DATE_FILTER} +req_agency_count_df, agency_count = generate_dataframe_from_api(req_agency_count_api_params, req_agency_count_api_params['field']) +# REQ_AGENCY_COUNT_DATA_API_PATH = REPORT_API_PATH_ROOT + urllib.parse.urlencode(req_agency_count_api_params) +# print(" * Downloading data from API path: " + REQ_AGENCY_COUNT_DATA_API_PATH) +# req_agency_count_df = pd.read_json(API_HOST + REQ_AGENCY_COUNT_DATA_API_PATH) +# agency_count = req_agency_count_df.groupby(['agency_name'])['counts'].sum() +# print(" * Dataframe has been loaded from API path: " + REQ_AGENCY_COUNT_DATA_API_PATH) # Request Share by Agency Pie Chart. -req_agency_count_df = agency_count.to_frame() +# req_agency_count_df = agency_count.to_frame() req_agency_count_df.sort_values('counts', ascending=False, inplace=True) req_agency_count_df.loc['Others'] = req_agency_count_df[4:].sum() req_agency_count_df.sort_values('counts', ascending=False, inplace=True) @@ -74,7 +92,7 @@ req_share_by_agency_pie_chart.update_layout(margin=dict(l=100, r=100, b=100, t=100)) # Request Type by Source Bar Chart. -req_source_count_df = req_source_count.to_frame() +# req_source_count_df = req_source_count.to_frame() req_source_count_df.sort_values('counts', ascending=False, inplace=True) req_source_count_df.loc['Others'] = req_source_count_df[4:].sum() req_source_count_df.sort_values('counts', ascending=False, inplace=True) From 8eb319afd4bb70c74c85615886a361d2fc81e9a9 Mon Sep 17 00:00:00 2001 From: Yi Heng Joshua Wu Date: Thu, 28 Jul 2022 20:50:18 -0700 Subject: [PATCH 41/53] Add helper func documenttaion --- server/dash/dashboards/overview_combined.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/server/dash/dashboards/overview_combined.py b/server/dash/dashboards/overview_combined.py index 3962cc70d..ea178c814 100644 --- a/server/dash/dashboards/overview_combined.py +++ b/server/dash/dashboards/overview_combined.py @@ -15,8 +15,18 @@ # DATA def generate_dataframe_from_api(api_params, group_by_col, sort_values=False): - """ + """Generates the dataframe output from the reports API. + + This function takes the "api_params" dictionary and "group_by_col" string and outputs the relevant fields + in the raw dataframe and groupby dataframe from the reports API. + + Args: + api_params: a dictionary of parameters for calling the API. + group_by_col: the column name we use to groupby and output the result_df_gb dataframe. + Returns: + result_df: the raw dataframe from calling the api with parameters from "api_params". + result_df_gb: dataframe output from result_df group by the column "group_by_col" and aggregating the sum. """ DATA_API_PATH = REPORT_API_PATH_ROOT + urllib.parse.urlencode(api_params) print(" * Downloading data from API path: " + DATA_API_PATH) From 71c99c852af5a4705f77d7a3fa69794d7c8b487a Mon Sep 17 00:00:00 2001 From: Yi Heng Joshua Wu Date: Thu, 28 Jul 2022 21:43:43 -0700 Subject: [PATCH 42/53] Utilize helper functions --- server/dash/dashboards/overview_combined.py | 45 ++++----------------- 1 file changed, 7 insertions(+), 38 deletions(-) diff --git a/server/dash/dashboards/overview_combined.py b/server/dash/dashboards/overview_combined.py index ea178c814..2c00192b4 100644 --- a/server/dash/dashboards/overview_combined.py +++ b/server/dash/dashboards/overview_combined.py @@ -14,7 +14,7 @@ CREATED_DATE_FILTER = 'created_date>=2016-01-01' # DATA -def generate_dataframe_from_api(api_params, group_by_col, sort_values=False): +def generate_dataframe_from_api(api_params, group_by_col): """Generates the dataframe output from the reports API. This function takes the "api_params" dictionary and "group_by_col" string and outputs the relevant fields @@ -35,61 +35,33 @@ def generate_dataframe_from_api(api_params, group_by_col, sort_values=False): print(" * Dataframe has been loaded from API path: " + DATA_API_PATH) return result_df, result_df_gb - - # Loading the dataframe for the NCs and corresponding requests. nc_req_count_api_params = {'field': 'council_name', 'filter': CREATED_DATE_FILTER} _, nc_req_count_df = generate_dataframe_from_api(nc_req_count_api_params, nc_req_count_api_params['field']) -# NC_REQ_COUNT_DATA_API_PATH = REPORT_API_PATH_ROOT + urllib.parse.urlencode(nc_req_count_api_params) -# print(" * Downloading data from API path: " + NC_REQ_COUNT_DATA_API_PATH) -# nc_req_count_df = pd.read_json(API_HOST + NC_REQ_COUNT_DATA_API_PATH) -# nc_req_count_df = nc_req_count_df.groupby(['council_name'])['counts'].sum().sort_values().to_frame() -# print(" * Dataframe has been loaded from API path: " + NC_REQ_COUNT_DATA_API_PATH) # Loading the data for the number of new requests. new_req_count_api_params = {'field': 'created_year', 'filter': CREATED_DATE_FILTER} _, new_req_count_df = generate_dataframe_from_api(new_req_count_api_params, new_req_count_api_params['field']) -# NEW_REQ_COUNT_DATA_API_PATH = REPORT_API_PATH_ROOT + urllib.parse.urlencode(new_req_count_api_params) -# print(" * Downloading data from API path: " + NEW_REQ_COUNT_DATA_API_PATH) -# new_req_count_df = pd.read_json(API_HOST + NEW_REQ_COUNT_DATA_API_PATH) -# new_req_count_df = new_req_count_df.groupby(['created_year'])['counts'].sum().to_frame() -# print(" * Dataframe has been loaded from API path: " + NEW_REQ_COUNT_DATA_API_PATH) # Loading the count of each request types overall. req_count_api_params = {'field': 'type_name', 'filter': CREATED_DATE_FILTER} req_count_df, req_count = generate_dataframe_from_api(new_req_count_api_params, new_req_count_api_params['field']) -# REQ_COUNT_DATA_API_PATH = REPORT_API_PATH_ROOT + urllib.parse.urlencode(req_count_api_params) -# print(" * Downloading data from API path: " + REQ_COUNT_DATA_API_PATH) -# req_count_df = pd.read_json(API_HOST + REQ_COUNT_DATA_API_PATH) -# req_count = req_count_df.groupby(['type_name'], as_index=False)['counts'].sum() -# print(" * Dataframe has been loaded from API path: " + REQ_COUNT_DATA_API_PATH) # Loading the total number of request source. req_source_count_api_params = {'field': 'source_name', 'filter': CREATED_DATE_FILTER} req_source_count_df, req_source_count = generate_dataframe_from_api(req_source_count_api_params, req_source_count_api_params['field']) -# REQ_SOURCE_COUNT_DATA_API_PATH = REPORT_API_PATH_ROOT + urllib.parse.urlencode(req_source_count_api_params) -# print(" * Downloading data from API path: " + REQ_SOURCE_COUNT_DATA_API_PATH) -# req_source_count_df = pd.read_json(API_HOST + REQ_SOURCE_COUNT_DATA_API_PATH) -# req_source_count = req_source_count_df.groupby(['source_name'])['counts'].sum() -# print(" * Dataframe has been loaded from API path: " + REQ_SOURCE_COUNT_DATA_API_PATH) # Loading the number of Request Agencies. req_agency_count_api_params = {'field': 'agency_name', 'filter': CREATED_DATE_FILTER} req_agency_count_df, agency_count = generate_dataframe_from_api(req_agency_count_api_params, req_agency_count_api_params['field']) -# REQ_AGENCY_COUNT_DATA_API_PATH = REPORT_API_PATH_ROOT + urllib.parse.urlencode(req_agency_count_api_params) -# print(" * Downloading data from API path: " + REQ_AGENCY_COUNT_DATA_API_PATH) -# req_agency_count_df = pd.read_json(API_HOST + REQ_AGENCY_COUNT_DATA_API_PATH) -# agency_count = req_agency_count_df.groupby(['agency_name'])['counts'].sum() -# print(" * Dataframe has been loaded from API path: " + REQ_AGENCY_COUNT_DATA_API_PATH) # Request Share by Agency Pie Chart. -# req_agency_count_df = agency_count.to_frame() +req_agency_count_df = agency_count req_agency_count_df.sort_values('counts', ascending=False, inplace=True) req_agency_count_df.loc['Others'] = req_agency_count_df[4:].sum() req_agency_count_df.sort_values('counts', ascending=False, inplace=True) req_agency_count_df = req_agency_count_df[:5] -req_agency_count_df.index = req_agency_count_df.index.map( - lambda x: '
'.join(textwrap.wrap(x, width=16))) +req_agency_count_df.index = req_agency_count_df.index.map(lambda x: '
'.join(textwrap.wrap(x, width=16))) req_share_by_agency_pie_chart = px.pie( req_agency_count_df, @@ -102,7 +74,7 @@ def generate_dataframe_from_api(api_params, group_by_col, sort_values=False): req_share_by_agency_pie_chart.update_layout(margin=dict(l=100, r=100, b=100, t=100)) # Request Type by Source Bar Chart. -# req_source_count_df = req_source_count.to_frame() +req_source_count_df = req_source_count req_source_count_df.sort_values('counts', ascending=False, inplace=True) req_source_count_df.loc['Others'] = req_source_count_df[4:].sum() req_source_count_df.sort_values('counts', ascending=False, inplace=True) @@ -145,13 +117,10 @@ def generate_dataframe_from_api(api_params, group_by_col, sort_values=False): start_date = datetime.date.today() - datetime.timedelta(days=365) end_date = datetime.date.today() - datetime.timedelta(days=1) date_range_req_data_params = {'filter':f"created_date>={start_date}", 'filter':f"created_date<={end_date}"} -DATE_RANGE_REQ_DATA_API_PATH = REPORT_API_PATH_ROOT + urllib.parse.urlencode(date_range_req_data_params) -print(" * Downloading data for dataframe from API path: " + DATE_RANGE_REQ_DATA_API_PATH) -date_range_req_df = pd.read_json(API_HOST + DATE_RANGE_REQ_DATA_API_PATH) -print(" * Dataframe has been loaded from API path: " + DATE_RANGE_REQ_DATA_API_PATH) +date_range_req_df, _ = generate_dataframe_from_api(date_range_req_data_params, 'created_date') date_range_req_df['created_date'] = pd.to_datetime(date_range_req_df['created_date']) -day_of_week_df = date_range_req_df.groupby(['created_date']).agg('sum').reset_index() -day_of_week_df['day_of_week'] = day_of_week_df['created_date'].dt.day_name() +date_range_req_df['day_of_week'] = date_range_req_df['created_date'].dt.day_name() +day_of_week_df = date_range_req_df.groupby(['day_of_week']).agg('sum').reset_index() num_req_by_day_bar_chart = px.bar( day_of_week_df, From 7d260bc714763244d38558c7ef1db88f46f8ae4e Mon Sep 17 00:00:00 2001 From: Yi Heng Joshua Wu Date: Thu, 28 Jul 2022 21:50:56 -0700 Subject: [PATCH 43/53] Add constant for slice index --- server/dash/dashboards/overview_combined.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/server/dash/dashboards/overview_combined.py b/server/dash/dashboards/overview_combined.py index 2c00192b4..d5aa4d9e5 100644 --- a/server/dash/dashboards/overview_combined.py +++ b/server/dash/dashboards/overview_combined.py @@ -9,11 +9,14 @@ from config import API_HOST from design import LABELS -TITLE = "OVERVIEW COMBINED DASHBOARD" -REPORT_API_PATH_ROOT = "/reports?" +# COMMON VARIABLES. CREATED_DATE_FILTER = 'created_date>=2016-01-01' -# DATA +REPORT_API_PATH_ROOT = "/reports?" +REQ_TYPE_STATS_API_PATH = '/types/stats' +TITLE = "OVERVIEW COMBINED DASHBOARD" +NON_TOP4_INDEX_START = 4 +# HELPER FUNCTIONS. def generate_dataframe_from_api(api_params, group_by_col): """Generates the dataframe output from the reports API. @@ -35,6 +38,8 @@ def generate_dataframe_from_api(api_params, group_by_col): print(" * Dataframe has been loaded from API path: " + DATA_API_PATH) return result_df, result_df_gb +# DATA + # Loading the dataframe for the NCs and corresponding requests. nc_req_count_api_params = {'field': 'council_name', 'filter': CREATED_DATE_FILTER} _, nc_req_count_df = generate_dataframe_from_api(nc_req_count_api_params, nc_req_count_api_params['field']) @@ -58,9 +63,9 @@ def generate_dataframe_from_api(api_params, group_by_col): # Request Share by Agency Pie Chart. req_agency_count_df = agency_count req_agency_count_df.sort_values('counts', ascending=False, inplace=True) -req_agency_count_df.loc['Others'] = req_agency_count_df[4:].sum() +req_agency_count_df.loc['Others'] = req_agency_count_df[NON_TOP4_INDEX_START:].sum() req_agency_count_df.sort_values('counts', ascending=False, inplace=True) -req_agency_count_df = req_agency_count_df[:5] +req_agency_count_df = req_agency_count_df[:NON_TOP4_INDEX_START + 1] req_agency_count_df.index = req_agency_count_df.index.map(lambda x: '
'.join(textwrap.wrap(x, width=16))) req_share_by_agency_pie_chart = px.pie( @@ -76,9 +81,9 @@ def generate_dataframe_from_api(api_params, group_by_col): # Request Type by Source Bar Chart. req_source_count_df = req_source_count req_source_count_df.sort_values('counts', ascending=False, inplace=True) -req_source_count_df.loc['Others'] = req_source_count_df[4:].sum() +req_source_count_df.loc['Others'] = req_source_count_df[NON_TOP4_INDEX_START:].sum() req_source_count_df.sort_values('counts', ascending=False, inplace=True) -req_source_count_df = req_source_count_df[:5] +req_source_count_df = req_source_count_df[:NON_TOP4_INDEX_START + 1] req_source_bar_chart = px.bar( req_source_count_df, @@ -89,7 +94,6 @@ def generate_dataframe_from_api(api_params, group_by_col): ) # Median Request Days to Close Box Plot. -REQ_TYPE_STATS_API_PATH = '/types/stats' print(" * Downloading data from API path: " + REQ_TYPE_STATS_API_PATH) stats_df = pd.read_json(API_HOST + REQ_TYPE_STATS_API_PATH) stats_df = stats_df.sort_values('median', ascending=False) From 77a13c8e37b7aa12b2bddf8fdea32e737c312500 Mon Sep 17 00:00:00 2001 From: Yi Heng Joshua Wu Date: Thu, 28 Jul 2022 21:56:39 -0700 Subject: [PATCH 44/53] Final adjustments --- server/dash/dashboards/overview_combined.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/dash/dashboards/overview_combined.py b/server/dash/dashboards/overview_combined.py index d5aa4d9e5..ab23ca321 100644 --- a/server/dash/dashboards/overview_combined.py +++ b/server/dash/dashboards/overview_combined.py @@ -11,10 +11,10 @@ # COMMON VARIABLES. CREATED_DATE_FILTER = 'created_date>=2016-01-01' +NON_TOP4_INDEX_START = 4 REPORT_API_PATH_ROOT = "/reports?" REQ_TYPE_STATS_API_PATH = '/types/stats' TITLE = "OVERVIEW COMBINED DASHBOARD" -NON_TOP4_INDEX_START = 4 # HELPER FUNCTIONS. def generate_dataframe_from_api(api_params, group_by_col): From 20555f230d5d86d90cc47cadfee19c871b25a098 Mon Sep 17 00:00:00 2001 From: Yi Heng Joshua Wu Date: Mon, 1 Aug 2022 19:08:01 -0700 Subject: [PATCH 45/53] Update result_count_gb name --- server/dash/dashboards/overview_combined.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/server/dash/dashboards/overview_combined.py b/server/dash/dashboards/overview_combined.py index ab23ca321..c440fa3eb 100644 --- a/server/dash/dashboards/overview_combined.py +++ b/server/dash/dashboards/overview_combined.py @@ -25,18 +25,18 @@ def generate_dataframe_from_api(api_params, group_by_col): Args: api_params: a dictionary of parameters for calling the API. - group_by_col: the column name we use to groupby and output the result_df_gb dataframe. + group_by_col: the column name we use to groupby and output the result_counts_df dataframe. Returns: result_df: the raw dataframe from calling the api with parameters from "api_params". - result_df_gb: dataframe output from result_df group by the column "group_by_col" and aggregating the sum. + result_counts_df: dataframe output from result_df group by the column "group_by_col" and aggregating the sum. """ DATA_API_PATH = REPORT_API_PATH_ROOT + urllib.parse.urlencode(api_params) print(" * Downloading data from API path: " + DATA_API_PATH) result_df = pd.read_json(API_HOST + DATA_API_PATH) - result_df_gb = result_df.groupby([group_by_col])['counts'].sum().sort_values().to_frame() + result_counts_df = result_df.groupby([group_by_col])['counts'].sum().sort_values().to_frame() print(" * Dataframe has been loaded from API path: " + DATA_API_PATH) - return result_df, result_df_gb + return result_df, result_counts_df # DATA From 38d117e6ad6547a6ea2cd3fb6f157789e58f5557 Mon Sep 17 00:00:00 2001 From: Yi Heng Joshua Wu Date: Mon, 1 Aug 2022 20:46:36 -0700 Subject: [PATCH 46/53] Use constant instaed of dict --- server/dash/dashboards/overview_combined.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/server/dash/dashboards/overview_combined.py b/server/dash/dashboards/overview_combined.py index c440fa3eb..3284ab905 100644 --- a/server/dash/dashboards/overview_combined.py +++ b/server/dash/dashboards/overview_combined.py @@ -10,11 +10,17 @@ from design import LABELS # COMMON VARIABLES. +AGENCY_NAME_FIELD = 'agency_name' +COUNCIL_NAME_FIELD = 'council_name' CREATED_DATE_FILTER = 'created_date>=2016-01-01' +CREATED_YEAR_FIELD = 'created_year' NON_TOP4_INDEX_START = 4 REPORT_API_PATH_ROOT = "/reports?" REQ_TYPE_STATS_API_PATH = '/types/stats' +SOURCE_NAME_FIELD = 'source_name' TITLE = "OVERVIEW COMBINED DASHBOARD" +TYPE_NAME_FIELD = 'type_name' + # HELPER FUNCTIONS. def generate_dataframe_from_api(api_params, group_by_col): @@ -42,23 +48,23 @@ def generate_dataframe_from_api(api_params, group_by_col): # Loading the dataframe for the NCs and corresponding requests. nc_req_count_api_params = {'field': 'council_name', 'filter': CREATED_DATE_FILTER} -_, nc_req_count_df = generate_dataframe_from_api(nc_req_count_api_params, nc_req_count_api_params['field']) +_, nc_req_count_df = generate_dataframe_from_api(nc_req_count_api_params, COUNCIL_NAME_FIELD) # Loading the data for the number of new requests. new_req_count_api_params = {'field': 'created_year', 'filter': CREATED_DATE_FILTER} -_, new_req_count_df = generate_dataframe_from_api(new_req_count_api_params, new_req_count_api_params['field']) +_, new_req_count_df = generate_dataframe_from_api(new_req_count_api_params, CREATED_YEAR_FIELD) # Loading the count of each request types overall. req_count_api_params = {'field': 'type_name', 'filter': CREATED_DATE_FILTER} -req_count_df, req_count = generate_dataframe_from_api(new_req_count_api_params, new_req_count_api_params['field']) +req_count_df, req_count = generate_dataframe_from_api(new_req_count_api_params, TYPE_NAME_FIELD) # Loading the total number of request source. req_source_count_api_params = {'field': 'source_name', 'filter': CREATED_DATE_FILTER} -req_source_count_df, req_source_count = generate_dataframe_from_api(req_source_count_api_params, req_source_count_api_params['field']) +req_source_count_df, req_source_count = generate_dataframe_from_api(req_source_count_api_params, SOURCE_NAME_FIELD) # Loading the number of Request Agencies. req_agency_count_api_params = {'field': 'agency_name', 'filter': CREATED_DATE_FILTER} -req_agency_count_df, agency_count = generate_dataframe_from_api(req_agency_count_api_params, req_agency_count_api_params['field']) +req_agency_count_df, agency_count = generate_dataframe_from_api(req_agency_count_api_params, AGENCY_NAME_FIELD) # Request Share by Agency Pie Chart. req_agency_count_df = agency_count From 34c162a90adb5833b0a42d8dd47d41f36adfd297 Mon Sep 17 00:00:00 2001 From: Yi Heng Joshua Wu Date: Mon, 1 Aug 2022 21:26:01 -0700 Subject: [PATCH 47/53] Encapsulating graph into functions --- server/dash/dashboards/overview_combined.py | 240 +++++++++++++------- 1 file changed, 161 insertions(+), 79 deletions(-) diff --git a/server/dash/dashboards/overview_combined.py b/server/dash/dashboards/overview_combined.py index 3284ab905..0231818e6 100644 --- a/server/dash/dashboards/overview_combined.py +++ b/server/dash/dashboards/overview_combined.py @@ -16,7 +16,7 @@ CREATED_YEAR_FIELD = 'created_year' NON_TOP4_INDEX_START = 4 REPORT_API_PATH_ROOT = "/reports?" -REQ_TYPE_STATS_API_PATH = '/types/stats' +REQ_TYPE_STATS_API_PATH = '/types/stats' SOURCE_NAME_FIELD = 'source_name' TITLE = "OVERVIEW COMBINED DASHBOARD" TYPE_NAME_FIELD = 'type_name' @@ -28,11 +28,11 @@ def generate_dataframe_from_api(api_params, group_by_col): This function takes the "api_params" dictionary and "group_by_col" string and outputs the relevant fields in the raw dataframe and groupby dataframe from the reports API. - + Args: api_params: a dictionary of parameters for calling the API. group_by_col: the column name we use to groupby and output the result_counts_df dataframe. - + Returns: result_df: the raw dataframe from calling the api with parameters from "api_params". result_counts_df: dataframe output from result_df group by the column "group_by_col" and aggregating the sum. @@ -60,96 +60,178 @@ def generate_dataframe_from_api(api_params, group_by_col): # Loading the total number of request source. req_source_count_api_params = {'field': 'source_name', 'filter': CREATED_DATE_FILTER} -req_source_count_df, req_source_count = generate_dataframe_from_api(req_source_count_api_params, SOURCE_NAME_FIELD) +req_source_count_df, req_source_count = generate_dataframe_from_api( + req_source_count_api_params, SOURCE_NAME_FIELD) # Loading the number of Request Agencies. req_agency_count_api_params = {'field': 'agency_name', 'filter': CREATED_DATE_FILTER} -req_agency_count_df, agency_count = generate_dataframe_from_api(req_agency_count_api_params, AGENCY_NAME_FIELD) +req_agency_count_df, agency_count = generate_dataframe_from_api( + req_agency_count_api_params, AGENCY_NAME_FIELD) # Request Share by Agency Pie Chart. -req_agency_count_df = agency_count -req_agency_count_df.sort_values('counts', ascending=False, inplace=True) -req_agency_count_df.loc['Others'] = req_agency_count_df[NON_TOP4_INDEX_START:].sum() -req_agency_count_df.sort_values('counts', ascending=False, inplace=True) -req_agency_count_df = req_agency_count_df[:NON_TOP4_INDEX_START + 1] -req_agency_count_df.index = req_agency_count_df.index.map(lambda x: '
'.join(textwrap.wrap(x, width=16))) - -req_share_by_agency_pie_chart = px.pie( - req_agency_count_df, - names=req_agency_count_df.index, - values='counts', - labels=LABELS, - hole=.3, - title="Total Requests by Agency", -) -req_share_by_agency_pie_chart.update_layout(margin=dict(l=100, r=100, b=100, t=100)) +def make_agency_pie_chart(agency_count): + """Generates the request share by agency pie chart. + + This function takes the "agency_count" dataframe, aggregate all agencies with less than top 4 request counts as "Others", + and outputs a pie chart "req_share_by_agency_pie_chart" showing the share of requests by agencies. + + Args: + agency_count: a pandas dataframe with the different agencies and corresopnding request counts. + + Returns: + req_share_by_agency_pie_chart: a pie chart showing the share of requests by agencies. + """ + req_agency_count_df = agency_count + req_agency_count_df.sort_values('counts', ascending=False, inplace=True) + req_agency_count_df.loc['Others'] = req_agency_count_df[NON_TOP4_INDEX_START:].sum() + req_agency_count_df.sort_values('counts', ascending=False, inplace=True) + req_agency_count_df = req_agency_count_df[:NON_TOP4_INDEX_START + 1] + req_agency_count_df.index = req_agency_count_df.index.map( + lambda x: '
'.join(textwrap.wrap(x, width=16))) + + req_share_by_agency_pie_chart = px.pie( + req_agency_count_df, + names=req_agency_count_df.index, + values='counts', + labels=LABELS, + hole=.3, + title="Total Requests by Agency", + ) + req_share_by_agency_pie_chart.update_layout(margin=dict(l=100, r=100, b=100, t=100)) + return req_share_by_agency_pie_chart + + +req_share_by_agency_pie_chart = make_agency_pie_chart(agency_count) # Request Type by Source Bar Chart. -req_source_count_df = req_source_count -req_source_count_df.sort_values('counts', ascending=False, inplace=True) -req_source_count_df.loc['Others'] = req_source_count_df[NON_TOP4_INDEX_START:].sum() -req_source_count_df.sort_values('counts', ascending=False, inplace=True) -req_source_count_df = req_source_count_df[:NON_TOP4_INDEX_START + 1] - -req_source_bar_chart = px.bar( - req_source_count_df, - y=req_source_count_df.index, - x='counts', - title="Total Requests by Source", - orientation='h' -) +def make_req_type_source_bar_chart(req_source_count): + """Generates the request type by source bar chart. + + This function takes the "req_source_count" dataframe, aggregate all request source with less than + top 4 request counts as "Others", and outputs a horizontal bar chart "req_source_bar_chart" showing + the share of requests by sources. + + Args: + req_source_count: a pandas dataframe with the different request sources and corresopnding request counts. + + Returns: + req_source_bar_chart: a horizontal bar chart showing the share of requests by sources. + """ + req_source_count_df = req_source_count + req_source_count_df.sort_values('counts', ascending=False, inplace=True) + req_source_count_df.loc['Others'] = req_source_count_df[NON_TOP4_INDEX_START:].sum() + req_source_count_df.sort_values('counts', ascending=False, inplace=True) + req_source_count_df = req_source_count_df[:NON_TOP4_INDEX_START + 1] + + req_source_bar_chart = px.bar( + req_source_count_df, + y=req_source_count_df.index, + x='counts', + title="Total Requests by Source", + orientation='h' + ) + return req_source_bar_chart + + +req_source_bar_chart = make_req_type_source_bar_chart(req_source_count) # Median Request Days to Close Box Plot. -print(" * Downloading data from API path: " + REQ_TYPE_STATS_API_PATH) -stats_df = pd.read_json(API_HOST + REQ_TYPE_STATS_API_PATH) -stats_df = stats_df.sort_values('median', ascending=False) -print(" * Dataframe has been loaded from API path: " + REQ_TYPE_STATS_API_PATH) - -med_days_to_close_box_plot = go.Figure() -med_days_to_close_box_plot.add_trace( - go.Box( - y=stats_df.type_name, - q1=stats_df['q1'], - median=stats_df['median'], - q3=stats_df['q3'], - marker_color='#29404F', - fillcolor='#E17C05', +def make_days_to_close_box_plot(): + """Generates the request days to close box plot. + + This function calls the stats api via "REQ_TYPE_STATS_API_PATH", retrieves quartile values for request + time to close, and generates a box plot to visualize median request day to close box plot. + + Returns: + med_days_to_close_box_plot: a box plot showing the median day to close for each request type. + """ + print(" * Downloading data from API path: " + REQ_TYPE_STATS_API_PATH) + stats_df = pd.read_json(API_HOST + REQ_TYPE_STATS_API_PATH) + stats_df = stats_df.sort_values('median', ascending=False) + print(" * Dataframe has been loaded from API path: " + REQ_TYPE_STATS_API_PATH) + + med_days_to_close_box_plot = go.Figure() + med_days_to_close_box_plot.add_trace( + go.Box( + y=stats_df.type_name, + q1=stats_df['q1'], + median=stats_df['median'], + q3=stats_df['q3'], + marker_color='#29404F', + fillcolor='#E17C05', + ) + ) + med_days_to_close_box_plot.update_xaxes( + dtick=5 ) -) -med_days_to_close_box_plot.update_xaxes( - dtick=5 -) -med_days_to_close_box_plot.update_layout( - title="Total Median Days to Close by Type", -) + med_days_to_close_box_plot.update_layout( + title="Total Median Days to Close by Type", + ) + return med_days_to_close_box_plot + + +med_days_to_close_box_plot = make_days_to_close_box_plot() # Day of Week Bar Chart. -start_date = datetime.date.today() - datetime.timedelta(days=365) -end_date = datetime.date.today() - datetime.timedelta(days=1) -date_range_req_data_params = {'filter':f"created_date>={start_date}", 'filter':f"created_date<={end_date}"} -date_range_req_df, _ = generate_dataframe_from_api(date_range_req_data_params, 'created_date') -date_range_req_df['created_date'] = pd.to_datetime(date_range_req_df['created_date']) -date_range_req_df['day_of_week'] = date_range_req_df['created_date'].dt.day_name() -day_of_week_df = date_range_req_df.groupby(['day_of_week']).agg('sum').reset_index() - -num_req_by_day_bar_chart = px.bar( - day_of_week_df, - x="day_of_week", - y="counts", - labels=LABELS, -) -num_req_by_day_bar_chart.update_xaxes(categoryorder='array', categoryarray=[ - "Sunday", "Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday"]) +def make_day_of_week_bar_chart(): + """Generates the day of week bar chart. + + This function calls the reports api , retrieves 1 year worth of request data, and generates + a vertical bar chart showing the number of requests for every day of the week. + + Returns: + num_req_by_day_bar_chart: a bar chart showing the total number of requests for every day + of the week throughout the past year. + """ + start_date = datetime.date.today() - datetime.timedelta(days=365) + end_date = datetime.date.today() - datetime.timedelta(days=1) + date_range_req_data_params = { + 'filter': f"created_date>={start_date}", 'filter': f"created_date<={end_date}"} + date_range_req_df, _ = generate_dataframe_from_api(date_range_req_data_params, 'created_date') + date_range_req_df['created_date'] = pd.to_datetime(date_range_req_df['created_date']) + date_range_req_df['day_of_week'] = date_range_req_df['created_date'].dt.day_name() + day_of_week_df = date_range_req_df.groupby(['day_of_week']).agg('sum').reset_index() + + num_req_by_day_bar_chart = px.bar( + day_of_week_df, + x="day_of_week", + y="counts", + labels=LABELS, + ) + num_req_by_day_bar_chart.update_xaxes(categoryorder='array', categoryarray=[ + "Sunday", "Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday"]) + return num_req_by_day_bar_chart + +num_req_by_day_bar_chart = make_day_of_week_bar_chart() + # Total Request by NC. -req_by_nc_bar_chart = px.bar( - nc_req_count_df, - x=nc_req_count_df.index, - y='counts', - labels=LABELS, - title="Total Requests by Neighborhood Councils", -) -req_by_nc_bar_chart.update_layout(font=dict(size=12)) +def make_req_by_nc_bar_chart(nc_req_count_df): + """Generates the total request by nc bar chart. + + This function takes the "nc_req_count_df" dataframe and outputs a bar chart + "req_by_nc_bar_chart" showing the total number of requests by Neighborhood Councils (NCs). + + Args: + nc_req_count_df: a pandas dataframe with the different NCs and corresopnding request counts. + + Returns: + req_source_bar_chart: a horizontal bar chart showing the share of requests by sources. + """ + req_by_nc_bar_chart = px.bar( + nc_req_count_df, + x=nc_req_count_df.index, + y='counts', + labels=LABELS, + title="Total Requests by Neighborhood Councils", + ) + req_by_nc_bar_chart.update_layout(font=dict(size=12)) + return req_by_nc_bar_chart + +# GRAPHS + + +req_by_nc_bar_chart = make_req_by_nc_bar_chart(nc_req_count_df) # LAYOUT VARIABLES. INDICATOR_CARD_STYLE = {"text-align": 'center', From 11b8dee6be9c07eb7173b8777de63fa387f93c0e Mon Sep 17 00:00:00 2001 From: Yi Heng Joshua Wu Date: Mon, 1 Aug 2022 21:46:07 -0700 Subject: [PATCH 48/53] Add func to cal summary viz --- server/dash/dashboards/overview_combined.py | 58 +++++++++++++-------- 1 file changed, 35 insertions(+), 23 deletions(-) diff --git a/server/dash/dashboards/overview_combined.py b/server/dash/dashboards/overview_combined.py index 0231818e6..865fed3ec 100644 --- a/server/dash/dashboards/overview_combined.py +++ b/server/dash/dashboards/overview_combined.py @@ -21,8 +21,7 @@ TITLE = "OVERVIEW COMBINED DASHBOARD" TYPE_NAME_FIELD = 'type_name' - -# HELPER FUNCTIONS. +# API HELPER FUNCTION. def generate_dataframe_from_api(api_params, group_by_col): """Generates the dataframe output from the reports API. @@ -44,7 +43,7 @@ def generate_dataframe_from_api(api_params, group_by_col): print(" * Dataframe has been loaded from API path: " + DATA_API_PATH) return result_df, result_counts_df -# DATA +# DATAFRAMES # Loading the dataframe for the NCs and corresponding requests. nc_req_count_api_params = {'field': 'council_name', 'filter': CREATED_DATE_FILTER} @@ -68,6 +67,28 @@ def generate_dataframe_from_api(api_params, group_by_col): req_agency_count_df, agency_count = generate_dataframe_from_api( req_agency_count_api_params, AGENCY_NAME_FIELD) +# VISUALS HELPER FUNCTIONS. + +# Compute values for the indicator visuals. +def get_counts_dict(): + """Compute values for the indicator visuals. + + This function compute the summary statistics including number of new requests, number of neighborhood + councils, number of requests, number of request sources, number of request agencies, and store them + in a dictionary. + + Returns: + indicator_count_dict: dictionary storing the summary statistics for indicator visuals. + """ + indicator_count_dict = {} + indicator_count_dict['new_req_count'] = new_req_count_df['counts'].sum() + indicator_count_dict['nc_count'] = nc_req_count_df.shape[0] - 1 + indicator_count_dict['req_count'] = req_count_df.shape[0] + indicator_count_dict['source_count'] = req_source_count.shape[0] + indicator_count_dict['agency_count'] = agency_count.shape[0] + + return indicator_count_dict + # Request Share by Agency Pie Chart. def make_agency_pie_chart(agency_count): """Generates the request share by agency pie chart. @@ -100,9 +121,6 @@ def make_agency_pie_chart(agency_count): req_share_by_agency_pie_chart.update_layout(margin=dict(l=100, r=100, b=100, t=100)) return req_share_by_agency_pie_chart - -req_share_by_agency_pie_chart = make_agency_pie_chart(agency_count) - # Request Type by Source Bar Chart. def make_req_type_source_bar_chart(req_source_count): """Generates the request type by source bar chart. @@ -132,9 +150,6 @@ def make_req_type_source_bar_chart(req_source_count): ) return req_source_bar_chart - -req_source_bar_chart = make_req_type_source_bar_chart(req_source_count) - # Median Request Days to Close Box Plot. def make_days_to_close_box_plot(): """Generates the request days to close box plot. @@ -169,9 +184,6 @@ def make_days_to_close_box_plot(): ) return med_days_to_close_box_plot - -med_days_to_close_box_plot = make_days_to_close_box_plot() - # Day of Week Bar Chart. def make_day_of_week_bar_chart(): """Generates the day of week bar chart. @@ -202,9 +214,6 @@ def make_day_of_week_bar_chart(): "Sunday", "Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday"]) return num_req_by_day_bar_chart -num_req_by_day_bar_chart = make_day_of_week_bar_chart() - - # Total Request by NC. def make_req_by_nc_bar_chart(nc_req_count_df): """Generates the total request by nc bar chart. @@ -228,9 +237,12 @@ def make_req_by_nc_bar_chart(nc_req_count_df): req_by_nc_bar_chart.update_layout(font=dict(size=12)) return req_by_nc_bar_chart -# GRAPHS - - +# VISUALS. +indicator_count_dict = get_counts_dict() +req_share_by_agency_pie_chart = make_agency_pie_chart(agency_count) +req_source_bar_chart = make_req_type_source_bar_chart(req_source_count) +med_days_to_close_box_plot = make_days_to_close_box_plot() +num_req_by_day_bar_chart = make_day_of_week_bar_chart() req_by_nc_bar_chart = make_req_by_nc_bar_chart(nc_req_count_df) # LAYOUT VARIABLES. @@ -250,15 +262,15 @@ def make_req_by_nc_bar_chart(nc_req_count_df): html.P(DASHBOARD_OUTLINE, style={'font-size': '18px', 'font-style': 'italic'}), html.Div([ - html.Div([html.H2(f"{new_req_count_df['counts'].sum():,}"), html.Label( + html.Div([html.H2(f"{indicator_count_dict['new_req_count']:,}"), html.Label( "Total Requests")], style=INDICATOR_CARD_STYLE), - html.Div([html.H2(nc_req_count_df.shape[0] - 1), html.Label("Neighborhoods")], + html.Div([html.H2(indicator_count_dict['nc_count']), html.Label("Neighborhoods")], style=INDICATOR_CARD_STYLE), - html.Div([html.H2(req_count_df.shape[0]), html.Label( + html.Div([html.H2(indicator_count_dict['req_count']), html.Label( "Request Types")], style=INDICATOR_CARD_STYLE), - html.Div([html.H2(req_source_count.shape[0]), html.Label( + html.Div([html.H2(indicator_count_dict['source_count']), html.Label( "Request Source")], style=INDICATOR_CARD_STYLE), - html.Div([html.H2(agency_count.shape[0]), html.Label( + html.Div([html.H2(indicator_count_dict['agency_count']), html.Label( "Request Agency")], style=INDICATOR_CARD_STYLE) ], style=EQUAL_SPACE_BOX_STYLE), From 2a082d195f5775e74b3e5fdfc939e829be12b426 Mon Sep 17 00:00:00 2001 From: Yi Heng Joshua Wu Date: Mon, 1 Aug 2022 22:11:09 -0700 Subject: [PATCH 49/53] Resolved groupby type error --- server/dash/dashboards/overview_combined.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/dash/dashboards/overview_combined.py b/server/dash/dashboards/overview_combined.py index 865fed3ec..3917dd8ba 100644 --- a/server/dash/dashboards/overview_combined.py +++ b/server/dash/dashboards/overview_combined.py @@ -55,7 +55,7 @@ def generate_dataframe_from_api(api_params, group_by_col): # Loading the count of each request types overall. req_count_api_params = {'field': 'type_name', 'filter': CREATED_DATE_FILTER} -req_count_df, req_count = generate_dataframe_from_api(new_req_count_api_params, TYPE_NAME_FIELD) +req_count_df, req_count = generate_dataframe_from_api(req_count_api_params, TYPE_NAME_FIELD) # Loading the total number of request source. req_source_count_api_params = {'field': 'source_name', 'filter': CREATED_DATE_FILTER} From 05d1d97f6f8caba79c7c12d3de69af59d9708368 Mon Sep 17 00:00:00 2001 From: Yi Heng Joshua Wu Date: Wed, 3 Aug 2022 19:33:46 -0700 Subject: [PATCH 50/53] Replace visual with visualizations --- server/dash/dashboards/overview_combined.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/server/dash/dashboards/overview_combined.py b/server/dash/dashboards/overview_combined.py index 3917dd8ba..322b6a7d9 100644 --- a/server/dash/dashboards/overview_combined.py +++ b/server/dash/dashboards/overview_combined.py @@ -67,18 +67,18 @@ def generate_dataframe_from_api(api_params, group_by_col): req_agency_count_df, agency_count = generate_dataframe_from_api( req_agency_count_api_params, AGENCY_NAME_FIELD) -# VISUALS HELPER FUNCTIONS. +# VISUALIZATION HELPER FUNCTIONS. -# Compute values for the indicator visuals. +# Compute values for the indicator visualization. def get_counts_dict(): - """Compute values for the indicator visuals. + """Compute values for the indicator visualization. This function compute the summary statistics including number of new requests, number of neighborhood councils, number of requests, number of request sources, number of request agencies, and store them in a dictionary. Returns: - indicator_count_dict: dictionary storing the summary statistics for indicator visuals. + indicator_count_dict: dictionary storing the summary statistics for indicator visualization. """ indicator_count_dict = {} indicator_count_dict['new_req_count'] = new_req_count_df['counts'].sum() @@ -237,7 +237,7 @@ def make_req_by_nc_bar_chart(nc_req_count_df): req_by_nc_bar_chart.update_layout(font=dict(size=12)) return req_by_nc_bar_chart -# VISUALS. +# VISUALIZATION. indicator_count_dict = get_counts_dict() req_share_by_agency_pie_chart = make_agency_pie_chart(agency_count) req_source_bar_chart = make_req_type_source_bar_chart(req_source_count) From 865f581357de92a582a8d58e8e83b27d96cf3d5b Mon Sep 17 00:00:00 2001 From: Yi Heng Joshua Wu Date: Wed, 3 Aug 2022 19:35:46 -0700 Subject: [PATCH 51/53] Remove return var in docstring --- server/dash/dashboards/overview_combined.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/server/dash/dashboards/overview_combined.py b/server/dash/dashboards/overview_combined.py index 322b6a7d9..592a58a9a 100644 --- a/server/dash/dashboards/overview_combined.py +++ b/server/dash/dashboards/overview_combined.py @@ -78,7 +78,7 @@ def get_counts_dict(): in a dictionary. Returns: - indicator_count_dict: dictionary storing the summary statistics for indicator visualization. + A dictionary storing the summary statistics for indicator visualization. """ indicator_count_dict = {} indicator_count_dict['new_req_count'] = new_req_count_df['counts'].sum() @@ -100,7 +100,7 @@ def make_agency_pie_chart(agency_count): agency_count: a pandas dataframe with the different agencies and corresopnding request counts. Returns: - req_share_by_agency_pie_chart: a pie chart showing the share of requests by agencies. + A pie chart showing the share of requests by agencies. """ req_agency_count_df = agency_count req_agency_count_df.sort_values('counts', ascending=False, inplace=True) @@ -133,7 +133,7 @@ def make_req_type_source_bar_chart(req_source_count): req_source_count: a pandas dataframe with the different request sources and corresopnding request counts. Returns: - req_source_bar_chart: a horizontal bar chart showing the share of requests by sources. + A horizontal bar chart showing the share of requests by sources. """ req_source_count_df = req_source_count req_source_count_df.sort_values('counts', ascending=False, inplace=True) @@ -158,7 +158,7 @@ def make_days_to_close_box_plot(): time to close, and generates a box plot to visualize median request day to close box plot. Returns: - med_days_to_close_box_plot: a box plot showing the median day to close for each request type. + A box plot showing the median day to close for each request type. """ print(" * Downloading data from API path: " + REQ_TYPE_STATS_API_PATH) stats_df = pd.read_json(API_HOST + REQ_TYPE_STATS_API_PATH) @@ -188,12 +188,11 @@ def make_days_to_close_box_plot(): def make_day_of_week_bar_chart(): """Generates the day of week bar chart. - This function calls the reports api , retrieves 1 year worth of request data, and generates + This function calls the reports api, retrieves 1 year worth of request data, and generates a vertical bar chart showing the number of requests for every day of the week. Returns: - num_req_by_day_bar_chart: a bar chart showing the total number of requests for every day - of the week throughout the past year. + A bar chart showing the total number of requests for every day of the week throughout the past year. """ start_date = datetime.date.today() - datetime.timedelta(days=365) end_date = datetime.date.today() - datetime.timedelta(days=1) @@ -225,7 +224,7 @@ def make_req_by_nc_bar_chart(nc_req_count_df): nc_req_count_df: a pandas dataframe with the different NCs and corresopnding request counts. Returns: - req_source_bar_chart: a horizontal bar chart showing the share of requests by sources. + A horizontal bar chart showing the share of requests by sources. """ req_by_nc_bar_chart = px.bar( nc_req_count_df, From 443e85273c67b2c2b5db014b9c778519db3a1d80 Mon Sep 17 00:00:00 2001 From: Yi Heng Joshua Wu Date: Wed, 3 Aug 2022 19:37:12 -0700 Subject: [PATCH 52/53] Remove excessive comments --- server/dash/dashboards/overview_combined.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/server/dash/dashboards/overview_combined.py b/server/dash/dashboards/overview_combined.py index 592a58a9a..f963285a2 100644 --- a/server/dash/dashboards/overview_combined.py +++ b/server/dash/dashboards/overview_combined.py @@ -69,7 +69,6 @@ def generate_dataframe_from_api(api_params, group_by_col): # VISUALIZATION HELPER FUNCTIONS. -# Compute values for the indicator visualization. def get_counts_dict(): """Compute values for the indicator visualization. @@ -89,7 +88,6 @@ def get_counts_dict(): return indicator_count_dict -# Request Share by Agency Pie Chart. def make_agency_pie_chart(agency_count): """Generates the request share by agency pie chart. @@ -121,7 +119,6 @@ def make_agency_pie_chart(agency_count): req_share_by_agency_pie_chart.update_layout(margin=dict(l=100, r=100, b=100, t=100)) return req_share_by_agency_pie_chart -# Request Type by Source Bar Chart. def make_req_type_source_bar_chart(req_source_count): """Generates the request type by source bar chart. @@ -150,7 +147,6 @@ def make_req_type_source_bar_chart(req_source_count): ) return req_source_bar_chart -# Median Request Days to Close Box Plot. def make_days_to_close_box_plot(): """Generates the request days to close box plot. @@ -184,7 +180,6 @@ def make_days_to_close_box_plot(): ) return med_days_to_close_box_plot -# Day of Week Bar Chart. def make_day_of_week_bar_chart(): """Generates the day of week bar chart. @@ -213,7 +208,6 @@ def make_day_of_week_bar_chart(): "Sunday", "Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday"]) return num_req_by_day_bar_chart -# Total Request by NC. def make_req_by_nc_bar_chart(nc_req_count_df): """Generates the total request by nc bar chart. From ff4eafbf29ebcc57d0fb9f93f4cba241e174ac8a Mon Sep 17 00:00:00 2001 From: Yi Heng Joshua Wu Date: Wed, 3 Aug 2022 19:47:49 -0700 Subject: [PATCH 53/53] Place variable in func --- server/dash/dashboards/overview_combined.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/server/dash/dashboards/overview_combined.py b/server/dash/dashboards/overview_combined.py index f963285a2..8ebd22c1f 100644 --- a/server/dash/dashboards/overview_combined.py +++ b/server/dash/dashboards/overview_combined.py @@ -49,22 +49,14 @@ def generate_dataframe_from_api(api_params, group_by_col): nc_req_count_api_params = {'field': 'council_name', 'filter': CREATED_DATE_FILTER} _, nc_req_count_df = generate_dataframe_from_api(nc_req_count_api_params, COUNCIL_NAME_FIELD) -# Loading the data for the number of new requests. -new_req_count_api_params = {'field': 'created_year', 'filter': CREATED_DATE_FILTER} -_, new_req_count_df = generate_dataframe_from_api(new_req_count_api_params, CREATED_YEAR_FIELD) - -# Loading the count of each request types overall. -req_count_api_params = {'field': 'type_name', 'filter': CREATED_DATE_FILTER} -req_count_df, req_count = generate_dataframe_from_api(req_count_api_params, TYPE_NAME_FIELD) - # Loading the total number of request source. req_source_count_api_params = {'field': 'source_name', 'filter': CREATED_DATE_FILTER} -req_source_count_df, req_source_count = generate_dataframe_from_api( +_, req_source_count = generate_dataframe_from_api( req_source_count_api_params, SOURCE_NAME_FIELD) # Loading the number of Request Agencies. req_agency_count_api_params = {'field': 'agency_name', 'filter': CREATED_DATE_FILTER} -req_agency_count_df, agency_count = generate_dataframe_from_api( +_, agency_count = generate_dataframe_from_api( req_agency_count_api_params, AGENCY_NAME_FIELD) # VISUALIZATION HELPER FUNCTIONS. @@ -80,6 +72,15 @@ def get_counts_dict(): A dictionary storing the summary statistics for indicator visualization. """ indicator_count_dict = {} + + # Loading the data for the number of new requests. + new_req_count_api_params = {'field': 'created_year', 'filter': CREATED_DATE_FILTER} + _, new_req_count_df = generate_dataframe_from_api(new_req_count_api_params, CREATED_YEAR_FIELD) + + # Loading the count of each request types overall. + req_count_api_params = {'field': 'type_name', 'filter': CREATED_DATE_FILTER} + req_count_df, _ = generate_dataframe_from_api(req_count_api_params, TYPE_NAME_FIELD) + indicator_count_dict['new_req_count'] = new_req_count_df['counts'].sum() indicator_count_dict['nc_count'] = nc_req_count_df.shape[0] - 1 indicator_count_dict['req_count'] = req_count_df.shape[0]