From fdbd206a8d8c65eccd061a25aa86271cb6d2a5c6 Mon Sep 17 00:00:00 2001 From: funbunch Date: Tue, 7 Jun 2022 21:37:08 -0700 Subject: [PATCH 01/37] Updated text on contact form. --- client/components/main/ContactForm.jsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/components/main/ContactForm.jsx b/client/components/main/ContactForm.jsx index 379e7b306..d0dad2dae 100644 --- a/client/components/main/ContactForm.jsx +++ b/client/components/main/ContactForm.jsx @@ -58,7 +58,7 @@ const ContactForm = () => { { state.succeeded && (
- Thank you for signing up! We will get back to you in 2-3 business days. + Thanks for contacting us! We will get back to you in 2-3 business days.
)} { !state.succeeded From ade1445c539c800f22610b725d7f1191128e2561 Mon Sep 17 00:00:00 2001 From: Yi Heng Joshua Wu Date: Wed, 22 Jun 2022 17:55:48 -0700 Subject: [PATCH 02/37] Resolve typeName key error --- server/dash/dashboards/neighborhood_recent.py | 40 +++++++++++++++---- 1 file changed, 33 insertions(+), 7 deletions(-) diff --git a/server/dash/dashboards/neighborhood_recent.py b/server/dash/dashboards/neighborhood_recent.py index 9ece97e37..97b2bec39 100644 --- a/server/dash/dashboards/neighborhood_recent.py +++ b/server/dash/dashboards/neighborhood_recent.py @@ -4,13 +4,15 @@ import dash_core_components as dcc import dash_html_components as html import dash_table +from dash.dependencies import Input, Output import pandas as pd import plotly.express as px +from flask import request + from app import app, batch_get_data from config import API_HOST -from dash.dependencies import Input, Output from design import CONFIG_OPTIONS, DISCRETE_COLORS, LABELS, apply_figure_style, DISCRETE_COLORS_MAP -from flask import request + pretty_columns = { 'srnumber': "SR Number", @@ -44,10 +46,34 @@ selected_council = 'Arleta' table_df = df.query(f"councilName == '{selected_council}'")[['srnumber', 'createdDate', 'closedDate', 'typeName', 'agencyName', 'sourceName', 'address']] # noqa -figure_df = df.query(f"councilName == '{selected_council}' and createdDate >= '{start_date}'").groupby(['createdDate', 'typeName'])['srnumber'].count().reset_index() # noqa +figure_df = df.query(f"councilName == '{selected_council}' and createdDate >= '{start_date}'").groupby(['createdDate', 'typeName'], as_index=False)['srnumber'].count() # noqa + +# Reload data query from 3 weeks ago if the current dataframe is empty +if figure_df.shape[0] == 0: + start_date = datetime.date.today() - datetime.timedelta(days=28) + end_date = datetime.date.today() - datetime.timedelta(days=21) + df_path = f"/requests/updated?start_date={start_date}&end_date={end_date}" + print(" * Downloading data for dataframe") + df = batch_get_data(API_HOST + df_path) + df['createdDate'] = pd.to_datetime( + df['createdDate'], errors='coerce').dt.strftime('%Y-%m-%d') + df['closedDate'] = pd.to_datetime( + df['closedDate'], errors='coerce').dt.strftime('%Y-%m-%d') + print(" * Dataframe has been loaded") + try: + selected_council = request.args.get('councilName') or 'Arleta' + except (RuntimeError): + selected_council = 'Arleta' + table_df = df.query(f"councilName == '{selected_council}'")[['srnumber', 'createdDate', 'closedDate', 'typeName', 'agencyName', 'sourceName', 'address']] # noqa + figure_df = df.query(f"councilName == '{selected_council}' and createdDate >= '{start_date}'").groupby(['createdDate', 'typeName'], as_index=False)['srnumber'].count() # noqa + + +print(figure_df.head()) # Populate the neighborhood dropdown + + def populate_options(): council_df_path = '/councils' council_df = pd.read_json(API_HOST + council_df_path) @@ -66,8 +92,8 @@ def populate_options(): y="srnumber", color="typeName", color_discrete_sequence=DISCRETE_COLORS, - color_discrete_map = DISCRETE_COLORS_MAP, - labels=LABELS, + color_discrete_map=DISCRETE_COLORS_MAP, + labels=LABELS ) fig.update_xaxes( @@ -84,9 +110,9 @@ def populate_options(): figure_df, names="typeName", values="srnumber", - color = 'typeName', + color="typeName", color_discrete_sequence=DISCRETE_COLORS, - color_discrete_map = DISCRETE_COLORS_MAP, + color_discrete_map=DISCRETE_COLORS_MAP, labels=LABELS, hole=.3, ) From 07b5a8472a08b4b353b3644c3fc69deb756d4eea Mon Sep 17 00:00:00 2001 From: Yi Heng Joshua Wu Date: Fri, 24 Jun 2022 14:27:08 -0700 Subject: [PATCH 03/37] Resolve empty data edge case --- server/dash/dashboards/neighborhood_recent.py | 88 ++++++------------- 1 file changed, 26 insertions(+), 62 deletions(-) diff --git a/server/dash/dashboards/neighborhood_recent.py b/server/dash/dashboards/neighborhood_recent.py index 40cfbf61d..d5a4fdfd0 100644 --- a/server/dash/dashboards/neighborhood_recent.py +++ b/server/dash/dashboards/neighborhood_recent.py @@ -24,8 +24,8 @@ 'address': "Address" } -start_date = datetime.date.today() - datetime.timedelta(days=7) -end_date = datetime.date.today() - datetime.timedelta(days=1) +start_date = datetime.date.today() + datetime.timedelta(days=3) +end_date = datetime.date.today() + datetime.timedelta(days=14) # TITLE title = "NEIGHBORHOOD WEEKLY REPORT" @@ -46,29 +46,15 @@ selected_council = 'Arleta' table_df = df.query(f"councilName == '{selected_council}'")[['srnumber', 'createdDate', 'closedDate', 'typeName', 'agencyName', 'sourceName', 'address']] # noqa -figure_df = df.query(f"councilName == '{selected_council}' and createdDate >= '{start_date}'").groupby(['createdDate', 'typeName'], as_index=False)['srnumber'].count() # noqa - -# Reload data query from 3 weeks ago if the current dataframe is empty -if figure_df.shape[0] == 0: - start_date = datetime.date.today() - datetime.timedelta(days=28) - end_date = datetime.date.today() - datetime.timedelta(days=21) - df_path = f"/requests/updated?start_date={start_date}&end_date={end_date}" - print(" * Downloading data for dataframe") - df = batch_get_data(API_HOST + df_path) - df['createdDate'] = pd.to_datetime( - df['createdDate'], errors='coerce').dt.strftime('%Y-%m-%d') - df['closedDate'] = pd.to_datetime( - df['closedDate'], errors='coerce').dt.strftime('%Y-%m-%d') - print(" * Dataframe has been loaded") - try: - selected_council = request.args.get('councilName') or 'Arleta' - except (RuntimeError): - selected_council = 'Arleta' - - table_df = df.query(f"councilName == '{selected_council}'")[['srnumber', 'createdDate', 'closedDate', 'typeName', 'agencyName', 'sourceName', 'address']] # noqa - figure_df = df.query(f"councilName == '{selected_council}' and createdDate >= '{start_date}'").groupby(['createdDate', 'typeName'], as_index=False)['srnumber'].count() # noqa +# figure_df = df.query(f"councilName == '{selected_council}' and createdDate >= '{start_date}'").groupby(['createdDate', 'typeName'], as_index=False)['srnumber'].count() # noqa # Populate the neighborhood dropdown +fig = px.line() +apply_figure_style(fig) + +fig2 = px.pie() +apply_figure_style(fig2) + def populate_options(): council_df_path = '/councils' council_df = pd.read_json(API_HOST + council_df_path) @@ -80,40 +66,6 @@ def populate_options(): }) return values - -fig = px.line( - figure_df, - x="createdDate", - y="srnumber", - color="typeName", - color_discrete_sequence=DISCRETE_COLORS, - color_discrete_map=DISCRETE_COLORS_MAP, - labels=LABELS -) - -fig.update_xaxes( - tickformat="%a\n%m/%d", -) - -fig.update_traces( - mode='markers+lines' -) # add markers to lines - -apply_figure_style(fig) - -pie_fig = px.pie( - figure_df, - names="typeName", - values="srnumber", - color="typeName", - color_discrete_sequence=DISCRETE_COLORS, - color_discrete_map=DISCRETE_COLORS_MAP, - labels=LABELS, - hole=.3, -) -apply_figure_style(pie_fig) - - # Layout layout = html.Div([ html.H1(title), @@ -141,11 +93,11 @@ def populate_options(): ], className="graph-row"), html.Div([ html.Div( - dcc.Graph(id='graph', figure=fig, config=CONFIG_OPTIONS), + dcc.Graph(id='graph', figure=fig ,config=CONFIG_OPTIONS), className="half-graph" ), html.Div( - dcc.Graph(id='pie_graph', figure=pie_fig, config=CONFIG_OPTIONS), + dcc.Graph(id='pie_graph', figure=fig2 , config=CONFIG_OPTIONS), className="half-graph" ) ]), @@ -183,7 +135,11 @@ def populate_options(): ) def update_table(selected_council): table_df = df.query(f"councilName == '{selected_council}'")[['srnumber', 'createdDate', 'closedDate', 'typeName', 'agencyName', 'sourceName', 'address']] # noqa - return table_df.to_dict('records') + if table_df.shape[0] == 0: + table_df = pd.DataFrame(columns=["Request Type"]) + table_df.loc[0] = ["There is no data right now"] + else: + return table_df.to_dict('records') @app.callback( @@ -197,7 +153,10 @@ def update_table(selected_council): def update_text(selected_council): create_count = df.query(f"councilName == '{selected_council}' and createdDate >= '{start_date}'")['srnumber'].count() # noqa close_count = df.query(f"councilName == '{selected_council}' and closedDate >= '{start_date}'")['srnumber'].count() # noqa - return create_count, close_count, create_count - close_count + if create_count == 0 and close_count == 0: + return 0, 0, 0 + else: + return create_count, close_count, create_count - close_count @app.callback( @@ -207,7 +166,9 @@ def update_text(selected_council): def update_figure(selected_council): figure_df = df.query(f"councilName == '{selected_council}' and createdDate >= '{start_date}'").groupby(['createdDate', 'typeName'])['srnumber'].count().reset_index() # noqa figure_df.typeName = figure_df.typeName.map(lambda x: '
'.join(textwrap.wrap(x, width=16))) # noqa - + if figure_df.shape[0] == 0: + figure_df = pd.DataFrame(columns = ['createdDate', "srnumber", "typeName"]) + figure_df.loc[0] = [start_date, 12345678, "No Request at this time"] fig = px.line( figure_df, x="createdDate", @@ -235,6 +196,9 @@ def update_figure(selected_council): ) def update_council_figure(selected_council): pie_df = df.query(f"councilName == '{selected_council}' and createdDate >= '{start_date}'").groupby(['typeName']).agg('count').reset_index() # noqa + if pie_df.shape[0] == 0: + pie_df = pd.DataFrame(columns = ["srnumber", "typeName"]) + pie_df.loc[0] = [12345678, "No Request at this time"] pie_fig = px.pie( pie_df, From 80444a335c69ffa05de6117b518a5728a1ea291f Mon Sep 17 00:00:00 2001 From: Nicholas Kwon Date: Mon, 27 Jun 2022 12:06:00 -0700 Subject: [PATCH 04/37] add some debug logging and the filter --- client/components/Map/layers/RequestsLayer.js | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/client/components/Map/layers/RequestsLayer.js b/client/components/Map/layers/RequestsLayer.js index e315c3aca..7098e60f2 100644 --- a/client/components/Map/layers/RequestsLayer.js +++ b/client/components/Map/layers/RequestsLayer.js @@ -40,6 +40,7 @@ class RequestsLayer extends React.Component { } componentDidUpdate(prev) { + console.log("component did update") const { activeLayer, selectedTypes, @@ -50,9 +51,10 @@ class RequestsLayer extends React.Component { if (activeLayer !== prev.activeLayer) this.setActiveLayer(activeLayer); - if (selectedTypes !== prev.selectedTypes) + if (selectedTypes !== prev.selectedTypes) { + console.log("selected types changed. new types:" + selectedTypes) this.setSelectedTypes(selectedTypes); - + } if (requests !== prev.requests && this.ready) this.setRequests(requests); @@ -94,7 +96,7 @@ class RequestsLayer extends React.Component { 'circle-color': circleColors(requestTypes), 'circle-opacity': 0.8, }, - // filter: typeFilter(selectedTypes), + filter: typeFilter(selectedTypes), }, BEFORE_ID); // this.map.addLayer({ @@ -112,7 +114,7 @@ class RequestsLayer extends React.Component { }; setActiveLayer = activeLayer => { - switch(activeLayer) { + switch (activeLayer) { case 'points': this.map.setLayoutProperty('request-circles', 'visibility', 'visible'); // this.map.setLayoutProperty('request-heatmap', 'visibility', 'none'); @@ -129,6 +131,7 @@ class RequestsLayer extends React.Component { }; setSelectedTypes = selectedTypes => { + console.log("setting new requested types") this.map.setFilter('request-circles', typeFilter(selectedTypes)); this.map.setFilter('request-heatmap', typeFilter(selectedTypes)); }; From 708d15c890c1a272188d86633bba0e4867351ee4 Mon Sep 17 00:00:00 2001 From: Nicholas Kwon Date: Wed, 29 Jun 2022 13:59:10 -0700 Subject: [PATCH 05/37] get requestType selector to work --- client/components/Map/Map.jsx | 39 ++++++++++--------- client/components/Map/layers/RequestsLayer.js | 25 ++++++++---- client/package-lock.json | 4 +- 3 files changed, 40 insertions(+), 28 deletions(-) diff --git a/client/components/Map/Map.jsx b/client/components/Map/Map.jsx index b876355ae..56cdd2cae 100644 --- a/client/components/Map/Map.jsx +++ b/client/components/Map/Map.jsx @@ -178,7 +178,7 @@ class Map extends React.Component { if ( this.state.filterGeo !== prevState.filterGeo || this.state.selectedTypes !== prevState.selectedTypes - ) this.map.once('idle', this.setFilteredRequestCounts); + ) this.map.once('idle', this.setFilteredRequestCounts); if (this.props.ncBoundaries != prevProps.ncBoundaries) { this.ncLayer.init({ @@ -235,14 +235,14 @@ class Map extends React.Component { filterGeo: geo, ...( center - ? { - locationInfo: { - location: `${center.lat.toFixed(6)} N ${center.lng.toFixed(6)} E`, - radius: 1, - nc: ncInfoFromLngLat(center), + ? { + locationInfo: { + location: `${center.lat.toFixed(6)} N ${center.lng.toFixed(6)} E`, + radius: 1, + nc: ncInfoFromLngLat(center), + } } - } - : {} + : {} ) }), }); @@ -335,7 +335,7 @@ class Map extends React.Component { const feature = features[i]; if (hoverables.includes(feature.layer.id) && !feature.state.selected) { - switch(feature.layer.id) { + switch (feature.layer.id) { case 'nc-fills': this.setState({ address: null }); updateNcId(feature.properties.council_id); @@ -367,12 +367,12 @@ class Map extends React.Component { updateNcId(result.id); } else { const address = result.place_name - .split(',') - .slice(0, -2) - .join(', '); - + .split(',') + .slice(0, -2) + .join(', '); + getNc({ longitude: result.center[0], latitude: result.center[1] }); - + this.setState({ address: address, }); @@ -409,7 +409,7 @@ class Map extends React.Component { getDistrictCounts = (geoFilterType, filterGeo, selectedTypes) => { const { ncCounts, ccCounts } = this.props; const { counts, regionId } = (() => { - switch(geoFilterType) { + switch (geoFilterType) { case GEO_FILTER_TYPES.nc: return { counts: ncCounts, regionId: filterGeo.properties.nc_id, @@ -534,7 +534,7 @@ class Map extends React.Component {
this.requestDetail = el}>
- { this.state.mapReady && requestTypes && ( + {this.state.mapReady && requestTypes && ( <> {/* } - + {/* ({ updateNcId: id => dispatch(updateNcId(id)), }); -export default connect(mapStateToProps, mapDispatchToProps)(withStyles(styles)(Map)); +// We need to specify forwardRef to allow refs on connected components. +// See https://github.com/reduxjs/react-redux/issues/1291#issuecomment-494185126 +// for more info. +export default connect(mapStateToProps, mapDispatchToProps, null, { forwardRef: true })(withStyles(styles)(Map)); diff --git a/client/components/Map/layers/RequestsLayer.js b/client/components/Map/layers/RequestsLayer.js index 7098e60f2..13b3acee0 100644 --- a/client/components/Map/layers/RequestsLayer.js +++ b/client/components/Map/layers/RequestsLayer.js @@ -2,6 +2,7 @@ import React from 'react'; import PropTypes from 'prop-types'; +import { connect } from 'react-redux'; // put layer underneath this layer (from original mapbox tiles) // so you don't cover up important labels @@ -19,10 +20,13 @@ function circleColors(requestTypes) { } function typeFilter(selectedTypes) { + // selectedTypes maps ints (in string form) to booleans, indicating whether the type is selected. + // Get an array of int typeIds that only includes typeIds whose corresponding value is true in selectedTypes. + var trueTypes = Object.keys(selectedTypes).map((type) => parseInt(type)).filter((type) => selectedTypes[type]); return [ 'in', - ['get', 'type'], - ['literal', selectedTypes], + ['get', 'typeId'], + ['literal', trueTypes] ]; } @@ -40,7 +44,6 @@ class RequestsLayer extends React.Component { } componentDidUpdate(prev) { - console.log("component did update") const { activeLayer, selectedTypes, @@ -52,7 +55,6 @@ class RequestsLayer extends React.Component { this.setActiveLayer(activeLayer); if (selectedTypes !== prev.selectedTypes) { - console.log("selected types changed. new types:" + selectedTypes) this.setSelectedTypes(selectedTypes); } if (requests !== prev.requests && this.ready) @@ -131,9 +133,9 @@ class RequestsLayer extends React.Component { }; setSelectedTypes = selectedTypes => { - console.log("setting new requested types") this.map.setFilter('request-circles', typeFilter(selectedTypes)); - this.map.setFilter('request-heatmap', typeFilter(selectedTypes)); + // Currently, we do not support heatmap. If we did, we'd want to update + // its filter here as well. }; setRequests = requests => { @@ -153,8 +155,6 @@ class RequestsLayer extends React.Component { } } -export default RequestsLayer; - RequestsLayer.propTypes = { activeLayer: PropTypes.oneOf(['points', 'heatmap']), selectedTypes: PropTypes.shape({}), @@ -168,3 +168,12 @@ RequestsLayer.defaultProps = { requests: {}, colorScheme: '', }; + +const mapStateToProps = state => ({ + selectedTypes: state.filters.requestTypes +}); + +// We need to specify forwardRef to allow refs on connected components. +// See https://github.com/reduxjs/react-redux/issues/1291#issuecomment-494185126 +// for more info. +export default connect(mapStateToProps, null, null, { forwardRef: true })(RequestsLayer); diff --git a/client/package-lock.json b/client/package-lock.json index 75ae87dae..037bd7588 100644 --- a/client/package-lock.json +++ b/client/package-lock.json @@ -8385,7 +8385,7 @@ "functional-red-black-tree": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/functional-red-black-tree/-/functional-red-black-tree-1.0.1.tgz", - "integrity": "sha1-GwqzvVU7Kg1jmdKcDj6gslIHgyc=", + "integrity": "sha512-dsKNQNdj6xA3T+QlADDA7mOSlX0qiMINjn0cgr+eGHGsbSHzTabcIogz2+p/iqP1Xs6EP/sS2SbqH+brGTbq0g==", "dev": true }, "functions-have-names": { @@ -10549,7 +10549,7 @@ "json-stable-stringify-without-jsonify": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz", - "integrity": "sha1-nbe1lJatPzz+8wp1FC0tkwrXJlE=", + "integrity": "sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==", "dev": true }, "json-stringify-safe": { From d9f75fa38f1049cd0d75e9ed5cb760e4050cddf4 Mon Sep 17 00:00:00 2001 From: Nicholas Kwon Date: Wed, 29 Jun 2022 14:01:45 -0700 Subject: [PATCH 06/37] clarify comment --- client/components/Map/layers/RequestsLayer.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/components/Map/layers/RequestsLayer.js b/client/components/Map/layers/RequestsLayer.js index 13b3acee0..585a0bef1 100644 --- a/client/components/Map/layers/RequestsLayer.js +++ b/client/components/Map/layers/RequestsLayer.js @@ -21,7 +21,7 @@ function circleColors(requestTypes) { function typeFilter(selectedTypes) { // selectedTypes maps ints (in string form) to booleans, indicating whether the type is selected. - // Get an array of int typeIds that only includes typeIds whose corresponding value is true in selectedTypes. + // Get an array of int typeIds corresponding value in selectedTypes is true. var trueTypes = Object.keys(selectedTypes).map((type) => parseInt(type)).filter((type) => selectedTypes[type]); return [ 'in', From 1cc16ea438dd57a3600234c8aaf753bb47893af2 Mon Sep 17 00:00:00 2001 From: Yi Heng Joshua Wu Date: Wed, 29 Jun 2022 18:44:00 -0700 Subject: [PATCH 07/37] Fill in dummy when no data available --- server/dash/dashboards/neighborhood_recent.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/server/dash/dashboards/neighborhood_recent.py b/server/dash/dashboards/neighborhood_recent.py index d5a4fdfd0..4e1a2b533 100644 --- a/server/dash/dashboards/neighborhood_recent.py +++ b/server/dash/dashboards/neighborhood_recent.py @@ -137,7 +137,9 @@ def update_table(selected_council): table_df = df.query(f"councilName == '{selected_council}'")[['srnumber', 'createdDate', 'closedDate', 'typeName', 'agencyName', 'sourceName', 'address']] # noqa if table_df.shape[0] == 0: table_df = pd.DataFrame(columns=["Request Type"]) - table_df.loc[0] = ["There is no data right now"] + #table_df.loc[0] = ["There is no data right now"] + for i in range(len(DISCRETE_COLORS_MAP.keys())): + table_df.loc[i] = [list(DISCRETE_COLORS_MAP.keys())[i]] else: return table_df.to_dict('records') @@ -168,7 +170,9 @@ def update_figure(selected_council): figure_df.typeName = figure_df.typeName.map(lambda x: '
'.join(textwrap.wrap(x, width=16))) # noqa if figure_df.shape[0] == 0: figure_df = pd.DataFrame(columns = ['createdDate', "srnumber", "typeName"]) - figure_df.loc[0] = [start_date, 12345678, "No Request at this time"] + for i in range(len(DISCRETE_COLORS_MAP.keys())): + figure_df.loc[i] = [start_date + datetime.timedelta(days=i), 12345678, list(DISCRETE_COLORS_MAP.keys())[i]] + #figure_df.loc[0] = [start_date, 12345678, "No Request at this time"] fig = px.line( figure_df, x="createdDate", @@ -191,15 +195,19 @@ def update_figure(selected_council): @app.callback( - Output("pie_graph", "pie_fig"), + Output("pie_graph", "figure"), Input("council_list", "value") ) def update_council_figure(selected_council): pie_df = df.query(f"councilName == '{selected_council}' and createdDate >= '{start_date}'").groupby(['typeName']).agg('count').reset_index() # noqa + print(pie_df.shape) + print(pie_df.shape[0] == 0) if pie_df.shape[0] == 0: pie_df = pd.DataFrame(columns = ["srnumber", "typeName"]) pie_df.loc[0] = [12345678, "No Request at this time"] - + for i in range(len(DISCRETE_COLORS_MAP.keys())): + pie_df.loc[i] = [12345678, list(DISCRETE_COLORS_MAP.keys())[i]] + print(pie_df.head()) pie_fig = px.pie( pie_df, names="typeName", From 158503bbaff3671df79fd23fab11d2add1b3432f Mon Sep 17 00:00:00 2001 From: Yi Heng Joshua Wu Date: Thu, 30 Jun 2022 17:19:49 -0700 Subject: [PATCH 08/37] Set data to 0 when no data --- server/dash/dashboards/neighborhood_recent.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/server/dash/dashboards/neighborhood_recent.py b/server/dash/dashboards/neighborhood_recent.py index 4e1a2b533..ec93f0984 100644 --- a/server/dash/dashboards/neighborhood_recent.py +++ b/server/dash/dashboards/neighborhood_recent.py @@ -170,9 +170,9 @@ def update_figure(selected_council): figure_df.typeName = figure_df.typeName.map(lambda x: '
'.join(textwrap.wrap(x, width=16))) # noqa if figure_df.shape[0] == 0: figure_df = pd.DataFrame(columns = ['createdDate', "srnumber", "typeName"]) - for i in range(len(DISCRETE_COLORS_MAP.keys())): - figure_df.loc[i] = [start_date + datetime.timedelta(days=i), 12345678, list(DISCRETE_COLORS_MAP.keys())[i]] - #figure_df.loc[0] = [start_date, 12345678, "No Request at this time"] + for j in range(len(DISCRETE_COLORS_MAP.keys())): + for i in range(len(DISCRETE_COLORS_MAP.keys())): + figure_df.loc[figure_df.shape[0]] = [start_date + datetime.timedelta(days=j), 0, list(DISCRETE_COLORS_MAP.keys())[i]] fig = px.line( figure_df, x="createdDate", @@ -200,14 +200,11 @@ def update_figure(selected_council): ) def update_council_figure(selected_council): pie_df = df.query(f"councilName == '{selected_council}' and createdDate >= '{start_date}'").groupby(['typeName']).agg('count').reset_index() # noqa - print(pie_df.shape) - print(pie_df.shape[0] == 0) if pie_df.shape[0] == 0: pie_df = pd.DataFrame(columns = ["srnumber", "typeName"]) pie_df.loc[0] = [12345678, "No Request at this time"] for i in range(len(DISCRETE_COLORS_MAP.keys())): pie_df.loc[i] = [12345678, list(DISCRETE_COLORS_MAP.keys())[i]] - print(pie_df.head()) pie_fig = px.pie( pie_df, names="typeName", From b9b4cb45970c0fe2a937797014364d68d604b798 Mon Sep 17 00:00:00 2001 From: Nicholas Kwon Date: Thu, 30 Jun 2022 20:05:30 -0700 Subject: [PATCH 09/37] debugging--doesn't work on first load --- client/components/Map/index.js | 21 +++++----- client/components/Map/layers/RequestsLayer.js | 41 +++++++++++++++++-- client/redux/reducers/data.js | 11 +++++ 3 files changed, 59 insertions(+), 14 deletions(-) diff --git a/client/components/Map/index.js b/client/components/Map/index.js index 1f521fe10..488c4fcbd 100644 --- a/client/components/Map/index.js +++ b/client/components/Map/index.js @@ -5,6 +5,7 @@ import PropTypes from 'proptypes'; import { connect } from 'react-redux'; import { withStyles } from '@material-ui/core/styles'; import axios from 'axios'; +import { getDataRequestSuccess } from '@reducers/data'; import { updateMapPosition } from '@reducers/ui'; import { trackMapExport } from '@reducers/analytics'; import CookieNotice from '../main/CookieNotice'; @@ -23,7 +24,6 @@ class MapContainer extends React.Component { super(props) this.state = { - requests: this.convertRequests([]), ncCounts: null, ccCounts: null, position: props.position, @@ -31,19 +31,17 @@ class MapContainer extends React.Component { selectedTypes: this.getSelectedTypes(), } - this.openRequests = null; this.isSubscribed = null; } componentDidMount() { - // TODO: redux-saga, add to store instead of local state this.isSubscribed = true; this.setData(); } componentDidUpdate(prevProps) { if (prevProps.activeMode !== this.props.activeMode || - prevProps.pins !== this.props.pins) + prevProps.pins !== this.props.pins) this.setData(); } @@ -52,7 +50,8 @@ class MapContainer extends React.Component { } getOpenRequests = async () => { - const url = `${process.env.API_URL}/requests/pins/open`; + // TODO: add date specification. See https://dev-api.311-data.org/docs#/default/get_all_service_requests_requests_get. + const url = `${process.env.API_URL}/requests`; const { data } = await axios.get(url); this.openRequests = data; }; @@ -65,9 +64,8 @@ class MapContainer extends React.Component { } if (this.isSubscribed) { - return this.setState({ - requests: this.convertRequests(this.openRequests), - }); + const { getDataSuccess } = this.props; + getDataSuccess(this.convertRequests(this.openRequests)); } }; @@ -78,6 +76,7 @@ class MapContainer extends React.Component { properties: { requestId: request.requestId, typeId: request.typeId, + closedDate: request.closedDate, }, geometry: { type: 'Point', @@ -99,8 +98,8 @@ class MapContainer extends React.Component { }; render() { - const { position, lastUpdated, updatePosition, exportMap, classes } = this.props; - const { requests, ncCounts, ccCounts, selectedTypes } = this.state; + const { position, lastUpdated, updatePosition, exportMap, classes, requests } = this.props; + const { ncCounts, ccCounts, selectedTypes } = this.state; return (
({ lastUpdated: state.metadata.lastPulledLocal, activeMode: state.ui.map.activeMode, requestTypes: state.filters.requestTypes, + requests: state.data.requests }); const mapDispatchToProps = dispatch => ({ updatePosition: position => dispatch(updateMapPosition(position)), exportMap: () => dispatch(trackMapExport()), + getDataSuccess: data => dispatch(getDataRequestSuccess(data)), }); MapContainer.propTypes = {}; diff --git a/client/components/Map/layers/RequestsLayer.js b/client/components/Map/layers/RequestsLayer.js index 585a0bef1..09b102ed9 100644 --- a/client/components/Map/layers/RequestsLayer.js +++ b/client/components/Map/layers/RequestsLayer.js @@ -30,6 +30,24 @@ function typeFilter(selectedTypes) { ]; } +function statusFilter(requestStatus) { + // requestStatus is an object with keys "open" and "closed", and boolean values. + if (requestStatus.open && requestStatus.closed) { + console.log("both"); + return null; + } + if (!requestStatus.open && !requestStatus.closed) { + console.log("neither"); + return ['==', ['literal', "a"], ['literal', "b"]]; + } + if (requestStatus.open) { + console.log("open only"); + return ['==', ['get', 'closedDate'], ['literal', null]]; + } + console.log("closed only") + return ['!=', ['get', 'closedDate'], ['literal', null]]; +} + class RequestsLayer extends React.Component { constructor(props) { super(props); @@ -47,6 +65,7 @@ class RequestsLayer extends React.Component { const { activeLayer, selectedTypes, + requestStatus, requests, colorScheme, } = this.props; @@ -55,13 +74,20 @@ class RequestsLayer extends React.Component { this.setActiveLayer(activeLayer); if (selectedTypes !== prev.selectedTypes) { + console.log("change selected types"); this.setSelectedTypes(selectedTypes); } - if (requests !== prev.requests && this.ready) + if (requestStatus.open !== prev.requestStatus.open || requestStatus.closed !== prev.requestStatus.closed) { + this.setRequestStatus(requestStatus); + } + if (requests !== prev.requests && this.ready) { + console.log("got new requests"); + console.log(requests); this.setRequests(requests); - - if (colorScheme !== prev.colorScheme) + } + if (colorScheme !== prev.colorScheme) { this.setColorScheme(colorScheme); + } } addSources = () => { @@ -138,6 +164,12 @@ class RequestsLayer extends React.Component { // its filter here as well. }; + setRequestStatus = requestStatus => { + this.map.setFilter('request-circles', statusFilter(requestStatus)); + // Currently, we do not support heatmap. If we did, we'd want to update + // its filter here as well. + }; + setRequests = requests => { this.map.getSource('requests').setData(requests); }; @@ -170,7 +202,8 @@ RequestsLayer.defaultProps = { }; const mapStateToProps = state => ({ - selectedTypes: state.filters.requestTypes + selectedTypes: state.filters.requestTypes, + requestStatus: state.filters.requestStatus, }); // We need to specify forwardRef to allow refs on connected components. diff --git a/client/redux/reducers/data.js b/client/redux/reducers/data.js index 351eb0c28..bcaeaa942 100644 --- a/client/redux/reducers/data.js +++ b/client/redux/reducers/data.js @@ -1,5 +1,6 @@ export const types = { GET_DATA_REQUEST: 'GET_DATA_REQUEST', + GET_DATA_REQUEST_SUCCESS: 'GET_DATA_REQUEST_SUCCESS', GET_PINS_SUCCESS: 'GET_PINS_SUCCESS', GET_PINS_FAILURE: 'GET_PINS_FAILURE', GET_OPEN_REQUESTS: 'GET_OPEN_REQUESTS', @@ -25,6 +26,11 @@ export const getDataRequest = () => ({ type: types.GET_DATA_REQUEST, }); +export const getDataRequestSuccess = response => ({ + type: types.GET_DATA_REQUEST_SUCCESS, + payload: response, +}); + export const getPinsSuccess = response => ({ type: types.GET_PINS_SUCCESS, payload: response, @@ -122,6 +128,11 @@ export default (state = initialState, action) => { isMapLoading: true, isVisLoading: true, }; + case types.GET_DATA_REQUEST_SUCCESS: + return { + ...state, + requests: action.payload, + }; case types.GET_PINS_SUCCESS: return { ...state, From 5c86d0712c51124548f63e62dd94c6287498309b Mon Sep 17 00:00:00 2001 From: Nicholas Kwon Date: Thu, 30 Jun 2022 20:30:14 -0700 Subject: [PATCH 10/37] clean up + don't pass requests as prop --- client/components/Map/Map.jsx | 2 -- client/components/Map/layers/RequestsLayer.js | 35 ++++++++----------- 2 files changed, 15 insertions(+), 22 deletions(-) diff --git a/client/components/Map/Map.jsx b/client/components/Map/Map.jsx index 56cdd2cae..6994c2925 100644 --- a/client/components/Map/Map.jsx +++ b/client/components/Map/Map.jsx @@ -486,7 +486,6 @@ class Map extends React.Component { } = this.props; const { - requests, geoFilterType, locationInfo, // filteredRequestCounts, @@ -508,7 +507,6 @@ class Map extends React.Component {
this.mapContainer = el} > this.requestsLayer = el} - requests={requests} activeLayer={activeRequestsLayer} selectedTypes={selectedTypes} colorScheme={colorScheme} diff --git a/client/components/Map/layers/RequestsLayer.js b/client/components/Map/layers/RequestsLayer.js index 09b102ed9..41848edc0 100644 --- a/client/components/Map/layers/RequestsLayer.js +++ b/client/components/Map/layers/RequestsLayer.js @@ -33,18 +33,16 @@ function typeFilter(selectedTypes) { function statusFilter(requestStatus) { // requestStatus is an object with keys "open" and "closed", and boolean values. if (requestStatus.open && requestStatus.closed) { - console.log("both"); - return null; + // Hack to allow ALL requests. + return ['==', ['literal', "a"], ['literal', "a"]]; } if (!requestStatus.open && !requestStatus.closed) { - console.log("neither"); + // Hack to filter ALL requests. return ['==', ['literal', "a"], ['literal', "b"]]; } if (requestStatus.open) { - console.log("open only"); return ['==', ['get', 'closedDate'], ['literal', null]]; } - console.log("closed only") return ['!=', ['get', 'closedDate'], ['literal', null]]; } @@ -73,12 +71,13 @@ class RequestsLayer extends React.Component { if (activeLayer !== prev.activeLayer) this.setActiveLayer(activeLayer); - if (selectedTypes !== prev.selectedTypes) { - console.log("change selected types"); - this.setSelectedTypes(selectedTypes); - } - if (requestStatus.open !== prev.requestStatus.open || requestStatus.closed !== prev.requestStatus.closed) { - this.setRequestStatus(requestStatus); + // Check if the selected types OR the request status has changed. + // These filters need to be updated together, since they are + // actually composed into a single filter. + if (selectedTypes !== prev.selectedTypes || + requestStatus.open !== prev.requestStatus.open || + requestStatus.closed !== prev.requestStatus.closed) { + this.setFilters(selectedTypes, requestStatus); } if (requests !== prev.requests && this.ready) { console.log("got new requests"); @@ -158,14 +157,9 @@ class RequestsLayer extends React.Component { } }; - setSelectedTypes = selectedTypes => { - this.map.setFilter('request-circles', typeFilter(selectedTypes)); - // Currently, we do not support heatmap. If we did, we'd want to update - // its filter here as well. - }; - - setRequestStatus = requestStatus => { - this.map.setFilter('request-circles', statusFilter(requestStatus)); + setFilters = (selectedTypes, requestStatus) => { + this.map.setFilter('request-circles', + ['all', typeFilter(selectedTypes), statusFilter(requestStatus)]); // Currently, we do not support heatmap. If we did, we'd want to update // its filter here as well. }; @@ -204,9 +198,10 @@ RequestsLayer.defaultProps = { const mapStateToProps = state => ({ selectedTypes: state.filters.requestTypes, requestStatus: state.filters.requestStatus, + requests: state.data.requests, }); // We need to specify forwardRef to allow refs on connected components. // See https://github.com/reduxjs/react-redux/issues/1291#issuecomment-494185126 // for more info. -export default connect(mapStateToProps, null, null, { forwardRef: true })(RequestsLayer); +export default connect(mapStateToProps, null, null, { forwardRef: true })(RequestsLayer); \ No newline at end of file From 793becffa2a5ead42d32dd54619dca546bda3360 Mon Sep 17 00:00:00 2001 From: Nicholas Kwon Date: Thu, 30 Jun 2022 20:37:45 -0700 Subject: [PATCH 11/37] fix request naming in index.js --- client/components/Map/index.js | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/client/components/Map/index.js b/client/components/Map/index.js index 488c4fcbd..17bddab29 100644 --- a/client/components/Map/index.js +++ b/client/components/Map/index.js @@ -31,6 +31,9 @@ class MapContainer extends React.Component { selectedTypes: this.getSelectedTypes(), } + // We store the raw requests from the API call here, but eventually they are + // converted and stored in the Redux store. + this.rawRequests = null; this.isSubscribed = null; } @@ -49,23 +52,24 @@ class MapContainer extends React.Component { this.isSubscribed = false; } - getOpenRequests = async () => { + getAllRequests = async () => { // TODO: add date specification. See https://dev-api.311-data.org/docs#/default/get_all_service_requests_requests_get. + // By default, this will only get the 1000 most recent requests. const url = `${process.env.API_URL}/requests`; const { data } = await axios.get(url); - this.openRequests = data; + this.rawRequests = data; }; setData = async () => { const { pins } = this.props; - if (!this.openRequests) { - await this.getOpenRequests(); + if (!this.rawRequests) { + await this.getAllRequests(); } if (this.isSubscribed) { const { getDataSuccess } = this.props; - getDataSuccess(this.convertRequests(this.openRequests)); + getDataSuccess(this.convertRequests(this.rawRequests)); } }; From 17533207c13453bd110be24f72dbac1da0db2f9d Mon Sep 17 00:00:00 2001 From: Nicholas Kwon Date: Thu, 30 Jun 2022 20:51:56 -0700 Subject: [PATCH 12/37] add constants --- client/components/Map/layers/RequestsLayer.js | 27 +++++++++++++------ 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/client/components/Map/layers/RequestsLayer.js b/client/components/Map/layers/RequestsLayer.js index 41848edc0..fb3e648e6 100644 --- a/client/components/Map/layers/RequestsLayer.js +++ b/client/components/Map/layers/RequestsLayer.js @@ -8,14 +8,25 @@ import { connect } from 'react-redux'; // so you don't cover up important labels const BEFORE_ID = 'poi-label'; +// Key for type id in store.data.requests. +const TYPE_ID = 'typeId'; +// Key for closed date in store.data.requests. +const CLOSED_DATE = 'closedDate'; + +// Constants required for Mapbox filtering. +const LITERAL = 'literal'; +const GET = 'get'; + +const WHITE_HEX = '#FFFFFF'; + function circleColors(requestTypes) { const colors = []; requestTypes.forEach(type => colors.push(type.typeId, type.color)) return [ 'match', - ['get', 'typeId'], + [GET, TYPE_ID], ...colors, - '#FFFFFF', + WHITE_HEX, ]; } @@ -25,8 +36,8 @@ function typeFilter(selectedTypes) { var trueTypes = Object.keys(selectedTypes).map((type) => parseInt(type)).filter((type) => selectedTypes[type]); return [ 'in', - ['get', 'typeId'], - ['literal', trueTypes] + [GET, TYPE_ID], + [LITERAL, trueTypes] ]; } @@ -34,16 +45,16 @@ function statusFilter(requestStatus) { // requestStatus is an object with keys "open" and "closed", and boolean values. if (requestStatus.open && requestStatus.closed) { // Hack to allow ALL requests. - return ['==', ['literal', "a"], ['literal', "a"]]; + return ['==', [LITERAL, 'a'], [LITERAL, 'a']]; } if (!requestStatus.open && !requestStatus.closed) { // Hack to filter ALL requests. - return ['==', ['literal', "a"], ['literal', "b"]]; + return ['==', [LITERAL, 'a'], [LITERAL, 'b']]; } if (requestStatus.open) { - return ['==', ['get', 'closedDate'], ['literal', null]]; + return ['==', [GET, CLOSED_DATE], [LITERAL, null]]; } - return ['!=', ['get', 'closedDate'], ['literal', null]]; + return ['!=', [GET, CLOSED_DATE], [LITERAL, null]]; } class RequestsLayer extends React.Component { From 80647728f8561e9a7c4527031aead873d074c6be Mon Sep 17 00:00:00 2001 From: Nicholas Kwon Date: Fri, 1 Jul 2022 13:16:10 -0700 Subject: [PATCH 13/37] add param for batch size --- client/components/Map/index.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/client/components/Map/index.js b/client/components/Map/index.js index 17bddab29..e0b18ed73 100644 --- a/client/components/Map/index.js +++ b/client/components/Map/index.js @@ -13,6 +13,8 @@ import CookieNotice from '../main/CookieNotice'; // import "mapbox-gl/dist/mapbox-gl.css"; import Map from './Map'; +const REQUEST_BATCH_SIZE = 5000; + const styles = theme => ({ root: { height: `calc(100vh - ${theme.header.height} - ${theme.footer.height})`, @@ -54,8 +56,9 @@ class MapContainer extends React.Component { getAllRequests = async () => { // TODO: add date specification. See https://dev-api.311-data.org/docs#/default/get_all_service_requests_requests_get. - // By default, this will only get the 1000 most recent requests. - const url = `${process.env.API_URL}/requests`; + const url = new URL(`${process.env.API_URL}/requests`); + url.searchParams.append("limit", `${REQUEST_BATCH_SIZE}`); + console.log(url); const { data } = await axios.get(url); this.rawRequests = data; }; From 8f6a980fd1eadb9c2d98a1e7cb491bac1d3d8aa6 Mon Sep 17 00:00:00 2001 From: funbunch Date: Sat, 2 Jul 2022 13:56:14 -0700 Subject: [PATCH 14/37] Initial add About page and added to header nav. --- client/Routes.jsx | 2 ++ client/components/Header.jsx | 4 ++-- client/components/main/About.jsx | 35 ++++++++++++++++++++++++++++++++ client/package-lock.json | 10 ++++----- 4 files changed, 44 insertions(+), 7 deletions(-) create mode 100644 client/components/main/About.jsx diff --git a/client/Routes.jsx b/client/Routes.jsx index 25144ff27..5b3f8f375 100644 --- a/client/Routes.jsx +++ b/client/Routes.jsx @@ -10,6 +10,7 @@ import Desktop from '@components/main/Desktop'; import Reports from '@components/main/Reports'; import Privacy from '@components/main/Privacy'; import Faqs from '@components/main/Faqs'; +import About from '@components/main/About'; import Blog from '@components/main/Blog'; import ContactForm from '@components/main/ContactForm'; @@ -25,6 +26,7 @@ export default function Routes() { + diff --git a/client/components/Header.jsx b/client/components/Header.jsx index 441e154f0..2e0523c1f 100644 --- a/client/components/Header.jsx +++ b/client/components/Header.jsx @@ -110,8 +110,8 @@ const Header = () => { - - + + diff --git a/client/components/main/About.jsx b/client/components/main/About.jsx new file mode 100644 index 000000000..c35f2330c --- /dev/null +++ b/client/components/main/About.jsx @@ -0,0 +1,35 @@ +import React from 'react'; +import { + makeStyles, + Container, +} from '@material-ui/core'; + +const useStyles = makeStyles({ + root: { + color: 'black', + backgroundColor: 'white', + padding: '2em', + '& h1': { + fontSize: '2.5em', + }, + '& img': { + maxWidth: '100%', + height: 'auto', + display: 'block', + marginLeft: 'auto', + marginRight: 'auto', + }, + }, +}); + +const About = () => { + const classes = useStyles(); + + return ( + +

About

+
+ ); +}; + +export default About; diff --git a/client/package-lock.json b/client/package-lock.json index 75ae87dae..699554356 100644 --- a/client/package-lock.json +++ b/client/package-lock.json @@ -4283,7 +4283,7 @@ "async-foreach": { "version": "0.1.3", "resolved": "https://registry.npmjs.org/async-foreach/-/async-foreach-0.1.3.tgz", - "integrity": "sha1-NhIfhFwFeBct5Bmpfb6x0W7DRUI=", + "integrity": "sha512-VUeSMD8nEGBWaZK4lizI1sf3yEC7pnAQ/mrI7pC2fBz2s/tq5jWWEngTwaf0Gruu/OoXRGLGg1XFqpYBiGTYJA==", "dev": true }, "async-limiter": { @@ -5774,7 +5774,7 @@ "console-control-strings": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/console-control-strings/-/console-control-strings-1.1.0.tgz", - "integrity": "sha1-PXz0Rk22RG6mRL9LOVB/mFEAjo4=", + "integrity": "sha512-ty/fTekppD2fIwRvnZAVdeOiGd1c7YXEixbgJTNzqcxJWKQnjJ/V1bNEEE6hygpM3WjwHFUVK6HTjWSzV4a8sQ==", "dev": true }, "constants-browserify": { @@ -6506,7 +6506,7 @@ "delegates": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz", - "integrity": "sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o=", + "integrity": "sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ==", "dev": true }, "density-clustering": { @@ -8498,7 +8498,7 @@ "get-stdin": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/get-stdin/-/get-stdin-4.0.1.tgz", - "integrity": "sha1-uWjGsKBDhDJJAui/Gl3zJXmkUP4=", + "integrity": "sha512-F5aQMywwJ2n85s4hJPTT9RPxGmubonuB10MNYo17/xph174n2MIR33HRguhzVag10O/npM7SPk73LMZNP+FaWw==", "dev": true }, "get-stream": { @@ -8750,7 +8750,7 @@ "has-unicode": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.1.tgz", - "integrity": "sha1-4Ob+aijPUROIVeCG0Wkedx3iqLk=", + "integrity": "sha512-8Rf9Y83NBReMnx0gFzA8JImQACstCYWUplepDa9xprwwtmgEZUF0h/i5xSA625zB/I37EtrswSST6OXxwaaIJQ==", "dev": true }, "has-value": { From 4ad35da056e19c3bc9b9c511e642c141eabf733e Mon Sep 17 00:00:00 2001 From: Nicholas Kwon Date: Sat, 2 Jul 2022 16:02:45 -0700 Subject: [PATCH 15/37] getting data for 1 week works now. missing comments --- client/components/Map/index.js | 26 ++++++++++++++----- client/components/Map/layers/RequestsLayer.js | 15 +++++------ client/components/common/CONSTANTS.js | 8 +++--- client/redux/reducers/data.js | 2 ++ client/redux/reducers/filters.js | 6 +++-- 5 files changed, 37 insertions(+), 20 deletions(-) diff --git a/client/components/Map/index.js b/client/components/Map/index.js index e0b18ed73..1f8b48864 100644 --- a/client/components/Map/index.js +++ b/client/components/Map/index.js @@ -9,9 +9,10 @@ import { getDataRequestSuccess } from '@reducers/data'; import { updateMapPosition } from '@reducers/ui'; import { trackMapExport } from '@reducers/analytics'; import CookieNotice from '../main/CookieNotice'; -// import { MAP_MODES } from '../common/CONSTANTS'; +import { DATE_SPEC } from '../common/CONSTANTS'; // import "mapbox-gl/dist/mapbox-gl.css"; import Map from './Map'; +import moment from 'moment'; const REQUEST_BATCH_SIZE = 5000; @@ -35,7 +36,7 @@ class MapContainer extends React.Component { // We store the raw requests from the API call here, but eventually they are // converted and stored in the Redux store. - this.rawRequests = null; + this.rawRequests = []; this.isSubscribed = null; } @@ -55,18 +56,27 @@ class MapContainer extends React.Component { } getAllRequests = async () => { - // TODO: add date specification. See https://dev-api.311-data.org/docs#/default/get_all_service_requests_requests_get. + const { startDate, endDate } = this.props; const url = new URL(`${process.env.API_URL}/requests`); + url.searchParams.append("start_date", moment(startDate, DATE_SPEC).format('YYYY-MM-DD')); + url.searchParams.append("end_date", moment(endDate, DATE_SPEC).format('YYYY-MM-DD')); url.searchParams.append("limit", `${REQUEST_BATCH_SIZE}`); - console.log(url); - const { data } = await axios.get(url); - this.rawRequests = data; + var returned_length = REQUEST_BATCH_SIZE; + var skip = 0; + while (returned_length === REQUEST_BATCH_SIZE) { + url.searchParams.append("skip", `${skip}`); + const { data } = await axios.get(url); + returned_length = data.length; + skip += returned_length; + this.rawRequests.push(...data); + url.searchParams.delete("skip"); + } }; setData = async () => { const { pins } = this.props; - if (!this.rawRequests) { + if (this.rawRequests.length === 0) { await this.getAllRequests(); } @@ -131,6 +141,8 @@ const mapStateToProps = state => ({ lastUpdated: state.metadata.lastPulledLocal, activeMode: state.ui.map.activeMode, requestTypes: state.filters.requestTypes, + startDate: state.filters.startDate, + endDate: state.filters.endDate, requests: state.data.requests }); diff --git a/client/components/Map/layers/RequestsLayer.js b/client/components/Map/layers/RequestsLayer.js index fb3e648e6..4a4287620 100644 --- a/client/components/Map/layers/RequestsLayer.js +++ b/client/components/Map/layers/RequestsLayer.js @@ -91,8 +91,6 @@ class RequestsLayer extends React.Component { this.setFilters(selectedTypes, requestStatus); } if (requests !== prev.requests && this.ready) { - console.log("got new requests"); - console.log(requests); this.setRequests(requests); } if (colorScheme !== prev.colorScheme) { @@ -114,6 +112,7 @@ class RequestsLayer extends React.Component { selectedTypes, colorScheme, requestTypes, + requestStatus, } = this.props; this.map.addLayer({ @@ -134,7 +133,7 @@ class RequestsLayer extends React.Component { 'circle-color': circleColors(requestTypes), 'circle-opacity': 0.8, }, - filter: typeFilter(selectedTypes), + filter: this.getFilterSpec(selectedTypes, requestStatus), }, BEFORE_ID); // this.map.addLayer({ @@ -168,9 +167,13 @@ class RequestsLayer extends React.Component { } }; + getFilterSpec = (selectedTypes, requestStatus) => { + return ['all', typeFilter(selectedTypes), statusFilter(requestStatus)]; + }; + setFilters = (selectedTypes, requestStatus) => { this.map.setFilter('request-circles', - ['all', typeFilter(selectedTypes), statusFilter(requestStatus)]); + this.getFilterSpec(selectedTypes, requestStatus)); // Currently, we do not support heatmap. If we did, we'd want to update // its filter here as well. }; @@ -194,15 +197,11 @@ class RequestsLayer extends React.Component { RequestsLayer.propTypes = { activeLayer: PropTypes.oneOf(['points', 'heatmap']), - selectedTypes: PropTypes.shape({}), - requests: PropTypes.shape({}), colorScheme: PropTypes.string, }; RequestsLayer.defaultProps = { activeLayer: 'points', - selectedTypes: {}, - requests: {}, colorScheme: '', }; diff --git a/client/components/common/CONSTANTS.js b/client/components/common/CONSTANTS.js index 319e2f20e..ea84eb9e9 100644 --- a/client/components/common/CONSTANTS.js +++ b/client/components/common/CONSTANTS.js @@ -829,9 +829,11 @@ export const MAP_DATE_RANGES = (() => { ]; })(); +export const DATE_SPEC = 'MM/DD/YYYY'; + export const DATE_RANGES = (() => { - const endDate = moment().format('MM/DD/YYYY'); - const priorDate = (num, timeInterval) => moment().subtract(num, timeInterval).format('MM/DD/YYYY'); + const endDate = moment().format(DATE_SPEC); + const priorDate = (num, timeInterval) => moment().subtract(num, timeInterval).format(DATE_SPEC); return [ { @@ -873,7 +875,7 @@ export const DATE_RANGES = (() => { { id: 'YEAR_TO_DATE', label: 'Year to Date', - startDate: moment().startOf('year').format('MM/DD/YYYY'), + startDate: moment().startOf('year').format(DATE_SPEC), endDate, }, { diff --git a/client/redux/reducers/data.js b/client/redux/reducers/data.js index bcaeaa942..0460dd744 100644 --- a/client/redux/reducers/data.js +++ b/client/redux/reducers/data.js @@ -118,6 +118,8 @@ const initialState = { pins: [], pinsInfo: {}, selectedNcId: null, + // Empty GeoJSON object. + requests: { type: 'FeatureCollection', features: [] }, }; export default (state = initialState, action) => { diff --git a/client/redux/reducers/filters.js b/client/redux/reducers/filters.js index 065dda3c4..bc9793be6 100644 --- a/client/redux/reducers/filters.js +++ b/client/redux/reducers/filters.js @@ -1,3 +1,5 @@ +import { DATE_RANGES } from '@components/common/CONSTANTS'; + export const types = { UPDATE_START_DATE: 'UPDATE_START_DATE', UPDATE_END_DATE: 'UPDATE_END_DATE', @@ -35,8 +37,8 @@ export const updateRequestStatus = status => ({ const initialState = { // dateRange: null, - startDate: null, - endDate: null, + startDate: DATE_RANGES[0].startDate, + endDate: DATE_RANGES[0].endDate, councilId: null, requestTypes: { 1: false, From a9df8a8c004d8f537dbf4dd7ec0c7aec919f13a9 Mon Sep 17 00:00:00 2001 From: Nicholas Kwon Date: Tue, 5 Jul 2022 12:28:08 -0700 Subject: [PATCH 16/37] add some comments --- client/components/Map/index.js | 8 ++++- client/components/Map/layers/RequestsLayer.js | 30 +++++++++++++++++-- 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/client/components/Map/index.js b/client/components/Map/index.js index 1f8b48864..1131937dc 100644 --- a/client/components/Map/index.js +++ b/client/components/Map/index.js @@ -54,7 +54,13 @@ class MapContainer extends React.Component { componentWillUnmount() { this.isSubscribed = false; } - + /** + * Gets all requests over the time range specified in the Redux store. + * + * Since the server is slow to retrieve all the requests at once, we need to + * make multiple API calls, using `skip` and `limit` to retrieve consecutive + * chunks of data. + */ getAllRequests = async () => { const { startDate, endDate } = this.props; const url = new URL(`${process.env.API_URL}/requests`); diff --git a/client/components/Map/layers/RequestsLayer.js b/client/components/Map/layers/RequestsLayer.js index 4a4287620..0d3e52cd1 100644 --- a/client/components/Map/layers/RequestsLayer.js +++ b/client/components/Map/layers/RequestsLayer.js @@ -30,8 +30,15 @@ function circleColors(requestTypes) { ]; } +/** + * Gets a MapBox GL JS filter specification to filter request types. + * + * @param {Object} selectedTypes A mapping of k:v, where k is an str request + * type, and v is a boolean indicating whether the request type is selected. + * @return {Array} A Mapbox GL JS filter specification that filters out the + * unselected types. + */ function typeFilter(selectedTypes) { - // selectedTypes maps ints (in string form) to booleans, indicating whether the type is selected. // Get an array of int typeIds corresponding value in selectedTypes is true. var trueTypes = Object.keys(selectedTypes).map((type) => parseInt(type)).filter((type) => selectedTypes[type]); return [ @@ -41,8 +48,16 @@ function typeFilter(selectedTypes) { ]; } +/** + * Gets a MapBox GL JS filter specification to filter request statuses. + * + * @param {Object} requestStatus A mapping of k:v, where k is a request status + * (either open or closed), and v is a boolean indicating whether the request + * status is selected. + * @return {Array} A Mapbox GL JS filter specification that filters out the + * unselected statuses. + */ function statusFilter(requestStatus) { - // requestStatus is an object with keys "open" and "closed", and boolean values. if (requestStatus.open && requestStatus.closed) { // Hack to allow ALL requests. return ['==', [LITERAL, 'a'], [LITERAL, 'a']]; @@ -167,6 +182,17 @@ class RequestsLayer extends React.Component { } }; + /** + * Gets a MapBox GL JS filter specification. + * + * @param {Object} selectedTypes A mapping of k:v, where k is an int request + * type, and v is a boolean indicating whether the request type is selected. + * @param {Object} requestStatus A mapping of k:v, where k is a request status + * (either open or closed), and v is a boolean indicating whether the request + * status is selected. + * @return {Array} A Mapbox GL JS filter specification that filters out the + * unselected types and statuses. + */ getFilterSpec = (selectedTypes, requestStatus) => { return ['all', typeFilter(selectedTypes), statusFilter(requestStatus)]; }; From a1a7008502204c1d4b464a01faec2294604e863c Mon Sep 17 00:00:00 2001 From: Nicholas Kwon Date: Wed, 6 Jul 2022 18:15:52 -0700 Subject: [PATCH 17/37] allow user to change date within past week --- client/components/Map/index.js | 8 +-- client/components/Map/layers/RequestsLayer.js | 49 +++++++++++++++---- client/components/common/CONSTANTS.js | 14 ++++-- .../common/ReactDayPicker/ReactDayPicker.jsx | 32 ++++++++++-- client/redux/reducers/filters.js | 8 +-- 5 files changed, 87 insertions(+), 24 deletions(-) diff --git a/client/components/Map/index.js b/client/components/Map/index.js index 1131937dc..054ae0e37 100644 --- a/client/components/Map/index.js +++ b/client/components/Map/index.js @@ -9,7 +9,6 @@ import { getDataRequestSuccess } from '@reducers/data'; import { updateMapPosition } from '@reducers/ui'; import { trackMapExport } from '@reducers/analytics'; import CookieNotice from '../main/CookieNotice'; -import { DATE_SPEC } from '../common/CONSTANTS'; // import "mapbox-gl/dist/mapbox-gl.css"; import Map from './Map'; import moment from 'moment'; @@ -64,8 +63,8 @@ class MapContainer extends React.Component { getAllRequests = async () => { const { startDate, endDate } = this.props; const url = new URL(`${process.env.API_URL}/requests`); - url.searchParams.append("start_date", moment(startDate, DATE_SPEC).format('YYYY-MM-DD')); - url.searchParams.append("end_date", moment(endDate, DATE_SPEC).format('YYYY-MM-DD')); + url.searchParams.append("start_date", startDate); + url.searchParams.append("end_date", endDate); url.searchParams.append("limit", `${REQUEST_BATCH_SIZE}`); var returned_length = REQUEST_BATCH_SIZE; var skip = 0; @@ -100,6 +99,9 @@ class MapContainer extends React.Component { requestId: request.requestId, typeId: request.typeId, closedDate: request.closedDate, + // Store this in milliseconds so that it's easy to do date comparisons + // using Mapbox GL JS filters. + createdDateMs: moment(request.createdDate).valueOf(), }, geometry: { type: 'Point', diff --git a/client/components/Map/layers/RequestsLayer.js b/client/components/Map/layers/RequestsLayer.js index 0d3e52cd1..6202c403a 100644 --- a/client/components/Map/layers/RequestsLayer.js +++ b/client/components/Map/layers/RequestsLayer.js @@ -3,6 +3,8 @@ import React from 'react'; import PropTypes from 'prop-types'; import { connect } from 'react-redux'; +import { INTERNAL_DATE_SPEC } from '../../common/CONSTANTS'; +import moment from 'moment'; // put layer underneath this layer (from original mapbox tiles) // so you don't cover up important labels @@ -72,6 +74,22 @@ function statusFilter(requestStatus) { return ['!=', [GET, CLOSED_DATE], [LITERAL, null]]; } +/** + * Gets a MapBox GL JS filter specification to filter requests by date range. + * + * @param {string} startDate The start date, in YYYY-MM-DD format. + * @param {string} endDate The end date, in YYYY-MM-DD format. + * @return {Array} A Mapbox GL JS filter specification that filters out + * requests outside of the date range. + */ +function dateFilter(startDate, endDate) { + const startDateMs = moment(startDate, INTERNAL_DATE_SPEC).valueOf(); + const endDateMs = moment(endDate, INTERNAL_DATE_SPEC).add(1, 'days').valueOf(); + const afterStartDate = ['>=', [GET, 'createdDateMs'], [LITERAL, startDateMs]]; + const beforeEndDate = ['<=', [GET, 'createdDateMs'], [LITERAL, endDateMs]]; + return ['all', afterStartDate, beforeEndDate]; +} + class RequestsLayer extends React.Component { constructor(props) { super(props); @@ -92,18 +110,23 @@ class RequestsLayer extends React.Component { requestStatus, requests, colorScheme, + startDate, + endDate, } = this.props; if (activeLayer !== prev.activeLayer) this.setActiveLayer(activeLayer); - // Check if the selected types OR the request status has changed. + // Check if the selected types OR the request status OR the date range has + // changed. // These filters need to be updated together, since they are // actually composed into a single filter. if (selectedTypes !== prev.selectedTypes || requestStatus.open !== prev.requestStatus.open || - requestStatus.closed !== prev.requestStatus.closed) { - this.setFilters(selectedTypes, requestStatus); + requestStatus.closed !== prev.requestStatus.closed || + startDate != prev.startDate || + endDate != prev.endDate) { + this.setFilters(selectedTypes, requestStatus, startDate, endDate); } if (requests !== prev.requests && this.ready) { this.setRequests(requests); @@ -128,6 +151,8 @@ class RequestsLayer extends React.Component { colorScheme, requestTypes, requestStatus, + startDate, + endDate, } = this.props; this.map.addLayer({ @@ -148,7 +173,8 @@ class RequestsLayer extends React.Component { 'circle-color': circleColors(requestTypes), 'circle-opacity': 0.8, }, - filter: this.getFilterSpec(selectedTypes, requestStatus), + filter: this.getFilterSpec(selectedTypes, requestStatus, startDate, + endDate), }, BEFORE_ID); // this.map.addLayer({ @@ -190,16 +216,19 @@ class RequestsLayer extends React.Component { * @param {Object} requestStatus A mapping of k:v, where k is a request status * (either open or closed), and v is a boolean indicating whether the request * status is selected. - * @return {Array} A Mapbox GL JS filter specification that filters out the + * @param {string} startDate The start date, in YYYY-MM-DD format. + * @param {string} endDate The end date, in YYYY-MM-DD format. + * @return {Array} A Mapbox GL JS filter specification that filters out the * unselected types and statuses. */ - getFilterSpec = (selectedTypes, requestStatus) => { - return ['all', typeFilter(selectedTypes), statusFilter(requestStatus)]; + getFilterSpec = (selectedTypes, requestStatus, startDate, endDate) => { + return ['all', typeFilter(selectedTypes), statusFilter(requestStatus), + dateFilter(startDate, endDate)]; }; - setFilters = (selectedTypes, requestStatus) => { + setFilters = (selectedTypes, requestStatus, startDate, endDate) => { this.map.setFilter('request-circles', - this.getFilterSpec(selectedTypes, requestStatus)); + this.getFilterSpec(selectedTypes, requestStatus, startDate, endDate)); // Currently, we do not support heatmap. If we did, we'd want to update // its filter here as well. }; @@ -235,6 +264,8 @@ const mapStateToProps = state => ({ selectedTypes: state.filters.requestTypes, requestStatus: state.filters.requestStatus, requests: state.data.requests, + startDate: state.filters.startDate, + endDate: state.filters.endDate, }); // We need to specify forwardRef to allow refs on connected components. diff --git a/client/components/common/CONSTANTS.js b/client/components/common/CONSTANTS.js index ea84eb9e9..a173b3938 100644 --- a/client/components/common/CONSTANTS.js +++ b/client/components/common/CONSTANTS.js @@ -829,11 +829,17 @@ export const MAP_DATE_RANGES = (() => { ]; })(); -export const DATE_SPEC = 'MM/DD/YYYY'; +// The user gets this date format since it's used most commonly in the US. +export const USER_DATE_SPEC = 'MM/DD/YYYY'; +// Internally, we use this date spec. This is what our server expects when we +// request data from a certain date range. +export const INTERNAL_DATE_SPEC = 'YYYY-MM-DD'; export const DATE_RANGES = (() => { - const endDate = moment().format(DATE_SPEC); - const priorDate = (num, timeInterval) => moment().subtract(num, timeInterval).format(DATE_SPEC); + const endDate = moment().format(USER_DATE_SPEC); + function priorDate(num, timeInterval) { + return moment().subtract(num, timeInterval).format(USER_DATE_SPEC); + } return [ { @@ -875,7 +881,7 @@ export const DATE_RANGES = (() => { { id: 'YEAR_TO_DATE', label: 'Year to Date', - startDate: moment().startOf('year').format(DATE_SPEC), + startDate: moment().startOf('year').format(USER_DATE_SPEC), endDate, }, { diff --git a/client/components/common/ReactDayPicker/ReactDayPicker.jsx b/client/components/common/ReactDayPicker/ReactDayPicker.jsx index 6bf898db8..345f03037 100644 --- a/client/components/common/ReactDayPicker/ReactDayPicker.jsx +++ b/client/components/common/ReactDayPicker/ReactDayPicker.jsx @@ -1,11 +1,19 @@ +import 'react-day-picker/lib/style.css'; + +import { + updateEndDate as reduxUpdateEndDate, + updateStartDate as reduxUpdateStartDate, +} from '@reducers/filters'; +import moment from 'moment'; +import PropTypes from 'prop-types'; import React, { useState } from 'react'; import DayPicker, { DateUtils } from 'react-day-picker'; -import PropTypes from 'prop-types'; +import { connect } from 'react-redux'; + +import { INTERNAL_DATE_SPEC } from '../CONSTANTS'; import Styles from './Styles'; import WeekDay from './Weekday'; -import 'react-day-picker/lib/style.css'; - const getInitialState = initialDates => { const [from, to] = initialDates; return { @@ -17,7 +25,10 @@ const getInitialState = initialDates => { const defaultState = { from: null, to: null }; -function ReactDayPicker({ onChange, initialDates, range }) { +function ReactDayPicker({ + onChange, initialDates, range, updateStartDate, + updateEndDate, +}) { const [state, setState] = useState(getInitialState(initialDates)); const isSelectingFirstDay = (from, to, day) => { @@ -37,6 +48,7 @@ function ReactDayPicker({ onChange, initialDates, range }) { to: null, enteredTo: null, })); + updateStartDate(moment(day).format(INTERNAL_DATE_SPEC)); onChange([day]); }; @@ -46,6 +58,7 @@ function ReactDayPicker({ onChange, initialDates, range }) { to: day, enteredTo: day, })); + updateEndDate(moment(day).format(INTERNAL_DATE_SPEC)); onChange([state.from, day]); }; @@ -106,12 +119,21 @@ ReactDayPicker.propTypes = { range: PropTypes.bool, onChange: PropTypes.func, initialDates: PropTypes.arrayOf(Date), + updateStartDate: PropTypes.func, + updateEndDate: PropTypes.func, }; ReactDayPicker.defaultProps = { range: false, onChange: null, initialDates: [], + updateStartDate: null, + updateEndDate: null, }; -export default ReactDayPicker; +const mapDispatchToProps = dispatch => ({ + updateStartDate: date => dispatch(reduxUpdateStartDate(date)), + updateEndDate: date => dispatch(reduxUpdateEndDate(date)), +}); + +export default connect(null, mapDispatchToProps)(ReactDayPicker); diff --git a/client/redux/reducers/filters.js b/client/redux/reducers/filters.js index bc9793be6..c4012a208 100644 --- a/client/redux/reducers/filters.js +++ b/client/redux/reducers/filters.js @@ -1,4 +1,5 @@ -import { DATE_RANGES } from '@components/common/CONSTANTS'; +import { DATE_RANGES, INTERNAL_DATE_SPEC, USER_DATE_SPEC } from '@components/common/CONSTANTS'; +import moment from 'moment'; export const types = { UPDATE_START_DATE: 'UPDATE_START_DATE', @@ -37,8 +38,9 @@ export const updateRequestStatus = status => ({ const initialState = { // dateRange: null, - startDate: DATE_RANGES[0].startDate, - endDate: DATE_RANGES[0].endDate, + // Always store dates using the INTERNAL_DATE_SPEC. + startDate: moment(DATE_RANGES[0].startDate, USER_DATE_SPEC).format(INTERNAL_DATE_SPEC), + endDate: moment(DATE_RANGES[0].endDate, USER_DATE_SPEC).format(INTERNAL_DATE_SPEC), councilId: null, requestTypes: { 1: false, From 48ef5ee2698b75a3c0682e04ad5579639175e2c4 Mon Sep 17 00:00:00 2001 From: funbunch Date: Wed, 6 Jul 2022 19:38:06 -0700 Subject: [PATCH 18/37] Adding Blog back to Nav --- client/components/Header.jsx | 3 +++ client/components/main/About.jsx | 37 +++++++++++++++++++++++++++++--- 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/client/components/Header.jsx b/client/components/Header.jsx index 2e0523c1f..1c703153c 100644 --- a/client/components/Header.jsx +++ b/client/components/Header.jsx @@ -113,6 +113,9 @@ const Header = () => { + + + diff --git a/client/components/main/About.jsx b/client/components/main/About.jsx index c35f2330c..969d4b157 100644 --- a/client/components/main/About.jsx +++ b/client/components/main/About.jsx @@ -1,13 +1,29 @@ import React from 'react'; +import ReactMarkdown from 'react-markdown'; import { makeStyles, Container, + Grid, } from '@material-ui/core'; +import useContentful from '../../hooks/useContentful'; + +const query = ` + query { + simplePageCollection(where: {slug: "about"}) { + items { + title + body + } + } + } +`; const useStyles = makeStyles({ root: { color: 'black', backgroundColor: 'white', + alignItems: 'center', + justifyContent: 'center', padding: '2em', '& h1': { fontSize: '2.5em', @@ -23,12 +39,27 @@ const useStyles = makeStyles({ }); const About = () => { + const { data, errors } = useContentful(query); const classes = useStyles(); + React.useEffect(() => { + if (errors) console.log(errors); + }, [errors]); + return ( - -

About

-
+ <> + { data + && ( + + + +

{data.simplePageCollection.items[0].title}

+ {data.simplePageCollection.items[0].body} +
+
+
+ )} + ); }; From f98935466134b54fe37fc1a4981faed758a3cc12 Mon Sep 17 00:00:00 2001 From: Nicholas Kwon Date: Thu, 7 Jul 2022 10:35:27 -0700 Subject: [PATCH 19/37] add comment about making end date inclusive --- client/components/Map/layers/RequestsLayer.js | 1 + 1 file changed, 1 insertion(+) diff --git a/client/components/Map/layers/RequestsLayer.js b/client/components/Map/layers/RequestsLayer.js index 6202c403a..969f9fa30 100644 --- a/client/components/Map/layers/RequestsLayer.js +++ b/client/components/Map/layers/RequestsLayer.js @@ -84,6 +84,7 @@ function statusFilter(requestStatus) { */ function dateFilter(startDate, endDate) { const startDateMs = moment(startDate, INTERNAL_DATE_SPEC).valueOf(); + // Make the end date inclusive by adding 1 day. const endDateMs = moment(endDate, INTERNAL_DATE_SPEC).add(1, 'days').valueOf(); const afterStartDate = ['>=', [GET, 'createdDateMs'], [LITERAL, startDateMs]]; const beforeEndDate = ['<=', [GET, 'createdDateMs'], [LITERAL, endDateMs]]; From f2e05e25745cff206f9d50d586fe62315c17ec40 Mon Sep 17 00:00:00 2001 From: Nicholas Kwon Date: Thu, 7 Jul 2022 13:23:33 -0700 Subject: [PATCH 20/37] make API requests per day, and make them parallel --- client/components/Map/index.js | 51 +++++++++++++++++++++++----------- 1 file changed, 35 insertions(+), 16 deletions(-) diff --git a/client/components/Map/index.js b/client/components/Map/index.js index 054ae0e37..3ce7bef63 100644 --- a/client/components/Map/index.js +++ b/client/components/Map/index.js @@ -8,12 +8,15 @@ import axios from 'axios'; import { getDataRequestSuccess } from '@reducers/data'; import { updateMapPosition } from '@reducers/ui'; import { trackMapExport } from '@reducers/analytics'; +import { INTERNAL_DATE_SPEC } from '../common/CONSTANTS'; import CookieNotice from '../main/CookieNotice'; // import "mapbox-gl/dist/mapbox-gl.css"; import Map from './Map'; import moment from 'moment'; -const REQUEST_BATCH_SIZE = 5000; +// We make API requests on a per-day basis. On average, there are about 4k +// requests per day, so 10k is a large safety margin. +const REQUEST_LIMIT = 10000; const styles = theme => ({ root: { @@ -53,29 +56,45 @@ class MapContainer extends React.Component { componentWillUnmount() { this.isSubscribed = false; } + + /** + * Gets all the dates within a given date range. + * @param {string} startDate A date in INTERNAL_DATE_SPEC format. + * @param {string} endDate A date in INTERNAL_DATE_SPEC format. + * @returns An array of string dates in INTERNAL_DATE_SPEC format, including + * the end date. + */ + getDatesInRange = (startDate, endDate) => { + var dateArray = []; + var currentDateMoment = moment(startDate, INTERNAL_DATE_SPEC); + const endDateMoment = moment(endDate, INTERNAL_DATE_SPEC); + while (currentDateMoment <= endDateMoment) { + dateArray.push(currentDateMoment.format(INTERNAL_DATE_SPEC)); + currentDateMoment = currentDateMoment.add(1, 'days'); + } + return dateArray; + } + /** * Gets all requests over the time range specified in the Redux store. * * Since the server is slow to retrieve all the requests at once, we need to - * make multiple API calls, using `skip` and `limit` to retrieve consecutive - * chunks of data. + * make multiple API calls, one for each day. */ getAllRequests = async () => { const { startDate, endDate } = this.props; - const url = new URL(`${process.env.API_URL}/requests`); - url.searchParams.append("start_date", startDate); - url.searchParams.append("end_date", endDate); - url.searchParams.append("limit", `${REQUEST_BATCH_SIZE}`); - var returned_length = REQUEST_BATCH_SIZE; - var skip = 0; - while (returned_length === REQUEST_BATCH_SIZE) { - url.searchParams.append("skip", `${skip}`); - const { data } = await axios.get(url); - returned_length = data.length; - skip += returned_length; - this.rawRequests.push(...data); - url.searchParams.delete("skip"); + const datesInRange = this.getDatesInRange(startDate, endDate); + var requests = []; + for (let i in datesInRange){ + const url = new URL(`${process.env.API_URL}/requests`); + url.searchParams.append("start_date", datesInRange[i]); + url.searchParams.append("end_date", datesInRange[i]); + url.searchParams.append("limit", `${REQUEST_LIMIT}`); + requests.push(axios.get(url)); } + await Promise.all(requests).then(responses => { + responses.forEach(response => this.rawRequests.push(...response.data)) + }); }; setData = async () => { From 8981c176c20c342a72262035e89a2d94b329bfb3 Mon Sep 17 00:00:00 2001 From: Yi Heng Joshua Wu Date: Thu, 7 Jul 2022 18:11:37 -0700 Subject: [PATCH 21/37] Revert test date changes and add constnat --- server/dash/dashboards/neighborhood_recent.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/server/dash/dashboards/neighborhood_recent.py b/server/dash/dashboards/neighborhood_recent.py index ec93f0984..a934cb294 100644 --- a/server/dash/dashboards/neighborhood_recent.py +++ b/server/dash/dashboards/neighborhood_recent.py @@ -7,7 +7,6 @@ from dash.dependencies import Input, Output import pandas as pd import plotly.express as px -from flask import request from app import app, batch_get_data from config import API_HOST @@ -24,8 +23,10 @@ 'address': "Address" } -start_date = datetime.date.today() + datetime.timedelta(days=3) -end_date = datetime.date.today() + datetime.timedelta(days=14) +START_DATE_DELTA = 7 +END_DATE_DELTA = 1 +start_date = datetime.date.today() - datetime.timedelta(days=START_DATE_DELTA) +end_date = datetime.date.today() - datetime.timedelta(days=END_DATE_DELTA) # TITLE title = "NEIGHBORHOOD WEEKLY REPORT" From 2b470dae8aaa86cb662cd40792ea7927d4b5bea7 Mon Sep 17 00:00:00 2001 From: Yi Heng Joshua Wu Date: Thu, 7 Jul 2022 18:13:59 -0700 Subject: [PATCH 22/37] Fix spacing issues --- server/dash/dashboards/neighborhood_recent.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/server/dash/dashboards/neighborhood_recent.py b/server/dash/dashboards/neighborhood_recent.py index a934cb294..d10ad9056 100644 --- a/server/dash/dashboards/neighborhood_recent.py +++ b/server/dash/dashboards/neighborhood_recent.py @@ -56,6 +56,7 @@ fig2 = px.pie() apply_figure_style(fig2) + def populate_options(): council_df_path = '/councils' council_df = pd.read_json(API_HOST + council_df_path) @@ -67,6 +68,7 @@ def populate_options(): }) return values + # Layout layout = html.Div([ html.H1(title), @@ -94,11 +96,11 @@ def populate_options(): ], className="graph-row"), html.Div([ html.Div( - dcc.Graph(id='graph', figure=fig ,config=CONFIG_OPTIONS), + dcc.Graph(id='graph', figure=fig, config=CONFIG_OPTIONS), className="half-graph" ), html.Div( - dcc.Graph(id='pie_graph', figure=fig2 , config=CONFIG_OPTIONS), + dcc.Graph(id='pie_graph', figure=fig2, config=CONFIG_OPTIONS), className="half-graph" ) ]), @@ -138,7 +140,6 @@ def update_table(selected_council): table_df = df.query(f"councilName == '{selected_council}'")[['srnumber', 'createdDate', 'closedDate', 'typeName', 'agencyName', 'sourceName', 'address']] # noqa if table_df.shape[0] == 0: table_df = pd.DataFrame(columns=["Request Type"]) - #table_df.loc[0] = ["There is no data right now"] for i in range(len(DISCRETE_COLORS_MAP.keys())): table_df.loc[i] = [list(DISCRETE_COLORS_MAP.keys())[i]] else: @@ -170,10 +171,11 @@ def update_figure(selected_council): figure_df = df.query(f"councilName == '{selected_council}' and createdDate >= '{start_date}'").groupby(['createdDate', 'typeName'])['srnumber'].count().reset_index() # noqa figure_df.typeName = figure_df.typeName.map(lambda x: '
'.join(textwrap.wrap(x, width=16))) # noqa if figure_df.shape[0] == 0: - figure_df = pd.DataFrame(columns = ['createdDate', "srnumber", "typeName"]) + figure_df = pd.DataFrame(columns=['createdDate', "srnumber", "typeName"]) for j in range(len(DISCRETE_COLORS_MAP.keys())): for i in range(len(DISCRETE_COLORS_MAP.keys())): - figure_df.loc[figure_df.shape[0]] = [start_date + datetime.timedelta(days=j), 0, list(DISCRETE_COLORS_MAP.keys())[i]] + figure_df.loc[figure_df.shape[0]] = [start_date + + datetime.timedelta(days=j), 0, list(DISCRETE_COLORS_MAP.keys())[i]] fig = px.line( figure_df, x="createdDate", @@ -202,7 +204,7 @@ def update_figure(selected_council): def update_council_figure(selected_council): pie_df = df.query(f"councilName == '{selected_council}' and createdDate >= '{start_date}'").groupby(['typeName']).agg('count').reset_index() # noqa if pie_df.shape[0] == 0: - pie_df = pd.DataFrame(columns = ["srnumber", "typeName"]) + pie_df = pd.DataFrame(columns=["srnumber", "typeName"]) pie_df.loc[0] = [12345678, "No Request at this time"] for i in range(len(DISCRETE_COLORS_MAP.keys())): pie_df.loc[i] = [12345678, list(DISCRETE_COLORS_MAP.keys())[i]] From ddaca7359046b75e74881b65ef8581187c6c20de Mon Sep 17 00:00:00 2001 From: Yi Heng Joshua Wu Date: Thu, 7 Jul 2022 18:16:55 -0700 Subject: [PATCH 23/37] Add flask request back --- server/dash/dashboards/neighborhood_recent.py | 1 + 1 file changed, 1 insertion(+) diff --git a/server/dash/dashboards/neighborhood_recent.py b/server/dash/dashboards/neighborhood_recent.py index d10ad9056..f3237561b 100644 --- a/server/dash/dashboards/neighborhood_recent.py +++ b/server/dash/dashboards/neighborhood_recent.py @@ -7,6 +7,7 @@ from dash.dependencies import Input, Output import pandas as pd import plotly.express as px +from flask import request from app import app, batch_get_data from config import API_HOST From edfbfaffddab8de67b57241745a7cf221f1ece2b Mon Sep 17 00:00:00 2001 From: Nicholas Kwon Date: Thu, 7 Jul 2022 18:25:48 -0700 Subject: [PATCH 24/37] beginning to work on date range logic --- client/components/Map/index.js | 93 ++++++++++++++++++++++++++++++---- client/redux/reducers/data.js | 12 +++++ 2 files changed, 95 insertions(+), 10 deletions(-) diff --git a/client/components/Map/index.js b/client/components/Map/index.js index 3ce7bef63..bc59bc3c3 100644 --- a/client/components/Map/index.js +++ b/client/components/Map/index.js @@ -5,7 +5,7 @@ import PropTypes from 'proptypes'; import { connect } from 'react-redux'; import { withStyles } from '@material-ui/core/styles'; import axios from 'axios'; -import { getDataRequestSuccess } from '@reducers/data'; +import { getDataRequestSuccess, updateDateRanges } from '@reducers/data'; import { updateMapPosition } from '@reducers/ui'; import { trackMapExport } from '@reducers/analytics'; import { INTERNAL_DATE_SPEC } from '../common/CONSTANTS'; @@ -48,15 +48,81 @@ class MapContainer extends React.Component { } componentDidUpdate(prevProps) { - if (prevProps.activeMode !== this.props.activeMode || - prevProps.pins !== this.props.pins) + const { activeMode, pins, startDate, endDate } = this.props; + if (prevProps.activeMode !== activeMode || prevProps.pins !== pins || + prevProps.startDate != startDate || prevProps.endDate != endDate) { this.setData(); + } } componentWillUnmount() { this.isSubscribed = false; } + getNonOverlappingRanges = (startA, endA, startB, endB) => { + var leftOverlap = null; + var rightOverlap = null; + if (startA < startB){ + leftOverlap = [startA, startB]; + } + if (endB < endA){ + rightOverlap = [endB, endA]; + } + return [leftOverlap, rightOverlap]; + } + + getMissingDateRanges = (startDate, endDate) => { + const {dateRangesWithRequests} = this.props; + var missingDateRanges = []; + var currentStartDate = startDate; + var currentEndDate = endDate; + for (let dateRange of dateRangesWithRequests.values()){ + const nonOverlappingRanges = this.getNonOverlappingRanges(currentStartDate, + currentEndDate, dateRange[0], dateRange[1]); + if (nonOverlappingRanges[0] !== null){ + missingDateRanges.push(nonOverlappingRanges[0]); + } + if (nonOverlappingRanges[1] === null){ + return missingDateRanges; + } + currentStartDate = nonOverlappingRanges[1][0]; + currentEndDate = nonOverlappingRanges[1][1]; + } + missingDateRanges.push([currentStartDate, currentEndDate]); + return missingDateRanges; + } + + resolveDateRanges = (startDate, endDate) => { + const {dateRangesWithRequests} = this.props; + if (dateRangesWithRequests.length === 0){ + return [[startDate, endDate]]; + } + var newDateRanges = []; + var currentStartDate = startDate; + var currentEndDate = endDate; + for (let dateRange of dateRangesWithRequests.values()){ + const nonOverlappingRanges = this.getNonOverlappingRanges(currentStartDate, + currentEndDate, dateRange[0], dateRange[1]); + const leftOverlap = nonOverlappingRanges[0]; + const rightOverlap = nonOverlappingRanges[1]; + if (leftOverlap === null && rightOverlap === null){ + newDateRanges.push([dateRange]); + } + if (leftOverlap !== null){ + currentStartDate = leftOverlap[0]; + } + if (rightOverlap === null){ + currentEndDate = dateRange[1]; + newDateRanges.push([currentStartDate, currentEndDate]); + } + } + // Only sometimes need to add this... + newDateRanges.push([currentStartDate, currentEndDate]); + // Sort newDateRanges by startDate. + // Merge adjacent date ranges. + return newDateRanges; + } + /** * Gets all the dates within a given date range. * @param {string} startDate A date in INTERNAL_DATE_SPEC format. @@ -81,8 +147,7 @@ class MapContainer extends React.Component { * Since the server is slow to retrieve all the requests at once, we need to * make multiple API calls, one for each day. */ - getAllRequests = async () => { - const { startDate, endDate } = this.props; + getAllRequests = async (startDate, endDate) => { const datesInRange = this.getDatesInRange(startDate, endDate); var requests = []; for (let i in datesInRange){ @@ -98,15 +163,21 @@ class MapContainer extends React.Component { }; setData = async () => { - const { pins } = this.props; + const { startDate, endDate } = this.props; - if (this.rawRequests.length === 0) { - await this.getAllRequests(); + const missingDateRanges = this.getMissingDateRanges(startDate, endDate); + if (missingDateRanges.length !== 0){ + this.rawRequests = []; + for (let i in missingDateRanges){ + await this.getAllRequests(missingDateRanges[i][0], missingDateRanges[i][1]); + } } if (this.isSubscribed) { - const { getDataSuccess } = this.props; + const { getDataSuccess, updateDateRangesWithRequests } = this.props; getDataSuccess(this.convertRequests(this.rawRequests)); + const newDateRangesWithRequests = this.resolveDateRanges(startDate, endDate); + updateDateRangesWithRequests(newDateRangesWithRequests); } }; @@ -170,13 +241,15 @@ const mapStateToProps = state => ({ requestTypes: state.filters.requestTypes, startDate: state.filters.startDate, endDate: state.filters.endDate, - requests: state.data.requests + requests: state.data.requests, + dateRangesWithRequests: state.data.dateRangesWithRequests, }); const mapDispatchToProps = dispatch => ({ updatePosition: position => dispatch(updateMapPosition(position)), exportMap: () => dispatch(trackMapExport()), getDataSuccess: data => dispatch(getDataRequestSuccess(data)), + updateDateRangesWithRequests: dateRanges => dispatch(updateDateRanges(dateRanges)), }); MapContainer.propTypes = {}; diff --git a/client/redux/reducers/data.js b/client/redux/reducers/data.js index 0460dd744..79347f2a5 100644 --- a/client/redux/reducers/data.js +++ b/client/redux/reducers/data.js @@ -1,6 +1,7 @@ export const types = { GET_DATA_REQUEST: 'GET_DATA_REQUEST', GET_DATA_REQUEST_SUCCESS: 'GET_DATA_REQUEST_SUCCESS', + UPDATE_DATE_RANGES: 'UPDATE_DATE_RANGES', GET_PINS_SUCCESS: 'GET_PINS_SUCCESS', GET_PINS_FAILURE: 'GET_PINS_FAILURE', GET_OPEN_REQUESTS: 'GET_OPEN_REQUESTS', @@ -31,6 +32,11 @@ export const getDataRequestSuccess = response => ({ payload: response, }); +export const updateDateRanges = dateRanges => ({ + type: types.UPDATE_DATE_RANGES, + payload: dateRanges, +}); + export const getPinsSuccess = response => ({ type: types.GET_PINS_SUCCESS, payload: response, @@ -120,6 +126,7 @@ const initialState = { selectedNcId: null, // Empty GeoJSON object. requests: { type: 'FeatureCollection', features: [] }, + dateRangesWithRequests: [], }; export default (state = initialState, action) => { @@ -135,6 +142,11 @@ export default (state = initialState, action) => { ...state, requests: action.payload, }; + case types.UPDATE_DATE_RANGES: + return { + ...state, + dateRangesWithRequests: action.payload, + }; case types.GET_PINS_SUCCESS: return { ...state, From 3aadeafca717a23ce3733ad6e876eb958b3c79b6 Mon Sep 17 00:00:00 2001 From: Yi Heng Joshua Wu Date: Fri, 8 Jul 2022 15:15:59 -0700 Subject: [PATCH 25/37] Add more descriptive names --- server/dash/dashboards/neighborhood_recent.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/server/dash/dashboards/neighborhood_recent.py b/server/dash/dashboards/neighborhood_recent.py index f3237561b..877969161 100644 --- a/server/dash/dashboards/neighborhood_recent.py +++ b/server/dash/dashboards/neighborhood_recent.py @@ -50,14 +50,13 @@ table_df = df.query(f"councilName == '{selected_council}'")[['srnumber', 'createdDate', 'closedDate', 'typeName', 'agencyName', 'sourceName', 'address']] # noqa # figure_df = df.query(f"councilName == '{selected_council}' and createdDate >= '{start_date}'").groupby(['createdDate', 'typeName'], as_index=False)['srnumber'].count() # noqa -# Populate the neighborhood dropdown -fig = px.line() -apply_figure_style(fig) - -fig2 = px.pie() -apply_figure_style(fig2) +req_type_line_base_graph = px.line() +apply_figure_style(req_type_line_base_graph) +req_type_pie_base_graph = px.pie() +apply_figure_style(req_type_pie_base_graph) +# Populate the neighborhood dropdown def populate_options(): council_df_path = '/councils' council_df = pd.read_json(API_HOST + council_df_path) @@ -97,11 +96,11 @@ def populate_options(): ], className="graph-row"), html.Div([ html.Div( - dcc.Graph(id='graph', figure=fig, config=CONFIG_OPTIONS), + dcc.Graph(id='graph', figure=req_type_line_base_graph, config=CONFIG_OPTIONS), className="half-graph" ), html.Div( - dcc.Graph(id='pie_graph', figure=fig2, config=CONFIG_OPTIONS), + dcc.Graph(id='pie_graph', figure=req_type_pie_base_graph, config=CONFIG_OPTIONS), className="half-graph" ) ]), From 0ac238281b0c7c42e1bb0ead0c5dc94e12a0d7af Mon Sep 17 00:00:00 2001 From: Yi Heng Joshua Wu Date: Fri, 8 Jul 2022 15:20:29 -0700 Subject: [PATCH 26/37] Callback return data --- server/dash/dashboards/neighborhood_recent.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/dash/dashboards/neighborhood_recent.py b/server/dash/dashboards/neighborhood_recent.py index 877969161..739449532 100644 --- a/server/dash/dashboards/neighborhood_recent.py +++ b/server/dash/dashboards/neighborhood_recent.py @@ -48,7 +48,6 @@ selected_council = 'Arleta' table_df = df.query(f"councilName == '{selected_council}'")[['srnumber', 'createdDate', 'closedDate', 'typeName', 'agencyName', 'sourceName', 'address']] # noqa -# figure_df = df.query(f"councilName == '{selected_council}' and createdDate >= '{start_date}'").groupby(['createdDate', 'typeName'], as_index=False)['srnumber'].count() # noqa req_type_line_base_graph = px.line() apply_figure_style(req_type_line_base_graph) @@ -138,12 +137,12 @@ def populate_options(): ) def update_table(selected_council): table_df = df.query(f"councilName == '{selected_council}'")[['srnumber', 'createdDate', 'closedDate', 'typeName', 'agencyName', 'sourceName', 'address']] # noqa + # The following check is to ensure Dash graphs are populate with dummy data when query returns empty dataframe if table_df.shape[0] == 0: table_df = pd.DataFrame(columns=["Request Type"]) for i in range(len(DISCRETE_COLORS_MAP.keys())): table_df.loc[i] = [list(DISCRETE_COLORS_MAP.keys())[i]] - else: - return table_df.to_dict('records') + return table_df.to_dict('records') @app.callback( @@ -170,6 +169,7 @@ def update_text(selected_council): def update_figure(selected_council): figure_df = df.query(f"councilName == '{selected_council}' and createdDate >= '{start_date}'").groupby(['createdDate', 'typeName'])['srnumber'].count().reset_index() # noqa figure_df.typeName = figure_df.typeName.map(lambda x: '
'.join(textwrap.wrap(x, width=16))) # noqa + #The following check is to ensure Dash graphs are populate with dummy data when query returns empty dataframe if figure_df.shape[0] == 0: figure_df = pd.DataFrame(columns=['createdDate', "srnumber", "typeName"]) for j in range(len(DISCRETE_COLORS_MAP.keys())): From 324b679ca91f99aff81c375ad1f35ec2ae577f4b Mon Sep 17 00:00:00 2001 From: Yi Heng Joshua Wu Date: Fri, 8 Jul 2022 15:26:30 -0700 Subject: [PATCH 27/37] Fix indicies --- server/dash/dashboards/neighborhood_recent.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/server/dash/dashboards/neighborhood_recent.py b/server/dash/dashboards/neighborhood_recent.py index 739449532..c476822c1 100644 --- a/server/dash/dashboards/neighborhood_recent.py +++ b/server/dash/dashboards/neighborhood_recent.py @@ -140,8 +140,10 @@ def update_table(selected_council): # The following check is to ensure Dash graphs are populate with dummy data when query returns empty dataframe if table_df.shape[0] == 0: table_df = pd.DataFrame(columns=["Request Type"]) - for i in range(len(DISCRETE_COLORS_MAP.keys())): - table_df.loc[i] = [list(DISCRETE_COLORS_MAP.keys())[i]] + i = 0 + for request_type in DISCRETE_COLORS_MAP: + table_df.loc[i] = [request_type] + i += 1 return table_df.to_dict('records') @@ -172,7 +174,7 @@ def update_figure(selected_council): #The following check is to ensure Dash graphs are populate with dummy data when query returns empty dataframe if figure_df.shape[0] == 0: figure_df = pd.DataFrame(columns=['createdDate', "srnumber", "typeName"]) - for j in range(len(DISCRETE_COLORS_MAP.keys())): + for j in range(START_DATE_DELTA): for i in range(len(DISCRETE_COLORS_MAP.keys())): figure_df.loc[figure_df.shape[0]] = [start_date + datetime.timedelta(days=j), 0, list(DISCRETE_COLORS_MAP.keys())[i]] @@ -205,9 +207,10 @@ def update_council_figure(selected_council): pie_df = df.query(f"councilName == '{selected_council}' and createdDate >= '{start_date}'").groupby(['typeName']).agg('count').reset_index() # noqa if pie_df.shape[0] == 0: pie_df = pd.DataFrame(columns=["srnumber", "typeName"]) - pie_df.loc[0] = [12345678, "No Request at this time"] - for i in range(len(DISCRETE_COLORS_MAP.keys())): - pie_df.loc[i] = [12345678, list(DISCRETE_COLORS_MAP.keys())[i]] + i = 0 + for request_type in DISCRETE_COLORS_MAP: + pie_df.loc[i] = [12345678, request_type] + i += 1 pie_fig = px.pie( pie_df, names="typeName", From ef5fcc84481ce7f7d693655d57b68be05a845d55 Mon Sep 17 00:00:00 2001 From: Yi Heng Joshua Wu Date: Fri, 8 Jul 2022 15:34:34 -0700 Subject: [PATCH 28/37] Update checking condition --- server/dash/dashboards/neighborhood_recent.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/server/dash/dashboards/neighborhood_recent.py b/server/dash/dashboards/neighborhood_recent.py index c476822c1..8a39820ec 100644 --- a/server/dash/dashboards/neighborhood_recent.py +++ b/server/dash/dashboards/neighborhood_recent.py @@ -56,6 +56,8 @@ apply_figure_style(req_type_pie_base_graph) # Populate the neighborhood dropdown + + def populate_options(): council_df_path = '/councils' council_df = pd.read_json(API_HOST + council_df_path) @@ -158,7 +160,7 @@ def update_table(selected_council): def update_text(selected_council): create_count = df.query(f"councilName == '{selected_council}' and createdDate >= '{start_date}'")['srnumber'].count() # noqa close_count = df.query(f"councilName == '{selected_council}' and closedDate >= '{start_date}'")['srnumber'].count() # noqa - if create_count == 0 and close_count == 0: + if create_count == 0 and close_count > 0: return 0, 0, 0 else: return create_count, close_count, create_count - close_count @@ -170,14 +172,14 @@ def update_text(selected_council): ) def update_figure(selected_council): figure_df = df.query(f"councilName == '{selected_council}' and createdDate >= '{start_date}'").groupby(['createdDate', 'typeName'])['srnumber'].count().reset_index() # noqa - figure_df.typeName = figure_df.typeName.map(lambda x: '
'.join(textwrap.wrap(x, width=16))) # noqa - #The following check is to ensure Dash graphs are populate with dummy data when query returns empty dataframe + # The following check is to ensure Dash graphs are populate with dummy data when query returns empty dataframe if figure_df.shape[0] == 0: - figure_df = pd.DataFrame(columns=['createdDate', "srnumber", "typeName"]) + figure_df = pd.DataFrame(columns=["createdDate", "srnumber", "typeName"]) for j in range(START_DATE_DELTA): - for i in range(len(DISCRETE_COLORS_MAP.keys())): - figure_df.loc[figure_df.shape[0]] = [start_date + - datetime.timedelta(days=j), 0, list(DISCRETE_COLORS_MAP.keys())[i]] + for request_type in DISCRETE_COLORS_MAP: + figure_df.loc[figure_df.shape[0]] = [ + start_date + datetime.timedelta(days=j), 0, request_type] + figure_df.typeName = figure_df.typeName.map(lambda x: '
'.join(textwrap.wrap(x, width=16))) # noqa fig = px.line( figure_df, x="createdDate", From 0925e408342e230d483055aa6d6b43c219f00c4c Mon Sep 17 00:00:00 2001 From: Yi Heng Joshua Wu Date: Fri, 8 Jul 2022 15:35:32 -0700 Subject: [PATCH 29/37] Update checking condiiton comment --- server/dash/dashboards/neighborhood_recent.py | 1 + 1 file changed, 1 insertion(+) diff --git a/server/dash/dashboards/neighborhood_recent.py b/server/dash/dashboards/neighborhood_recent.py index 8a39820ec..c15ca0c86 100644 --- a/server/dash/dashboards/neighborhood_recent.py +++ b/server/dash/dashboards/neighborhood_recent.py @@ -160,6 +160,7 @@ def update_table(selected_council): def update_text(selected_council): create_count = df.query(f"councilName == '{selected_council}' and createdDate >= '{start_date}'")['srnumber'].count() # noqa close_count = df.query(f"councilName == '{selected_council}' and closedDate >= '{start_date}'")['srnumber'].count() # noqa + # This check is to ensure data quality issues doesn't flow downstream to the dashboard (closed request exist without new request) if create_count == 0 and close_count > 0: return 0, 0, 0 else: From 76699aeb9822eedfdccd0d8f7db71b25e42bbcaf Mon Sep 17 00:00:00 2001 From: Yi Heng Joshua Wu Date: Fri, 8 Jul 2022 15:41:39 -0700 Subject: [PATCH 30/37] Pie chart value set to 0 --- server/dash/dashboards/neighborhood_recent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/dash/dashboards/neighborhood_recent.py b/server/dash/dashboards/neighborhood_recent.py index c15ca0c86..57cff8488 100644 --- a/server/dash/dashboards/neighborhood_recent.py +++ b/server/dash/dashboards/neighborhood_recent.py @@ -212,7 +212,7 @@ def update_council_figure(selected_council): pie_df = pd.DataFrame(columns=["srnumber", "typeName"]) i = 0 for request_type in DISCRETE_COLORS_MAP: - pie_df.loc[i] = [12345678, request_type] + pie_df.loc[i] = [0, request_type] i += 1 pie_fig = px.pie( pie_df, From eef86257f829b2538b765f66f89e2c919c8d75bc Mon Sep 17 00:00:00 2001 From: Nicholas Kwon Date: Mon, 11 Jul 2022 14:41:17 -0700 Subject: [PATCH 31/37] resolve date ranges --- client/components/Map/index.js | 128 +++++++++++++++++++++------------ client/redux/reducers/data.js | 9 ++- 2 files changed, 89 insertions(+), 48 deletions(-) diff --git a/client/components/Map/index.js b/client/components/Map/index.js index bc59bc3c3..2cee4b009 100644 --- a/client/components/Map/index.js +++ b/client/components/Map/index.js @@ -59,18 +59,42 @@ class MapContainer extends React.Component { this.isSubscribed = false; } + /** + * Returns the non-overlapping date ranges of A before and after B. + * @param {string} startA The start date of range A in INTERNAL_DATE_SPEC format. + * @param {string} endA The end date of range A in INTERNAL_DATE_SPEC format. + * @param {string} startB The start date of range B in INTERNAL_DATE_SPEC format. + * @param {string} endB The end date of range B in INTERNAL_DATE_SPEC format. + * @returns An array of two elements: the first element is the non-overlapping + * range of A before B; the second is the non-overlapping range of A after B. + * Each element can be null if there is no non-overlappping range. + */ getNonOverlappingRanges = (startA, endA, startB, endB) => { var leftOverlap = null; var rightOverlap = null; - if (startA < startB){ - leftOverlap = [startA, startB]; + if (moment(startA) < moment(startB)){ + leftOverlap = [startA, moment(startB).subtract(1, 'days').format(INTERNAL_DATE_SPEC)]; } - if (endB < endA){ - rightOverlap = [endB, endA]; + if (moment(endB) < moment(endA)){ + rightOverlap = [moment(endB).add(1, 'days').format(INTERNAL_DATE_SPEC), endA]; } return [leftOverlap, rightOverlap]; } + /** + * Returns the missing date ranges of a new date range against the existing + * date ranges in the Redux store. + * + * In our Redux store, we keep track of date ranges that we already have 311 + * requests for. When the user changes the date range, we need to check + * whether we need to retrieve more data; if we do, we only want to pull the + * data from the date ranges that aren't already in the store. + * + * @param {*} startDate The start date in INTERNAL_DATE_SPEC format. + * @param {*} endDate The end date in INTERNAL_DATE_SPEC format. + * @returns An array of date ranges, where each date range is represented as + * an array of string start and end dates. + */ getMissingDateRanges = (startDate, endDate) => { const {dateRangesWithRequests} = this.props; var missingDateRanges = []; @@ -92,35 +116,44 @@ class MapContainer extends React.Component { return missingDateRanges; } - resolveDateRanges = (startDate, endDate) => { + /** + * Returns the updated date ranges given the date ranges that we just pulled + * data for. + * @param {Array} newDateRanges The new date ranges that we just pulled data for. + * @returns The updated, complete array of date ranges for which we have data + * in the Redux store. + */ + resolveDateRanges = (newDateRanges) => { const {dateRangesWithRequests} = this.props; - if (dateRangesWithRequests.length === 0){ - return [[startDate, endDate]]; - } - var newDateRanges = []; - var currentStartDate = startDate; - var currentEndDate = endDate; - for (let dateRange of dateRangesWithRequests.values()){ - const nonOverlappingRanges = this.getNonOverlappingRanges(currentStartDate, - currentEndDate, dateRange[0], dateRange[1]); - const leftOverlap = nonOverlappingRanges[0]; - const rightOverlap = nonOverlappingRanges[1]; - if (leftOverlap === null && rightOverlap === null){ - newDateRanges.push([dateRange]); - } - if (leftOverlap !== null){ - currentStartDate = leftOverlap[0]; + var allDateRanges = dateRangesWithRequests.concat(newDateRanges); + // Sort date ranges by startDate. Since newDateRanges was retrieved using + // getMissingDateRanges, there should be no overlapping date ranges in the + // allDateRanges. + const sortedDateRanges = allDateRanges.sort(function(dateRangeA, dateRangeB){ + return moment(dateRangeA[0]) - moment(dateRangeB[0])}); + var resolvedDateRanges = []; + var currentStart = null; + var currentEnd = null; + for (const dateRange of sortedDateRanges){ + if (currentStart === null){ + currentStart = dateRange[0]; + currentEnd = dateRange[1]; + continue; } - if (rightOverlap === null){ - currentEndDate = dateRange[1]; - newDateRanges.push([currentStartDate, currentEndDate]); + // Check if the current date range is adjacent to the next date range. + if (moment(currentEnd).add(1, 'days').valueOf() === moment(dateRange[0]).valueOf()){ + // Extend the current date range to include the next date range. + currentEnd = dateRange[1]; + } else { + resolvedDateRanges.push([currentStart, currentEnd]); + currentStart = null; + currentEnd = null; } } - // Only sometimes need to add this... - newDateRanges.push([currentStartDate, currentEndDate]); - // Sort newDateRanges by startDate. - // Merge adjacent date ranges. - return newDateRanges; + if (currentStart !== null){ + resolvedDateRanges.push([currentStart, currentEnd]); + } + return resolvedDateRanges; } /** @@ -147,43 +180,46 @@ class MapContainer extends React.Component { * Since the server is slow to retrieve all the requests at once, we need to * make multiple API calls, one for each day. */ - getAllRequests = async (startDate, endDate) => { + getAllRequests = (startDate, endDate) => { const datesInRange = this.getDatesInRange(startDate, endDate); var requests = []; - for (let i in datesInRange){ + for (const date of datesInRange){ const url = new URL(`${process.env.API_URL}/requests`); - url.searchParams.append("start_date", datesInRange[i]); - url.searchParams.append("end_date", datesInRange[i]); + url.searchParams.append("start_date", date); + url.searchParams.append("end_date", date); url.searchParams.append("limit", `${REQUEST_LIMIT}`); requests.push(axios.get(url)); } - await Promise.all(requests).then(responses => { - responses.forEach(response => this.rawRequests.push(...response.data)) - }); + return requests; }; setData = async () => { const { startDate, endDate } = this.props; const missingDateRanges = this.getMissingDateRanges(startDate, endDate); - if (missingDateRanges.length !== 0){ - this.rawRequests = []; - for (let i in missingDateRanges){ - await this.getAllRequests(missingDateRanges[i][0], missingDateRanges[i][1]); - } + if (missingDateRanges.length === 0){ + return; + } + this.rawRequests = []; + var allRequestPromises = []; + for (const missingDateRange of missingDateRanges){ + const requestPromises = this.getAllRequests(missingDateRange[0], + missingDateRange[1]); + allRequestPromises.push(...requestPromises); } + await Promise.all(allRequestPromises).then(responses => { + responses.forEach(response => this.rawRequests.push(...response.data)) + }); if (this.isSubscribed) { const { getDataSuccess, updateDateRangesWithRequests } = this.props; getDataSuccess(this.convertRequests(this.rawRequests)); - const newDateRangesWithRequests = this.resolveDateRanges(startDate, endDate); + const newDateRangesWithRequests = this.resolveDateRanges(missingDateRanges); updateDateRangesWithRequests(newDateRangesWithRequests); } }; - convertRequests = requests => ({ - type: 'FeatureCollection', - features: requests.map(request => ({ + convertRequests = requests => (requests.map(request => ({ type: 'Feature', properties: { requestId: request.requestId, @@ -201,7 +237,7 @@ class MapContainer extends React.Component { ] } })) - }); + ); // TODO: fix this getSelectedTypes = () => { diff --git a/client/redux/reducers/data.js b/client/redux/reducers/data.js index 79347f2a5..09543f39c 100644 --- a/client/redux/reducers/data.js +++ b/client/redux/reducers/data.js @@ -137,11 +137,16 @@ export default (state = initialState, action) => { isMapLoading: true, isVisLoading: true, }; - case types.GET_DATA_REQUEST_SUCCESS: + case types.GET_DATA_REQUEST_SUCCESS: { + const newRequests = { + type: 'FeatureCollection', + features: [...state.requests.features, ...action.payload], + }; return { ...state, - requests: action.payload, + requests: newRequests, }; + } case types.UPDATE_DATE_RANGES: return { ...state, From b72a404869aec3e97a87b708a7c062102adfef44 Mon Sep 17 00:00:00 2001 From: Yi Heng Joshua Wu Date: Mon, 11 Jul 2022 20:48:42 -0700 Subject: [PATCH 32/37] Using enumerate --- server/dash/dashboards/neighborhood_recent.py | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/server/dash/dashboards/neighborhood_recent.py b/server/dash/dashboards/neighborhood_recent.py index 57cff8488..227cc3a96 100644 --- a/server/dash/dashboards/neighborhood_recent.py +++ b/server/dash/dashboards/neighborhood_recent.py @@ -139,13 +139,11 @@ def populate_options(): ) def update_table(selected_council): table_df = df.query(f"councilName == '{selected_council}'")[['srnumber', 'createdDate', 'closedDate', 'typeName', 'agencyName', 'sourceName', 'address']] # noqa - # The following check is to ensure Dash graphs are populate with dummy data when query returns empty dataframe + # The following check is to ensure Dash graphs are populated with dummy data when query returns empty dataframe. if table_df.shape[0] == 0: table_df = pd.DataFrame(columns=["Request Type"]) - i = 0 - for request_type in DISCRETE_COLORS_MAP: + for i, request_type in enumerate(DISCRETE_COLORS_MAP): table_df.loc[i] = [request_type] - i += 1 return table_df.to_dict('records') @@ -160,7 +158,7 @@ def update_table(selected_council): def update_text(selected_council): create_count = df.query(f"councilName == '{selected_council}' and createdDate >= '{start_date}'")['srnumber'].count() # noqa close_count = df.query(f"councilName == '{selected_council}' and closedDate >= '{start_date}'")['srnumber'].count() # noqa - # This check is to ensure data quality issues doesn't flow downstream to the dashboard (closed request exist without new request) + # This check is to ensure data quality issues don't flow downstream to the dashboard (i.e., closed requests exist without any new requests). if create_count == 0 and close_count > 0: return 0, 0, 0 else: @@ -173,13 +171,12 @@ def update_text(selected_council): ) def update_figure(selected_council): figure_df = df.query(f"councilName == '{selected_council}' and createdDate >= '{start_date}'").groupby(['createdDate', 'typeName'])['srnumber'].count().reset_index() # noqa - # The following check is to ensure Dash graphs are populate with dummy data when query returns empty dataframe + # The following check is to ensure Dash graphs are populated with dummy data when query returns empty dataframe. if figure_df.shape[0] == 0: figure_df = pd.DataFrame(columns=["createdDate", "srnumber", "typeName"]) for j in range(START_DATE_DELTA): for request_type in DISCRETE_COLORS_MAP: - figure_df.loc[figure_df.shape[0]] = [ - start_date + datetime.timedelta(days=j), 0, request_type] + figure_df.loc[j] = [start_date + datetime.timedelta(days=j), 0, request_type] figure_df.typeName = figure_df.typeName.map(lambda x: '
'.join(textwrap.wrap(x, width=16))) # noqa fig = px.line( figure_df, @@ -210,10 +207,8 @@ def update_council_figure(selected_council): pie_df = df.query(f"councilName == '{selected_council}' and createdDate >= '{start_date}'").groupby(['typeName']).agg('count').reset_index() # noqa if pie_df.shape[0] == 0: pie_df = pd.DataFrame(columns=["srnumber", "typeName"]) - i = 0 - for request_type in DISCRETE_COLORS_MAP: + for i, request_type in enumerate(DISCRETE_COLORS_MAP): pie_df.loc[i] = [0, request_type] - i += 1 pie_fig = px.pie( pie_df, names="typeName", From f72cb0575d8616a5118e49e498ff1c303255d9f8 Mon Sep 17 00:00:00 2001 From: Yi Heng Joshua Wu Date: Mon, 11 Jul 2022 22:24:00 -0700 Subject: [PATCH 33/37] Added nc_recent func docstring --- server/dash/dashboards/neighborhood_recent.py | 72 +++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/server/dash/dashboards/neighborhood_recent.py b/server/dash/dashboards/neighborhood_recent.py index 227cc3a96..758d7ddb6 100644 --- a/server/dash/dashboards/neighborhood_recent.py +++ b/server/dash/dashboards/neighborhood_recent.py @@ -59,6 +59,19 @@ def populate_options(): + """Function to populate the dropdown menu with a list of neighborhood councils. + + This function calls the councils API to get a list of neighborhood council and + return a unique list of council as a dictionary sorted in ascending order of requests. + + Typical usage example: + + dcc.Dropdown( + ... + options=populate_options() + ... + ) + """ council_df_path = '/councils' council_df = pd.read_json(API_HOST + council_df_path) values = [] @@ -138,7 +151,21 @@ def populate_options(): Input("council_list", "value") ) def update_table(selected_council): + """Dash Callback Function to update the LA 311 request data table at the bottom of the dashboard. + + This function takes the selected neighborhood council (nc) value from the "council_list" dropdown and + outputs a list of requests associated with that nc as a data table in dictionary form + with id "council_table" in the layout. + + Typical usage example: + + dash_table.DataTable( + id='council_table', + ... + ) + """ table_df = df.query(f"councilName == '{selected_council}'")[['srnumber', 'createdDate', 'closedDate', 'typeName', 'agencyName', 'sourceName', 'address']] # noqa + # The following check is to ensure Dash graphs are populated with dummy data when query returns empty dataframe. if table_df.shape[0] == 0: table_df = pd.DataFrame(columns=["Request Type"]) @@ -156,8 +183,23 @@ def update_table(selected_council): Input("council_list", "value") ) def update_text(selected_council): + """Dash Callback Function to update the indicator cards at the top of the dashboard. + + This function takes the selected neighborhood council (nc) value from the "council_list" dropdown and + outputs the values for the number of new requests, number of closed requests, and net change in requests + (i.e. # closed requests - # new requests) for visualizations on the indicator visuals in the layout. The + corresponding IDs of the indicator visuals are "created_txt", "closed_txt", and "net_txt". + + Typical usage example (using net_txt as example, created_txt and closed_txt are similar): + + html.Div( + [html.H2(id="net_txt"), html.Label("Net Change")], + className="stats-label" + ) + """ create_count = df.query(f"councilName == '{selected_council}' and createdDate >= '{start_date}'")['srnumber'].count() # noqa close_count = df.query(f"councilName == '{selected_council}' and closedDate >= '{start_date}'")['srnumber'].count() # noqa + # This check is to ensure data quality issues don't flow downstream to the dashboard (i.e., closed requests exist without any new requests). if create_count == 0 and close_count > 0: return 0, 0, 0 @@ -170,7 +212,22 @@ def update_text(selected_council): Input("council_list", "value") ) def update_figure(selected_council): + """Dash Callback Function to update the Request Type Line Chart at the middle of the dashboard. + + This function takes the selected neighborhood council (nc) value from the "council_list" dropdown and + outputs the request type line chart that shows the trend of of different requests types over + the time range of the data available in the selected neighborhood conucil. The line chart will + show up inside dcc.Graph object as long as id "graph" is passed in. + + Typical usage example: + + html.Div( + dcc.Graph(id='graph', + ... + ) + """ figure_df = df.query(f"councilName == '{selected_council}' and createdDate >= '{start_date}'").groupby(['createdDate', 'typeName'])['srnumber'].count().reset_index() # noqa + # The following check is to ensure Dash graphs are populated with dummy data when query returns empty dataframe. if figure_df.shape[0] == 0: figure_df = pd.DataFrame(columns=["createdDate", "srnumber", "typeName"]) @@ -204,7 +261,22 @@ def update_figure(selected_council): Input("council_list", "value") ) def update_council_figure(selected_council): + """Dash Callback Function to update the Request Type Pie Chart at the middle of the dashboard. + + This function takes the selected neighborhood council (nc) value from the "council_list" dropdown and + outputs the the pie chart showing the share of each request types. The pie chart will + show up inside dcc.Graph object as long as id "pie_graph" is passed in. + + Typical usage example: + + html.Div( + dcc.Graph(id='pie_graph', + ... + ) + """ pie_df = df.query(f"councilName == '{selected_council}' and createdDate >= '{start_date}'").groupby(['typeName']).agg('count').reset_index() # noqa + + # The following check is to ensure Dash graphs are populated with dummy data when query returns empty dataframe. if pie_df.shape[0] == 0: pie_df = pd.DataFrame(columns=["srnumber", "typeName"]) for i, request_type in enumerate(DISCRETE_COLORS_MAP): From b35dd6af880d189b9eb291137a37a893041f4a76 Mon Sep 17 00:00:00 2001 From: Yi Heng Joshua Wu Date: Wed, 13 Jul 2022 09:18:51 -0700 Subject: [PATCH 34/37] Resolve empty df issues --- server/dash/dashboards/neighborhood_recent.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/server/dash/dashboards/neighborhood_recent.py b/server/dash/dashboards/neighborhood_recent.py index 758d7ddb6..0eb4e2c82 100644 --- a/server/dash/dashboards/neighborhood_recent.py +++ b/server/dash/dashboards/neighborhood_recent.py @@ -56,8 +56,6 @@ apply_figure_style(req_type_pie_base_graph) # Populate the neighborhood dropdown - - def populate_options(): """Function to populate the dropdown menu with a list of neighborhood councils. @@ -233,7 +231,7 @@ def update_figure(selected_council): figure_df = pd.DataFrame(columns=["createdDate", "srnumber", "typeName"]) for j in range(START_DATE_DELTA): for request_type in DISCRETE_COLORS_MAP: - figure_df.loc[j] = [start_date + datetime.timedelta(days=j), 0, request_type] + figure_df.loc[figure_df.shape[0]] = [start_date + datetime.timedelta(days=j), 0, request_type] figure_df.typeName = figure_df.typeName.map(lambda x: '
'.join(textwrap.wrap(x, width=16))) # noqa fig = px.line( figure_df, @@ -280,7 +278,7 @@ def update_council_figure(selected_council): if pie_df.shape[0] == 0: pie_df = pd.DataFrame(columns=["srnumber", "typeName"]) for i, request_type in enumerate(DISCRETE_COLORS_MAP): - pie_df.loc[i] = [0, request_type] + pie_df.loc[i] = [1, request_type] pie_fig = px.pie( pie_df, names="typeName", From b72cf5718cab0870f06fb67679f47be15a8301df Mon Sep 17 00:00:00 2001 From: Yi Heng Joshua Wu Date: Wed, 13 Jul 2022 21:02:53 -0700 Subject: [PATCH 35/37] Add docstrings args returns --- server/dash/dashboards/neighborhood_recent.py | 61 ++++++++++++++++--- 1 file changed, 53 insertions(+), 8 deletions(-) diff --git a/server/dash/dashboards/neighborhood_recent.py b/server/dash/dashboards/neighborhood_recent.py index 0eb4e2c82..5af92a547 100644 --- a/server/dash/dashboards/neighborhood_recent.py +++ b/server/dash/dashboards/neighborhood_recent.py @@ -57,12 +57,22 @@ # Populate the neighborhood dropdown def populate_options(): - """Function to populate the dropdown menu with a list of neighborhood councils. + """Gets a list of neighborhood councils to populate the dropdown menu. This function calls the councils API to get a list of neighborhood council and return a unique list of council as a dictionary sorted in ascending order of requests. - - Typical usage example: + + Returns: + A list of dictionary mapping label and value to corresponding councilName ordered by the + total number of requests. For example: + + [ + {'label': 'Arleta', 'value': 'Arleta'}, + {'label': 'Arroyo Seco', 'value': 'Arroyo Seco'}, + ... + ] + + Typical usage example: dcc.Dropdown( ... @@ -149,12 +159,19 @@ def populate_options(): Input("council_list", "value") ) def update_table(selected_council): - """Dash Callback Function to update the LA 311 request data table at the bottom of the dashboard. + """Filters the LA 311 request data table based on selected_council. This function takes the selected neighborhood council (nc) value from the "council_list" dropdown and outputs a list of requests associated with that nc as a data table in dictionary form with id "council_table" in the layout. + Args: + selected_council: + A string argument automatically detected by Dash callback function when "council_list" element is selected in the layout. + + Returns: + A list of dictionary mapping column names to values. For example: [{'srnumber':1234567}, {'createdDate':'2022-07-11'}, {'closeDate':'2022-07-14'}...] + Typical usage example: dash_table.DataTable( @@ -181,13 +198,25 @@ def update_table(selected_council): Input("council_list", "value") ) def update_text(selected_council): - """Dash Callback Function to update the indicator cards at the top of the dashboard. + """Updates the indicator cards based on data filtered by selected_council. This function takes the selected neighborhood council (nc) value from the "council_list" dropdown and outputs the values for the number of new requests, number of closed requests, and net change in requests (i.e. # closed requests - # new requests) for visualizations on the indicator visuals in the layout. The corresponding IDs of the indicator visuals are "created_txt", "closed_txt", and "net_txt". + Args: + selected_council: + A string argument automatically detected by Dash callback function when "council_list" element is selected in the layout. + + Returns: + created_txt: + Integer for the number of new requests created since the start date. + closed_txt: + Integer for the number of close requests since the start date. + net_txt: + Integer for the difference in close requests and new requests since the start date. + Typical usage example (using net_txt as example, created_txt and closed_txt are similar): html.Div( @@ -202,7 +231,7 @@ def update_text(selected_council): if create_count == 0 and close_count > 0: return 0, 0, 0 else: - return create_count, close_count, create_count - close_count + return create_count, close_count, close_count - create_count @app.callback( @@ -210,13 +239,21 @@ def update_text(selected_council): Input("council_list", "value") ) def update_figure(selected_council): - """Dash Callback Function to update the Request Type Line Chart at the middle of the dashboard. + """Updates the Request Type Line Chart based on data filtered by selected_council. This function takes the selected neighborhood council (nc) value from the "council_list" dropdown and outputs the request type line chart that shows the trend of of different requests types over the time range of the data available in the selected neighborhood conucil. The line chart will show up inside dcc.Graph object as long as id "graph" is passed in. + Args: + selected_council: + A string argument automatically detected by Dash callback function when "council_list" element is selected in the layout. + + Returns: + graph: + Plotly line chart of the total number of requests over time (createdDate) separated by request type. + Typical usage example: html.Div( @@ -259,12 +296,20 @@ def update_figure(selected_council): Input("council_list", "value") ) def update_council_figure(selected_council): - """Dash Callback Function to update the Request Type Pie Chart at the middle of the dashboard. + """Updates the Request Type Pie Chart based on data filtered by selected_council. This function takes the selected neighborhood council (nc) value from the "council_list" dropdown and outputs the the pie chart showing the share of each request types. The pie chart will show up inside dcc.Graph object as long as id "pie_graph" is passed in. + Args: + selected_council: + A string argument automatically detected by Dash callback function when "council_list" element is selected in the layout. + + Returns: + pie_graph + Plotly pie chart for the share of different request types. + Typical usage example: html.Div( From 373eab016eb5decef2310acee13f03d37c269af3 Mon Sep 17 00:00:00 2001 From: Nicholas Kwon Date: Thu, 14 Jul 2022 13:03:44 -0700 Subject: [PATCH 36/37] update comments for date range logic --- client/components/Map/index.js | 35 +++++++++++++++++++++++++++------- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/client/components/Map/index.js b/client/components/Map/index.js index 2cee4b009..33caf09ce 100644 --- a/client/components/Map/index.js +++ b/client/components/Map/index.js @@ -70,15 +70,21 @@ class MapContainer extends React.Component { * Each element can be null if there is no non-overlappping range. */ getNonOverlappingRanges = (startA, endA, startB, endB) => { - var leftOverlap = null; - var rightOverlap = null; + var leftNonOverlap = null; + var rightNonOverlap = null; + // If date range A starts before date range B, then it has a subrange that + // does not overlap with B. if (moment(startA) < moment(startB)){ - leftOverlap = [startA, moment(startB).subtract(1, 'days').format(INTERNAL_DATE_SPEC)]; + leftNonOverlap = [startA, + moment(startB).subtract(1, 'days').format(INTERNAL_DATE_SPEC)]; } + // If date range A ends after date range B, then it has a subrange that does + // not overlap with B. if (moment(endB) < moment(endA)){ - rightOverlap = [moment(endB).add(1, 'days').format(INTERNAL_DATE_SPEC), endA]; + rightNonOverlap = [moment(endB).add(1, 'days').format(INTERNAL_DATE_SPEC), + endA]; } - return [leftOverlap, rightOverlap]; + return [leftNonOverlap, rightNonOverlap]; } /** @@ -90,8 +96,8 @@ class MapContainer extends React.Component { * whether we need to retrieve more data; if we do, we only want to pull the * data from the date ranges that aren't already in the store. * - * @param {*} startDate The start date in INTERNAL_DATE_SPEC format. - * @param {*} endDate The end date in INTERNAL_DATE_SPEC format. + * @param {string} startDate The start date in INTERNAL_DATE_SPEC format. + * @param {string} endDate The end date in INTERNAL_DATE_SPEC format. * @returns An array of date ranges, where each date range is represented as * an array of string start and end dates. */ @@ -100,6 +106,16 @@ class MapContainer extends React.Component { var missingDateRanges = []; var currentStartDate = startDate; var currentEndDate = endDate; + // Compare the input date range with each date range with requests, which + // are ordered chronologically from first to last. Every left non-overlapping + // date range (i.e., a portion of the input date range that comes before the + // existing date range with requests) is immediately added to the list of + // missing date ranges. Otherwise, if there is overlap on the left (i.e., + // the input range is covered on the left side by the date range with + // requests), we push the start date for our input range forward to the end + // of the date range with requests. The process continues for every date + // range with requests. + // It stops when the input date range is covered on the right side. for (let dateRange of dateRangesWithRequests.values()){ const nonOverlappingRanges = this.getNonOverlappingRanges(currentStartDate, currentEndDate, dateRange[0], dateRange[1]); @@ -179,6 +195,11 @@ class MapContainer extends React.Component { * * Since the server is slow to retrieve all the requests at once, we need to * make multiple API calls, one for each day. + * + * @param {string} startDate A date in INTERNAL_DATE_SPEC format. + * @param {string} endDate A date in INTERNAL_DATE_SPEC format. + * @returns An array of Promises, each representing an API request for a + * particular day in the input date range. */ getAllRequests = (startDate, endDate) => { const datesInRange = this.getDatesInRange(startDate, endDate); From a1721a529e88d07e304070109c7eea67f6c7ee3e Mon Sep 17 00:00:00 2001 From: Yi Heng Joshua Wu Date: Thu, 14 Jul 2022 17:51:40 -0700 Subject: [PATCH 37/37] Update docstrings --- server/dash/dashboards/neighborhood_recent.py | 34 +++++++------------ 1 file changed, 13 insertions(+), 21 deletions(-) diff --git a/server/dash/dashboards/neighborhood_recent.py b/server/dash/dashboards/neighborhood_recent.py index 5af92a547..bc440fefd 100644 --- a/server/dash/dashboards/neighborhood_recent.py +++ b/server/dash/dashboards/neighborhood_recent.py @@ -63,7 +63,7 @@ def populate_options(): return a unique list of council as a dictionary sorted in ascending order of requests. Returns: - A list of dictionary mapping label and value to corresponding councilName ordered by the + A list of dictionaries mapping label and value to corresponding councilName ordered by the total number of requests. For example: [ @@ -166,11 +166,10 @@ def update_table(selected_council): with id "council_table" in the layout. Args: - selected_council: - A string argument automatically detected by Dash callback function when "council_list" element is selected in the layout. + selected_council: A string argument automatically detected by Dash callback function when "council_list" element is selected in the layout. - Returns: - A list of dictionary mapping column names to values. For example: [{'srnumber':1234567}, {'createdDate':'2022-07-11'}, {'closeDate':'2022-07-14'}...] + Returns: + A list of dictionaries mapping column names to values. For example: [{'srnumber':1234567, 'createdDate':'2022-07-11', 'closeDate':'2022-07-14'...}, {...}, ... ] Typical usage example: @@ -206,16 +205,13 @@ def update_text(selected_council): corresponding IDs of the indicator visuals are "created_txt", "closed_txt", and "net_txt". Args: - selected_council: - A string argument automatically detected by Dash callback function when "council_list" element is selected in the layout. + selected_council: a string argument automatically detected by Dash callback function when "council_list" element is selected in the layout. Returns: - created_txt: - Integer for the number of new requests created since the start date. - closed_txt: - Integer for the number of close requests since the start date. - net_txt: - Integer for the difference in close requests and new requests since the start date. + A tuple containing three integers: + 1) Integer for the number of new requests created since the start date. + 2) Integer for the number of close requests since the start date. + 3) Integer for the difference in close requests and new requests since the start date. Typical usage example (using net_txt as example, created_txt and closed_txt are similar): @@ -247,12 +243,10 @@ def update_figure(selected_council): show up inside dcc.Graph object as long as id "graph" is passed in. Args: - selected_council: - A string argument automatically detected by Dash callback function when "council_list" element is selected in the layout. + selected_council: a string argument automatically detected by Dash callback function when "council_list" element is selected in the layout. Returns: - graph: - Plotly line chart of the total number of requests over time (createdDate) separated by request type. + Plotly line chart of the total number of requests over time (createdDate) separated by request type. Typical usage example: @@ -303,12 +297,10 @@ def update_council_figure(selected_council): show up inside dcc.Graph object as long as id "pie_graph" is passed in. Args: - selected_council: - A string argument automatically detected by Dash callback function when "council_list" element is selected in the layout. + selected_council: a string argument automatically detected by Dash callback function when "council_list" element is selected in the layout. Returns: - pie_graph - Plotly pie chart for the share of different request types. + Plotly pie chart for the share of different request types. Typical usage example: