From 80647728f8561e9a7c4527031aead873d074c6be Mon Sep 17 00:00:00 2001 From: Nicholas Kwon Date: Fri, 1 Jul 2022 13:16:10 -0700 Subject: [PATCH 1/3] add param for batch size --- client/components/Map/index.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/client/components/Map/index.js b/client/components/Map/index.js index 17bddab29..e0b18ed73 100644 --- a/client/components/Map/index.js +++ b/client/components/Map/index.js @@ -13,6 +13,8 @@ import CookieNotice from '../main/CookieNotice'; // import "mapbox-gl/dist/mapbox-gl.css"; import Map from './Map'; +const REQUEST_BATCH_SIZE = 5000; + const styles = theme => ({ root: { height: `calc(100vh - ${theme.header.height} - ${theme.footer.height})`, @@ -54,8 +56,9 @@ class MapContainer extends React.Component { getAllRequests = async () => { // TODO: add date specification. See https://dev-api.311-data.org/docs#/default/get_all_service_requests_requests_get. - // By default, this will only get the 1000 most recent requests. - const url = `${process.env.API_URL}/requests`; + const url = new URL(`${process.env.API_URL}/requests`); + url.searchParams.append("limit", `${REQUEST_BATCH_SIZE}`); + console.log(url); const { data } = await axios.get(url); this.rawRequests = data; }; From 4ad35da056e19c3bc9b9c511e642c141eabf733e Mon Sep 17 00:00:00 2001 From: Nicholas Kwon Date: Sat, 2 Jul 2022 16:02:45 -0700 Subject: [PATCH 2/3] getting data for 1 week works now. missing comments --- client/components/Map/index.js | 26 ++++++++++++++----- client/components/Map/layers/RequestsLayer.js | 15 +++++------ client/components/common/CONSTANTS.js | 8 +++--- client/redux/reducers/data.js | 2 ++ client/redux/reducers/filters.js | 6 +++-- 5 files changed, 37 insertions(+), 20 deletions(-) diff --git a/client/components/Map/index.js b/client/components/Map/index.js index e0b18ed73..1f8b48864 100644 --- a/client/components/Map/index.js +++ b/client/components/Map/index.js @@ -9,9 +9,10 @@ import { getDataRequestSuccess } from '@reducers/data'; import { updateMapPosition } from '@reducers/ui'; import { trackMapExport } from '@reducers/analytics'; import CookieNotice from '../main/CookieNotice'; -// import { MAP_MODES } from '../common/CONSTANTS'; +import { DATE_SPEC } from '../common/CONSTANTS'; // import "mapbox-gl/dist/mapbox-gl.css"; import Map from './Map'; +import moment from 'moment'; const REQUEST_BATCH_SIZE = 5000; @@ -35,7 +36,7 @@ class MapContainer extends React.Component { // We store the raw requests from the API call here, but eventually they are // converted and stored in the Redux store. - this.rawRequests = null; + this.rawRequests = []; this.isSubscribed = null; } @@ -55,18 +56,27 @@ class MapContainer extends React.Component { } getAllRequests = async () => { - // TODO: add date specification. See https://dev-api.311-data.org/docs#/default/get_all_service_requests_requests_get. + const { startDate, endDate } = this.props; const url = new URL(`${process.env.API_URL}/requests`); + url.searchParams.append("start_date", moment(startDate, DATE_SPEC).format('YYYY-MM-DD')); + url.searchParams.append("end_date", moment(endDate, DATE_SPEC).format('YYYY-MM-DD')); url.searchParams.append("limit", `${REQUEST_BATCH_SIZE}`); - console.log(url); - const { data } = await axios.get(url); - this.rawRequests = data; + var returned_length = REQUEST_BATCH_SIZE; + var skip = 0; + while (returned_length === REQUEST_BATCH_SIZE) { + url.searchParams.append("skip", `${skip}`); + const { data } = await axios.get(url); + returned_length = data.length; + skip += returned_length; + this.rawRequests.push(...data); + url.searchParams.delete("skip"); + } }; setData = async () => { const { pins } = this.props; - if (!this.rawRequests) { + if (this.rawRequests.length === 0) { await this.getAllRequests(); } @@ -131,6 +141,8 @@ const mapStateToProps = state => ({ lastUpdated: state.metadata.lastPulledLocal, activeMode: state.ui.map.activeMode, requestTypes: state.filters.requestTypes, + startDate: state.filters.startDate, + endDate: state.filters.endDate, requests: state.data.requests }); diff --git a/client/components/Map/layers/RequestsLayer.js b/client/components/Map/layers/RequestsLayer.js index fb3e648e6..4a4287620 100644 --- a/client/components/Map/layers/RequestsLayer.js +++ b/client/components/Map/layers/RequestsLayer.js @@ -91,8 +91,6 @@ class RequestsLayer extends React.Component { this.setFilters(selectedTypes, requestStatus); } if (requests !== prev.requests && this.ready) { - console.log("got new requests"); - console.log(requests); this.setRequests(requests); } if (colorScheme !== prev.colorScheme) { @@ -114,6 +112,7 @@ class RequestsLayer extends React.Component { selectedTypes, colorScheme, requestTypes, + requestStatus, } = this.props; this.map.addLayer({ @@ -134,7 +133,7 @@ class RequestsLayer extends React.Component { 'circle-color': circleColors(requestTypes), 'circle-opacity': 0.8, }, - filter: typeFilter(selectedTypes), + filter: this.getFilterSpec(selectedTypes, requestStatus), }, BEFORE_ID); // this.map.addLayer({ @@ -168,9 +167,13 @@ class RequestsLayer extends React.Component { } }; + getFilterSpec = (selectedTypes, requestStatus) => { + return ['all', typeFilter(selectedTypes), statusFilter(requestStatus)]; + }; + setFilters = (selectedTypes, requestStatus) => { this.map.setFilter('request-circles', - ['all', typeFilter(selectedTypes), statusFilter(requestStatus)]); + this.getFilterSpec(selectedTypes, requestStatus)); // Currently, we do not support heatmap. If we did, we'd want to update // its filter here as well. }; @@ -194,15 +197,11 @@ class RequestsLayer extends React.Component { RequestsLayer.propTypes = { activeLayer: PropTypes.oneOf(['points', 'heatmap']), - selectedTypes: PropTypes.shape({}), - requests: PropTypes.shape({}), colorScheme: PropTypes.string, }; RequestsLayer.defaultProps = { activeLayer: 'points', - selectedTypes: {}, - requests: {}, colorScheme: '', }; diff --git a/client/components/common/CONSTANTS.js b/client/components/common/CONSTANTS.js index 319e2f20e..ea84eb9e9 100644 --- a/client/components/common/CONSTANTS.js +++ b/client/components/common/CONSTANTS.js @@ -829,9 +829,11 @@ export const MAP_DATE_RANGES = (() => { ]; })(); +export const DATE_SPEC = 'MM/DD/YYYY'; + export const DATE_RANGES = (() => { - const endDate = moment().format('MM/DD/YYYY'); - const priorDate = (num, timeInterval) => moment().subtract(num, timeInterval).format('MM/DD/YYYY'); + const endDate = moment().format(DATE_SPEC); + const priorDate = (num, timeInterval) => moment().subtract(num, timeInterval).format(DATE_SPEC); return [ { @@ -873,7 +875,7 @@ export const DATE_RANGES = (() => { { id: 'YEAR_TO_DATE', label: 'Year to Date', - startDate: moment().startOf('year').format('MM/DD/YYYY'), + startDate: moment().startOf('year').format(DATE_SPEC), endDate, }, { diff --git a/client/redux/reducers/data.js b/client/redux/reducers/data.js index bcaeaa942..0460dd744 100644 --- a/client/redux/reducers/data.js +++ b/client/redux/reducers/data.js @@ -118,6 +118,8 @@ const initialState = { pins: [], pinsInfo: {}, selectedNcId: null, + // Empty GeoJSON object. + requests: { type: 'FeatureCollection', features: [] }, }; export default (state = initialState, action) => { diff --git a/client/redux/reducers/filters.js b/client/redux/reducers/filters.js index 065dda3c4..bc9793be6 100644 --- a/client/redux/reducers/filters.js +++ b/client/redux/reducers/filters.js @@ -1,3 +1,5 @@ +import { DATE_RANGES } from '@components/common/CONSTANTS'; + export const types = { UPDATE_START_DATE: 'UPDATE_START_DATE', UPDATE_END_DATE: 'UPDATE_END_DATE', @@ -35,8 +37,8 @@ export const updateRequestStatus = status => ({ const initialState = { // dateRange: null, - startDate: null, - endDate: null, + startDate: DATE_RANGES[0].startDate, + endDate: DATE_RANGES[0].endDate, councilId: null, requestTypes: { 1: false, From a9df8a8c004d8f537dbf4dd7ec0c7aec919f13a9 Mon Sep 17 00:00:00 2001 From: Nicholas Kwon Date: Tue, 5 Jul 2022 12:28:08 -0700 Subject: [PATCH 3/3] add some comments --- client/components/Map/index.js | 8 ++++- client/components/Map/layers/RequestsLayer.js | 30 +++++++++++++++++-- 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/client/components/Map/index.js b/client/components/Map/index.js index 1f8b48864..1131937dc 100644 --- a/client/components/Map/index.js +++ b/client/components/Map/index.js @@ -54,7 +54,13 @@ class MapContainer extends React.Component { componentWillUnmount() { this.isSubscribed = false; } - + /** + * Gets all requests over the time range specified in the Redux store. + * + * Since the server is slow to retrieve all the requests at once, we need to + * make multiple API calls, using `skip` and `limit` to retrieve consecutive + * chunks of data. + */ getAllRequests = async () => { const { startDate, endDate } = this.props; const url = new URL(`${process.env.API_URL}/requests`); diff --git a/client/components/Map/layers/RequestsLayer.js b/client/components/Map/layers/RequestsLayer.js index 4a4287620..0d3e52cd1 100644 --- a/client/components/Map/layers/RequestsLayer.js +++ b/client/components/Map/layers/RequestsLayer.js @@ -30,8 +30,15 @@ function circleColors(requestTypes) { ]; } +/** + * Gets a MapBox GL JS filter specification to filter request types. + * + * @param {Object} selectedTypes A mapping of k:v, where k is an str request + * type, and v is a boolean indicating whether the request type is selected. + * @return {Array} A Mapbox GL JS filter specification that filters out the + * unselected types. + */ function typeFilter(selectedTypes) { - // selectedTypes maps ints (in string form) to booleans, indicating whether the type is selected. // Get an array of int typeIds corresponding value in selectedTypes is true. var trueTypes = Object.keys(selectedTypes).map((type) => parseInt(type)).filter((type) => selectedTypes[type]); return [ @@ -41,8 +48,16 @@ function typeFilter(selectedTypes) { ]; } +/** + * Gets a MapBox GL JS filter specification to filter request statuses. + * + * @param {Object} requestStatus A mapping of k:v, where k is a request status + * (either open or closed), and v is a boolean indicating whether the request + * status is selected. + * @return {Array} A Mapbox GL JS filter specification that filters out the + * unselected statuses. + */ function statusFilter(requestStatus) { - // requestStatus is an object with keys "open" and "closed", and boolean values. if (requestStatus.open && requestStatus.closed) { // Hack to allow ALL requests. return ['==', [LITERAL, 'a'], [LITERAL, 'a']]; @@ -167,6 +182,17 @@ class RequestsLayer extends React.Component { } }; + /** + * Gets a MapBox GL JS filter specification. + * + * @param {Object} selectedTypes A mapping of k:v, where k is an int request + * type, and v is a boolean indicating whether the request type is selected. + * @param {Object} requestStatus A mapping of k:v, where k is a request status + * (either open or closed), and v is a boolean indicating whether the request + * status is selected. + * @return {Array} A Mapbox GL JS filter specification that filters out the + * unselected types and statuses. + */ getFilterSpec = (selectedTypes, requestStatus) => { return ['all', typeFilter(selectedTypes), statusFilter(requestStatus)]; };