From edfbfaffddab8de67b57241745a7cf221f1ece2b Mon Sep 17 00:00:00 2001 From: Nicholas Kwon Date: Thu, 7 Jul 2022 18:25:48 -0700 Subject: [PATCH 1/3] beginning to work on date range logic --- client/components/Map/index.js | 93 ++++++++++++++++++++++++++++++---- client/redux/reducers/data.js | 12 +++++ 2 files changed, 95 insertions(+), 10 deletions(-) diff --git a/client/components/Map/index.js b/client/components/Map/index.js index 3ce7bef63..bc59bc3c3 100644 --- a/client/components/Map/index.js +++ b/client/components/Map/index.js @@ -5,7 +5,7 @@ import PropTypes from 'proptypes'; import { connect } from 'react-redux'; import { withStyles } from '@material-ui/core/styles'; import axios from 'axios'; -import { getDataRequestSuccess } from '@reducers/data'; +import { getDataRequestSuccess, updateDateRanges } from '@reducers/data'; import { updateMapPosition } from '@reducers/ui'; import { trackMapExport } from '@reducers/analytics'; import { INTERNAL_DATE_SPEC } from '../common/CONSTANTS'; @@ -48,15 +48,81 @@ class MapContainer extends React.Component { } componentDidUpdate(prevProps) { - if (prevProps.activeMode !== this.props.activeMode || - prevProps.pins !== this.props.pins) + const { activeMode, pins, startDate, endDate } = this.props; + if (prevProps.activeMode !== activeMode || prevProps.pins !== pins || + prevProps.startDate != startDate || prevProps.endDate != endDate) { this.setData(); + } } componentWillUnmount() { this.isSubscribed = false; } + getNonOverlappingRanges = (startA, endA, startB, endB) => { + var leftOverlap = null; + var rightOverlap = null; + if (startA < startB){ + leftOverlap = [startA, startB]; + } + if (endB < endA){ + rightOverlap = [endB, endA]; + } + return [leftOverlap, rightOverlap]; + } + + getMissingDateRanges = (startDate, endDate) => { + const {dateRangesWithRequests} = this.props; + var missingDateRanges = []; + var currentStartDate = startDate; + var currentEndDate = endDate; + for (let dateRange of dateRangesWithRequests.values()){ + const nonOverlappingRanges = this.getNonOverlappingRanges(currentStartDate, + currentEndDate, dateRange[0], dateRange[1]); + if (nonOverlappingRanges[0] !== null){ + missingDateRanges.push(nonOverlappingRanges[0]); + } + if (nonOverlappingRanges[1] === null){ + return missingDateRanges; + } + currentStartDate = nonOverlappingRanges[1][0]; + currentEndDate = nonOverlappingRanges[1][1]; + } + missingDateRanges.push([currentStartDate, currentEndDate]); + return missingDateRanges; + } + + resolveDateRanges = (startDate, endDate) => { + const {dateRangesWithRequests} = this.props; + if (dateRangesWithRequests.length === 0){ + return [[startDate, endDate]]; + } + var newDateRanges = []; + var currentStartDate = startDate; + var currentEndDate = endDate; + for (let dateRange of dateRangesWithRequests.values()){ + const nonOverlappingRanges = this.getNonOverlappingRanges(currentStartDate, + currentEndDate, dateRange[0], dateRange[1]); + const leftOverlap = nonOverlappingRanges[0]; + const rightOverlap = nonOverlappingRanges[1]; + if (leftOverlap === null && rightOverlap === null){ + newDateRanges.push([dateRange]); + } + if (leftOverlap !== null){ + currentStartDate = leftOverlap[0]; + } + if (rightOverlap === null){ + currentEndDate = dateRange[1]; + newDateRanges.push([currentStartDate, currentEndDate]); + } + } + // Only sometimes need to add this... + newDateRanges.push([currentStartDate, currentEndDate]); + // Sort newDateRanges by startDate. + // Merge adjacent date ranges. + return newDateRanges; + } + /** * Gets all the dates within a given date range. * @param {string} startDate A date in INTERNAL_DATE_SPEC format. @@ -81,8 +147,7 @@ class MapContainer extends React.Component { * Since the server is slow to retrieve all the requests at once, we need to * make multiple API calls, one for each day. */ - getAllRequests = async () => { - const { startDate, endDate } = this.props; + getAllRequests = async (startDate, endDate) => { const datesInRange = this.getDatesInRange(startDate, endDate); var requests = []; for (let i in datesInRange){ @@ -98,15 +163,21 @@ class MapContainer extends React.Component { }; setData = async () => { - const { pins } = this.props; + const { startDate, endDate } = this.props; - if (this.rawRequests.length === 0) { - await this.getAllRequests(); + const missingDateRanges = this.getMissingDateRanges(startDate, endDate); + if (missingDateRanges.length !== 0){ + this.rawRequests = []; + for (let i in missingDateRanges){ + await this.getAllRequests(missingDateRanges[i][0], missingDateRanges[i][1]); + } } if (this.isSubscribed) { - const { getDataSuccess } = this.props; + const { getDataSuccess, updateDateRangesWithRequests } = this.props; getDataSuccess(this.convertRequests(this.rawRequests)); + const newDateRangesWithRequests = this.resolveDateRanges(startDate, endDate); + updateDateRangesWithRequests(newDateRangesWithRequests); } }; @@ -170,13 +241,15 @@ const mapStateToProps = state => ({ requestTypes: state.filters.requestTypes, startDate: state.filters.startDate, endDate: state.filters.endDate, - requests: state.data.requests + requests: state.data.requests, + dateRangesWithRequests: state.data.dateRangesWithRequests, }); const mapDispatchToProps = dispatch => ({ updatePosition: position => dispatch(updateMapPosition(position)), exportMap: () => dispatch(trackMapExport()), getDataSuccess: data => dispatch(getDataRequestSuccess(data)), + updateDateRangesWithRequests: dateRanges => dispatch(updateDateRanges(dateRanges)), }); MapContainer.propTypes = {}; diff --git a/client/redux/reducers/data.js b/client/redux/reducers/data.js index 0460dd744..79347f2a5 100644 --- a/client/redux/reducers/data.js +++ b/client/redux/reducers/data.js @@ -1,6 +1,7 @@ export const types = { GET_DATA_REQUEST: 'GET_DATA_REQUEST', GET_DATA_REQUEST_SUCCESS: 'GET_DATA_REQUEST_SUCCESS', + UPDATE_DATE_RANGES: 'UPDATE_DATE_RANGES', GET_PINS_SUCCESS: 'GET_PINS_SUCCESS', GET_PINS_FAILURE: 'GET_PINS_FAILURE', GET_OPEN_REQUESTS: 'GET_OPEN_REQUESTS', @@ -31,6 +32,11 @@ export const getDataRequestSuccess = response => ({ payload: response, }); +export const updateDateRanges = dateRanges => ({ + type: types.UPDATE_DATE_RANGES, + payload: dateRanges, +}); + export const getPinsSuccess = response => ({ type: types.GET_PINS_SUCCESS, payload: response, @@ -120,6 +126,7 @@ const initialState = { selectedNcId: null, // Empty GeoJSON object. requests: { type: 'FeatureCollection', features: [] }, + dateRangesWithRequests: [], }; export default (state = initialState, action) => { @@ -135,6 +142,11 @@ export default (state = initialState, action) => { ...state, requests: action.payload, }; + case types.UPDATE_DATE_RANGES: + return { + ...state, + dateRangesWithRequests: action.payload, + }; case types.GET_PINS_SUCCESS: return { ...state, From eef86257f829b2538b765f66f89e2c919c8d75bc Mon Sep 17 00:00:00 2001 From: Nicholas Kwon Date: Mon, 11 Jul 2022 14:41:17 -0700 Subject: [PATCH 2/3] resolve date ranges --- client/components/Map/index.js | 128 +++++++++++++++++++++------------ client/redux/reducers/data.js | 9 ++- 2 files changed, 89 insertions(+), 48 deletions(-) diff --git a/client/components/Map/index.js b/client/components/Map/index.js index bc59bc3c3..2cee4b009 100644 --- a/client/components/Map/index.js +++ b/client/components/Map/index.js @@ -59,18 +59,42 @@ class MapContainer extends React.Component { this.isSubscribed = false; } + /** + * Returns the non-overlapping date ranges of A before and after B. + * @param {string} startA The start date of range A in INTERNAL_DATE_SPEC format. + * @param {string} endA The end date of range A in INTERNAL_DATE_SPEC format. + * @param {string} startB The start date of range B in INTERNAL_DATE_SPEC format. + * @param {string} endB The end date of range B in INTERNAL_DATE_SPEC format. + * @returns An array of two elements: the first element is the non-overlapping + * range of A before B; the second is the non-overlapping range of A after B. + * Each element can be null if there is no non-overlappping range. + */ getNonOverlappingRanges = (startA, endA, startB, endB) => { var leftOverlap = null; var rightOverlap = null; - if (startA < startB){ - leftOverlap = [startA, startB]; + if (moment(startA) < moment(startB)){ + leftOverlap = [startA, moment(startB).subtract(1, 'days').format(INTERNAL_DATE_SPEC)]; } - if (endB < endA){ - rightOverlap = [endB, endA]; + if (moment(endB) < moment(endA)){ + rightOverlap = [moment(endB).add(1, 'days').format(INTERNAL_DATE_SPEC), endA]; } return [leftOverlap, rightOverlap]; } + /** + * Returns the missing date ranges of a new date range against the existing + * date ranges in the Redux store. + * + * In our Redux store, we keep track of date ranges that we already have 311 + * requests for. When the user changes the date range, we need to check + * whether we need to retrieve more data; if we do, we only want to pull the + * data from the date ranges that aren't already in the store. + * + * @param {*} startDate The start date in INTERNAL_DATE_SPEC format. + * @param {*} endDate The end date in INTERNAL_DATE_SPEC format. + * @returns An array of date ranges, where each date range is represented as + * an array of string start and end dates. + */ getMissingDateRanges = (startDate, endDate) => { const {dateRangesWithRequests} = this.props; var missingDateRanges = []; @@ -92,35 +116,44 @@ class MapContainer extends React.Component { return missingDateRanges; } - resolveDateRanges = (startDate, endDate) => { + /** + * Returns the updated date ranges given the date ranges that we just pulled + * data for. + * @param {Array} newDateRanges The new date ranges that we just pulled data for. + * @returns The updated, complete array of date ranges for which we have data + * in the Redux store. + */ + resolveDateRanges = (newDateRanges) => { const {dateRangesWithRequests} = this.props; - if (dateRangesWithRequests.length === 0){ - return [[startDate, endDate]]; - } - var newDateRanges = []; - var currentStartDate = startDate; - var currentEndDate = endDate; - for (let dateRange of dateRangesWithRequests.values()){ - const nonOverlappingRanges = this.getNonOverlappingRanges(currentStartDate, - currentEndDate, dateRange[0], dateRange[1]); - const leftOverlap = nonOverlappingRanges[0]; - const rightOverlap = nonOverlappingRanges[1]; - if (leftOverlap === null && rightOverlap === null){ - newDateRanges.push([dateRange]); - } - if (leftOverlap !== null){ - currentStartDate = leftOverlap[0]; + var allDateRanges = dateRangesWithRequests.concat(newDateRanges); + // Sort date ranges by startDate. Since newDateRanges was retrieved using + // getMissingDateRanges, there should be no overlapping date ranges in the + // allDateRanges. + const sortedDateRanges = allDateRanges.sort(function(dateRangeA, dateRangeB){ + return moment(dateRangeA[0]) - moment(dateRangeB[0])}); + var resolvedDateRanges = []; + var currentStart = null; + var currentEnd = null; + for (const dateRange of sortedDateRanges){ + if (currentStart === null){ + currentStart = dateRange[0]; + currentEnd = dateRange[1]; + continue; } - if (rightOverlap === null){ - currentEndDate = dateRange[1]; - newDateRanges.push([currentStartDate, currentEndDate]); + // Check if the current date range is adjacent to the next date range. + if (moment(currentEnd).add(1, 'days').valueOf() === moment(dateRange[0]).valueOf()){ + // Extend the current date range to include the next date range. + currentEnd = dateRange[1]; + } else { + resolvedDateRanges.push([currentStart, currentEnd]); + currentStart = null; + currentEnd = null; } } - // Only sometimes need to add this... - newDateRanges.push([currentStartDate, currentEndDate]); - // Sort newDateRanges by startDate. - // Merge adjacent date ranges. - return newDateRanges; + if (currentStart !== null){ + resolvedDateRanges.push([currentStart, currentEnd]); + } + return resolvedDateRanges; } /** @@ -147,43 +180,46 @@ class MapContainer extends React.Component { * Since the server is slow to retrieve all the requests at once, we need to * make multiple API calls, one for each day. */ - getAllRequests = async (startDate, endDate) => { + getAllRequests = (startDate, endDate) => { const datesInRange = this.getDatesInRange(startDate, endDate); var requests = []; - for (let i in datesInRange){ + for (const date of datesInRange){ const url = new URL(`${process.env.API_URL}/requests`); - url.searchParams.append("start_date", datesInRange[i]); - url.searchParams.append("end_date", datesInRange[i]); + url.searchParams.append("start_date", date); + url.searchParams.append("end_date", date); url.searchParams.append("limit", `${REQUEST_LIMIT}`); requests.push(axios.get(url)); } - await Promise.all(requests).then(responses => { - responses.forEach(response => this.rawRequests.push(...response.data)) - }); + return requests; }; setData = async () => { const { startDate, endDate } = this.props; const missingDateRanges = this.getMissingDateRanges(startDate, endDate); - if (missingDateRanges.length !== 0){ - this.rawRequests = []; - for (let i in missingDateRanges){ - await this.getAllRequests(missingDateRanges[i][0], missingDateRanges[i][1]); - } + if (missingDateRanges.length === 0){ + return; + } + this.rawRequests = []; + var allRequestPromises = []; + for (const missingDateRange of missingDateRanges){ + const requestPromises = this.getAllRequests(missingDateRange[0], + missingDateRange[1]); + allRequestPromises.push(...requestPromises); } + await Promise.all(allRequestPromises).then(responses => { + responses.forEach(response => this.rawRequests.push(...response.data)) + }); if (this.isSubscribed) { const { getDataSuccess, updateDateRangesWithRequests } = this.props; getDataSuccess(this.convertRequests(this.rawRequests)); - const newDateRangesWithRequests = this.resolveDateRanges(startDate, endDate); + const newDateRangesWithRequests = this.resolveDateRanges(missingDateRanges); updateDateRangesWithRequests(newDateRangesWithRequests); } }; - convertRequests = requests => ({ - type: 'FeatureCollection', - features: requests.map(request => ({ + convertRequests = requests => (requests.map(request => ({ type: 'Feature', properties: { requestId: request.requestId, @@ -201,7 +237,7 @@ class MapContainer extends React.Component { ] } })) - }); + ); // TODO: fix this getSelectedTypes = () => { diff --git a/client/redux/reducers/data.js b/client/redux/reducers/data.js index 79347f2a5..09543f39c 100644 --- a/client/redux/reducers/data.js +++ b/client/redux/reducers/data.js @@ -137,11 +137,16 @@ export default (state = initialState, action) => { isMapLoading: true, isVisLoading: true, }; - case types.GET_DATA_REQUEST_SUCCESS: + case types.GET_DATA_REQUEST_SUCCESS: { + const newRequests = { + type: 'FeatureCollection', + features: [...state.requests.features, ...action.payload], + }; return { ...state, - requests: action.payload, + requests: newRequests, }; + } case types.UPDATE_DATE_RANGES: return { ...state, From 373eab016eb5decef2310acee13f03d37c269af3 Mon Sep 17 00:00:00 2001 From: Nicholas Kwon Date: Thu, 14 Jul 2022 13:03:44 -0700 Subject: [PATCH 3/3] update comments for date range logic --- client/components/Map/index.js | 35 +++++++++++++++++++++++++++------- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/client/components/Map/index.js b/client/components/Map/index.js index 2cee4b009..33caf09ce 100644 --- a/client/components/Map/index.js +++ b/client/components/Map/index.js @@ -70,15 +70,21 @@ class MapContainer extends React.Component { * Each element can be null if there is no non-overlappping range. */ getNonOverlappingRanges = (startA, endA, startB, endB) => { - var leftOverlap = null; - var rightOverlap = null; + var leftNonOverlap = null; + var rightNonOverlap = null; + // If date range A starts before date range B, then it has a subrange that + // does not overlap with B. if (moment(startA) < moment(startB)){ - leftOverlap = [startA, moment(startB).subtract(1, 'days').format(INTERNAL_DATE_SPEC)]; + leftNonOverlap = [startA, + moment(startB).subtract(1, 'days').format(INTERNAL_DATE_SPEC)]; } + // If date range A ends after date range B, then it has a subrange that does + // not overlap with B. if (moment(endB) < moment(endA)){ - rightOverlap = [moment(endB).add(1, 'days').format(INTERNAL_DATE_SPEC), endA]; + rightNonOverlap = [moment(endB).add(1, 'days').format(INTERNAL_DATE_SPEC), + endA]; } - return [leftOverlap, rightOverlap]; + return [leftNonOverlap, rightNonOverlap]; } /** @@ -90,8 +96,8 @@ class MapContainer extends React.Component { * whether we need to retrieve more data; if we do, we only want to pull the * data from the date ranges that aren't already in the store. * - * @param {*} startDate The start date in INTERNAL_DATE_SPEC format. - * @param {*} endDate The end date in INTERNAL_DATE_SPEC format. + * @param {string} startDate The start date in INTERNAL_DATE_SPEC format. + * @param {string} endDate The end date in INTERNAL_DATE_SPEC format. * @returns An array of date ranges, where each date range is represented as * an array of string start and end dates. */ @@ -100,6 +106,16 @@ class MapContainer extends React.Component { var missingDateRanges = []; var currentStartDate = startDate; var currentEndDate = endDate; + // Compare the input date range with each date range with requests, which + // are ordered chronologically from first to last. Every left non-overlapping + // date range (i.e., a portion of the input date range that comes before the + // existing date range with requests) is immediately added to the list of + // missing date ranges. Otherwise, if there is overlap on the left (i.e., + // the input range is covered on the left side by the date range with + // requests), we push the start date for our input range forward to the end + // of the date range with requests. The process continues for every date + // range with requests. + // It stops when the input date range is covered on the right side. for (let dateRange of dateRangesWithRequests.values()){ const nonOverlappingRanges = this.getNonOverlappingRanges(currentStartDate, currentEndDate, dateRange[0], dateRange[1]); @@ -179,6 +195,11 @@ class MapContainer extends React.Component { * * Since the server is slow to retrieve all the requests at once, we need to * make multiple API calls, one for each day. + * + * @param {string} startDate A date in INTERNAL_DATE_SPEC format. + * @param {string} endDate A date in INTERNAL_DATE_SPEC format. + * @returns An array of Promises, each representing an API request for a + * particular day in the input date range. */ getAllRequests = (startDate, endDate) => { const datesInRange = this.getDatesInRange(startDate, endDate);