Merge pull request #1268 from hackforla/1266-get-a-weeks-worth-of-req…

…uests-by-default-on-dev-site

1266 get a weeks worth of requests by default on dev site

client/components/Map/index.js

@@ -9,9 +9,12 @@ import { getDataRequestSuccess } from '@reducers/data';
 import { updateMapPosition } from '@reducers/ui';
 import { trackMapExport } from '@reducers/analytics';
 import CookieNotice from '../main/CookieNotice';
-// import { MAP_MODES } from '../common/CONSTANTS';
+import { DATE_SPEC } from '../common/CONSTANTS';
 // import "mapbox-gl/dist/mapbox-gl.css";
 import Map from './Map';
+import moment from 'moment';
+
+const REQUEST_BATCH_SIZE = 5000;
 
 const styles = theme => ({
   root: {
@@ -33,7 +36,7 @@ class MapContainer extends React.Component {
 
     // We store the raw requests from the API call here, but eventually they are
     // converted and stored in the Redux store.
-    this.rawRequests = null;
+    this.rawRequests = [];
     this.isSubscribed = null;
   }
 
@@ -51,19 +54,35 @@ class MapContainer extends React.Component {
   componentWillUnmount() {
     this.isSubscribed = false;
   }
-
+  /**
+   * Gets all requests over the time range specified in the Redux store.
+   * 
+   * Since the server is slow to retrieve all the requests at once, we need to
+   * make multiple API calls, using `skip` and `limit` to retrieve consecutive
+   * chunks of data.
+   */
   getAllRequests = async () => {
-    // TODO: add date specification. See https://dev-api.311-data.org/docs#/default/get_all_service_requests_requests_get.
-    // By default, this will only get the 1000 most recent requests.
-    const url = `${process.env.API_URL}/requests`;
-    const { data } = await axios.get(url);
-    this.rawRequests = data;
+    const { startDate, endDate } = this.props;
+    const url = new URL(`${process.env.API_URL}/requests`);
+    url.searchParams.append("start_date", moment(startDate, DATE_SPEC).format('YYYY-MM-DD'));
+    url.searchParams.append("end_date", moment(endDate, DATE_SPEC).format('YYYY-MM-DD'));
+    url.searchParams.append("limit", `${REQUEST_BATCH_SIZE}`);
+    var returned_length = REQUEST_BATCH_SIZE;
+    var skip = 0;
+    while (returned_length === REQUEST_BATCH_SIZE) {
+      url.searchParams.append("skip", `${skip}`);
+      const { data } = await axios.get(url);
+      returned_length = data.length;
+      skip += returned_length;
+      this.rawRequests.push(...data);
+      url.searchParams.delete("skip");
+    }
   };
 
   setData = async () => {
     const { pins } = this.props;
 
-    if (!this.rawRequests) {
+    if (this.rawRequests.length === 0) {
       await this.getAllRequests();
     }
 
@@ -128,6 +147,8 @@ const mapStateToProps = state => ({
   lastUpdated: state.metadata.lastPulledLocal,
   activeMode: state.ui.map.activeMode,
   requestTypes: state.filters.requestTypes,
+  startDate: state.filters.startDate,
+  endDate: state.filters.endDate,
   requests: state.data.requests
 });
 

client/components/Map/layers/RequestsLayer.js

@@ -30,8 +30,15 @@ function circleColors(requestTypes) {
   ];
 }
 
+/**
+ * Gets a MapBox GL JS filter specification to filter request types.
+ * 
+ * @param {Object} selectedTypes A mapping of k:v, where k is an str request
+ * type, and v is a boolean indicating whether the request type is selected.
+ * @return {Array} A Mapbox GL JS filter specification that filters out the
+ * unselected types.
+ */
 function typeFilter(selectedTypes) {
-  // selectedTypes maps ints (in string form) to booleans, indicating whether the type is selected.
   // Get an array of int typeIds corresponding value in selectedTypes is true.
   var trueTypes = Object.keys(selectedTypes).map((type) => parseInt(type)).filter((type) => selectedTypes[type]);
   return [
@@ -41,8 +48,16 @@ function typeFilter(selectedTypes) {
   ];
 }
 
+/**
+ * Gets a MapBox GL JS filter specification to filter request statuses.
+ * 
+ * @param {Object} requestStatus A mapping of k:v, where k is a request status
+ * (either open or closed), and v is a boolean indicating whether the request
+ * status is selected.
+ * @return {Array} A Mapbox GL JS filter specification that filters out the
+ * unselected statuses.
+ */
 function statusFilter(requestStatus) {
-  // requestStatus is an object with keys "open" and "closed", and boolean values.
   if (requestStatus.open && requestStatus.closed) {
     // Hack to allow ALL requests.
     return ['==', [LITERAL, 'a'], [LITERAL, 'a']];
@@ -91,8 +106,6 @@ class RequestsLayer extends React.Component {
       this.setFilters(selectedTypes, requestStatus);
     }
     if (requests !== prev.requests && this.ready) {
-      console.log("got new requests");
-      console.log(requests);
       this.setRequests(requests);
     }
     if (colorScheme !== prev.colorScheme) {
@@ -114,6 +127,7 @@ class RequestsLayer extends React.Component {
       selectedTypes,
       colorScheme,
       requestTypes,
+      requestStatus,
     } = this.props;
 
     this.map.addLayer({
@@ -134,7 +148,7 @@ class RequestsLayer extends React.Component {
         'circle-color': circleColors(requestTypes),
         'circle-opacity': 0.8,
       },
-      filter: typeFilter(selectedTypes),
+      filter: this.getFilterSpec(selectedTypes, requestStatus),
     }, BEFORE_ID);
 
     // this.map.addLayer({
@@ -168,9 +182,24 @@ class RequestsLayer extends React.Component {
     }
   };
 
+  /**
+   * Gets a MapBox GL JS filter specification.
+   * 
+   * @param {Object} selectedTypes A mapping of k:v, where k is an int request
+   * type, and v is a boolean indicating whether the request type is selected.
+   * @param {Object} requestStatus A mapping of k:v, where k is a request status
+   * (either open or closed), and v is a boolean indicating whether the request
+   * status is selected.
+   * @return {Array} A Mapbox GL JS filter specification that filters out the
+   * unselected types and statuses.
+   */
+  getFilterSpec = (selectedTypes, requestStatus) => {
+    return ['all', typeFilter(selectedTypes), statusFilter(requestStatus)];
+  };
+
   setFilters = (selectedTypes, requestStatus) => {
     this.map.setFilter('request-circles',
-      ['all', typeFilter(selectedTypes), statusFilter(requestStatus)]);
+      this.getFilterSpec(selectedTypes, requestStatus));
     // Currently, we do not support heatmap. If we did, we'd want to update
     // its filter here as well.
   };
@@ -194,15 +223,11 @@ class RequestsLayer extends React.Component {
 
 RequestsLayer.propTypes = {
   activeLayer: PropTypes.oneOf(['points', 'heatmap']),
-  selectedTypes: PropTypes.shape({}),
-  requests: PropTypes.shape({}),
   colorScheme: PropTypes.string,
 };
 
 RequestsLayer.defaultProps = {
   activeLayer: 'points',
-  selectedTypes: {},
-  requests: {},
   colorScheme: '',
 };
 

client/components/common/CONSTANTS.js

@@ -829,9 +829,11 @@ export const MAP_DATE_RANGES = (() => {
   ];
 })();
 
+export const DATE_SPEC = 'MM/DD/YYYY';
+
 export const DATE_RANGES = (() => {
-  const endDate = moment().format('MM/DD/YYYY');
-  const priorDate = (num, timeInterval) => moment().subtract(num, timeInterval).format('MM/DD/YYYY');
+  const endDate = moment().format(DATE_SPEC);
+  const priorDate = (num, timeInterval) => moment().subtract(num, timeInterval).format(DATE_SPEC);
 
   return [
     {
@@ -873,7 +875,7 @@ export const DATE_RANGES = (() => {
     {
       id: 'YEAR_TO_DATE',
       label: 'Year to Date',
-      startDate: moment().startOf('year').format('MM/DD/YYYY'),
+      startDate: moment().startOf('year').format(DATE_SPEC),
       endDate,
     },
     {

client/redux/reducers/data.js

@@ -118,6 +118,8 @@ const initialState = {
   pins: [],
   pinsInfo: {},
   selectedNcId: null,
+  // Empty GeoJSON object.
+  requests: { type: 'FeatureCollection', features: [] },
 };
 
 export default (state = initialState, action) => {

client/redux/reducers/filters.js

@@ -1,3 +1,5 @@
+import { DATE_RANGES } from '@components/common/CONSTANTS';
+
 export const types = {
   UPDATE_START_DATE: 'UPDATE_START_DATE',
   UPDATE_END_DATE: 'UPDATE_END_DATE',
@@ -35,8 +37,8 @@ export const updateRequestStatus = status => ({
 
 const initialState = {
   // dateRange: null,
-  startDate: null,
-  endDate: null,
+  startDate: DATE_RANGES[0].startDate,
+  endDate: DATE_RANGES[0].endDate,
   councilId: null,
   requestTypes: {
     1: false,