From 4f8a900a3ce1595e131c54be976c7a97e76365b4 Mon Sep 17 00:00:00 2001 From: Adam Kendis Date: Thu, 28 Jan 2021 12:04:19 -0800 Subject: [PATCH 01/12] update request_types colors --- server/api/alembic/seeds/request_types.csv | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/server/api/alembic/seeds/request_types.csv b/server/api/alembic/seeds/request_types.csv index 5ce0dbe7a..fa6dc6f25 100644 --- a/server/api/alembic/seeds/request_types.csv +++ b/server/api/alembic/seeds/request_types.csv @@ -1,12 +1,12 @@ type_id,type_name,type_group,color,data_code,sort_order -1,Animal Remains,,#ECB800,Dead Animal Removal,1 -2,Homeless Encampment,,#FF0000,Homeless Encampment,2 -3,Graffiti,Repairs,#FFE6BB,Graffiti Removal,3 -4,Multiple Streetlights,Repairs,#AD7B56,Multiple Streetlight Issue,4 -5,Single Streetlight,Repairs,#FEFE6B,Single Streetlight Issue,5 -6,Bulky Items,Trash,#4FEFEF,Bulky Items,6 -7,Electronic Waste,Trash,#AD7B56,Electronic Waste,7 -8,Illegal Dumping,Trash,#F7ADAD,Illegal Dumping Pickup,8 -9,Metal/Appliances,Trash,#ECB800,Metal/Household Appliances,9 -10,Other,,#999999,Other,10 -11,Feedback,,#333333,Feedback,11 \ No newline at end of file +1,Animal Remains,,#267370,Dead Animal Removal,1 +2,Homeless Encampment,,#11975F,Homeless Encampment,2 +3,Graffiti,Repairs,#BF82BA,Graffiti Removal,3 +4,Multiple Streetlights,Repairs,#EDAD08,Multiple Streetlight Issue,4 +5,Single Streetlight,Repairs,#79B74E,Single Streetlight Issue,5 +6,Bulky Items,Trash,#D05F4E,Bulky Items,6 +7,Electronic Waste,Trash,#AE3D51,Electronic Waste,7 +8,Illegal Dumping,Trash,#685DB1,Illegal Dumping Pickup,8 +9,Metal/Appliances,Trash,#8B508B,Metal/Household Appliances,9 +10,Other,,#1D6996,Other,10 +11,Feedback,,#E17C05,Feedback,11 \ No newline at end of file From c2c8025b9d1a764d13dbbe7a5f6078595179c6a5 Mon Sep 17 00:00:00 2001 From: Adam Kendis Date: Thu, 28 Jan 2021 17:01:22 -0800 Subject: [PATCH 02/12] added ncGeojson action creator, reducer, saga --- client/redux/reducers/metadata.js | 12 ++++++++++++ client/redux/sagas/metadata.js | 12 +++++++++++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/client/redux/reducers/metadata.js b/client/redux/reducers/metadata.js index 24e41979b..2765fd046 100644 --- a/client/redux/reducers/metadata.js +++ b/client/redux/reducers/metadata.js @@ -5,6 +5,7 @@ export const types = { GET_REQUEST_TYPES_SUCCESS: 'GET_REQUEST_TYPES_SUCCESS', GET_COUNCILS_SUCCESS: 'GET_COUNCILS_SUCCESS', GET_REGIONS_SUCCESS: 'GET_REGIONS_SUCCESS', + GET_NC_GEOJSON_SUCCESS: 'GET_NC_GEOJSON_SUCCESS', }; export const getMetadataRequest = () => ({ @@ -36,6 +37,11 @@ export const getRegionsSuccess = response => ({ payload: response, }); +export const getNcGeojsonSuccess = response => ({ + type: types.GET_NC_GEOJSON_SUCCESS, + payload: response, +}); + const initialState = { currentTimeUTC: null, currentTimeLocal: null, @@ -46,6 +52,7 @@ const initialState = { requestTypes: null, councils: null, regions: null, + ncGeojson: null, }; export default (state = initialState, action) => { @@ -70,6 +77,11 @@ export default (state = initialState, action) => { ...state, regions: action.payload, }; + case types.GET_NC_GEOJSON_SUCCESS: + return { + ...state, + ncGeojson: action.payload, + }; case types.GET_METADATA_FAILURE: { const { response: { status }, diff --git a/client/redux/sagas/metadata.js b/client/redux/sagas/metadata.js index cad1b4d80..0cd358ee9 100644 --- a/client/redux/sagas/metadata.js +++ b/client/redux/sagas/metadata.js @@ -12,28 +12,38 @@ import { getRequestTypesSuccess, getCouncilsSuccess, getRegionsSuccess, + getNcGeojsonSuccess, getMetadataFailure, } from '../reducers/metadata'; function* getMetadata() { const baseUrl = process.env.API_URL; try { - const [metadata, requestTypes, councils, regions] = yield all([ + const [ + metadata, + requestTypes, + councils, + regions, + ncGeojson, + ] = yield all([ call(axios.get, `${baseUrl}/status/api`), call(axios.get, `${baseUrl}/types`), call(axios.get, `${baseUrl}/councils`), call(axios.get, `${baseUrl}/regions`), + call(axios.get, `${baseUrl}/geojson`), ]); const { data: statusMetadata } = metadata; const { data: typesMetadata } = requestTypes; const { data: councilsMetadata } = councils; const { data: regionsMetadata } = regions; + const { data: ncGeojsonMetadata} = ncGeojson; yield all([ put(getMetadataSuccess(statusMetadata)), put(getRequestTypesSuccess(typesMetadata)), put(getCouncilsSuccess(councilsMetadata)), put(getRegionsSuccess(regionsMetadata)), + put(getNcGeojsonSuccess(ncGeojsonMetadata)), ]); } catch (e) { yield put(getMetadataFailure(e)); From 349c52592381d134d53dd96f57efe88914588339 Mon Sep 17 00:00:00 2001 From: Adam Kendis Date: Thu, 28 Jan 2021 17:05:56 -0800 Subject: [PATCH 03/12] updated Map to consume fetched council boundaries geojson --- client/components/Map/Map.jsx | 79 +++++++++++++++++++--------------- client/components/Map/index.js | 10 +++-- 2 files changed, 50 insertions(+), 39 deletions(-) diff --git a/client/components/Map/Map.jsx b/client/components/Map/Map.jsx index 30b21bc30..d033f7fae 100644 --- a/client/components/Map/Map.jsx +++ b/client/components/Map/Map.jsx @@ -2,6 +2,7 @@ import React from 'react'; // import PropTypes from 'proptypes'; +import { connect } from 'react-redux'; import { withStyles } from '@material-ui/core/styles'; import mapboxgl from 'mapbox-gl'; @@ -15,7 +16,8 @@ import { } from './constants'; import { - ncBoundaries, + // fetching ncBoundaries from api + // ncBoundaries, ccBoundaries, ncNameFromId, ccNameFromId, @@ -62,7 +64,7 @@ class Map extends React.Component { activeRequestsLayer: 'points', selectedTypes: props.selectedTypes, locationInfo: INITIAL_LOCATION, - geoFilterType: GEO_FILTER_TYPES.address, + geoFilterType: GEO_FILTER_TYPES.nc, filterGeo: null, filteredRequestCounts: {}, hoveredRegionName: null, @@ -127,8 +129,39 @@ class Map extends React.Component { if ( this.state.filterGeo !== prevState.filterGeo || this.state.selectedTypes !== prevState.selectedTypes - ) - this.map.once('idle', this.setFilteredRequestCounts); + ) this.map.once('idle', this.setFilteredRequestCounts); + + if (this.props.ncBoundaries != prevProps.ncBoundaries) { + this.ncLayer.init({ + map: this.map, + addListeners: true, + sourceId: 'nc', + sourceData: this.props.ncBoundaries, + idProperty: 'nc_id', + onSelectRegion: geo => { + this.setState({ + locationInfo: { + nc: { + name: ncNameFromId(geo.properties.nc_id), + // url removed from /geojson payload + // need to map from /councils response + // url: geo.properties.waddress || geo.properties.dwebsite, + }, + }, + }); + this.map.once('zoomend', () => { + this.setState({ filterGeo: geo }); + }); + }, + onHoverRegion: geo => { + this.setState({ + hoveredRegionName: geo + ? ncNameFromId(geo.properties.nc_id) + : null + }); + } + }); + } } initLayers = addListeners => { @@ -155,34 +188,6 @@ class Map extends React.Component { }), }); - this.ncLayer.init({ - map: this.map, - addListeners, - sourceId: 'nc', - sourceData: ncBoundaries, - idProperty: 'nc_id', - onSelectRegion: geo => { - this.setState({ - locationInfo: { - nc: { - name: ncNameFromId(geo.properties.nc_id), - url: geo.properties.waddress || geo.properties.dwebsite, - }, - }, - }); - this.map.once('zoomend', () => { - this.setState({ filterGeo: geo }); - }); - }, - onHoverRegion: geo => { - this.setState({ - hoveredRegionName: geo - ? ncNameFromId(geo.properties.nc_id) - : null - }); - } - }); - this.ccLayer.init({ map: this.map, addListeners, @@ -257,7 +262,7 @@ class Map extends React.Component { const features = this.map.queryRenderedFeatures(e.point, { layers: [ - // 'request-circles', + 'request-circles', ...masks, ...hoverables ] @@ -315,7 +320,7 @@ class Map extends React.Component { } }); - this.addressLayer.zoomTo(lngLat); + // this.addressLayer.zoomTo(lngLat); }; zoomOut = () => { @@ -503,6 +508,10 @@ Map.propTypes = {}; Map.defaultProps = {}; -export default withStyles(styles)(Map); +const mapStateToProps = state => ({ + ncBoundaries: state.metadata.ncGeojson, +}); + +export default connect(mapStateToProps, null)(withStyles(styles)(Map)); // export default Map; diff --git a/client/components/Map/index.js b/client/components/Map/index.js index f2e5f85ff..dbdad1d65 100644 --- a/client/components/Map/index.js +++ b/client/components/Map/index.js @@ -47,6 +47,8 @@ class MapContainer extends React.Component { } getOpenRequests = async () => { + // const url = `${process.env.API_URL}/requests/pins/open`; + // const { data } = await axios.get(url); const url = `${process.env.API_URL}/open-requests`; const { data } = await axios.post(url); this.openRequests = data; @@ -61,14 +63,14 @@ class MapContainer extends React.Component { return this.setState({ requests: this.convertRequests(this.openRequests.requests), - ncCounts: this.openRequests.counts.nc, - ccCounts: this.openRequests.counts.cc, + // ncCounts: this.openRequests.counts.nc, + // ccCounts: this.openRequests.counts.cc, }); case MAP_MODES.CLOSED: return this.setState({ requests: this.convertRequests(pins), - ncCounts: null, - ccCounts: null, + // ncCounts: null, + // ccCounts: null, }); } }; From 17c0c5ff6b004a25b9f724664ef833d4488a3cfe Mon Sep 17 00:00:00 2001 From: Adam Kendis Date: Thu, 28 Jan 2021 17:10:56 -0800 Subject: [PATCH 04/12] comments --- client/components/Map/Map.jsx | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/client/components/Map/Map.jsx b/client/components/Map/Map.jsx index d033f7fae..b36d34a27 100644 --- a/client/components/Map/Map.jsx +++ b/client/components/Map/Map.jsx @@ -16,8 +16,6 @@ import { } from './constants'; import { - // fetching ncBoundaries from api - // ncBoundaries, ccBoundaries, ncNameFromId, ccNameFromId, @@ -320,7 +318,7 @@ class Map extends React.Component { } }); - // this.addressLayer.zoomTo(lngLat); + this.addressLayer.zoomTo(lngLat); }; zoomOut = () => { @@ -513,5 +511,3 @@ const mapStateToProps = state => ({ }); export default connect(mapStateToProps, null)(withStyles(styles)(Map)); - -// export default Map; From 1ea63dab972b7da48ed473dcfc2add95cc014890 Mon Sep 17 00:00:00 2001 From: Adam Kendis Date: Thu, 28 Jan 2021 17:12:33 -0800 Subject: [PATCH 05/12] linting --- client/redux/sagas/metadata.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/redux/sagas/metadata.js b/client/redux/sagas/metadata.js index 0cd358ee9..e0f4cf845 100644 --- a/client/redux/sagas/metadata.js +++ b/client/redux/sagas/metadata.js @@ -36,7 +36,7 @@ function* getMetadata() { const { data: typesMetadata } = requestTypes; const { data: councilsMetadata } = councils; const { data: regionsMetadata } = regions; - const { data: ncGeojsonMetadata} = ncGeojson; + const { data: ncGeojsonMetadata } = ncGeojson; yield all([ put(getMetadataSuccess(statusMetadata)), From f3b49ba06c56ffa790df74490d2e0d8778e96e57 Mon Sep 17 00:00:00 2001 From: Adam Kendis Date: Thu, 28 Jan 2021 17:52:16 -0800 Subject: [PATCH 06/12] added husky and pre-commit linting hook --- client/package-lock.json | 226 +++++++++++++++++++++++++++++++++++++++ client/package.json | 7 ++ 2 files changed, 233 insertions(+) diff --git a/client/package-lock.json b/client/package-lock.json index d9a57c869..7c4b3bd13 100644 --- a/client/package-lock.json +++ b/client/package-lock.json @@ -3332,6 +3332,12 @@ "resolved": "https://registry.npmjs.org/@types/normalize-package-data/-/normalize-package-data-2.4.0.tgz", "integrity": "sha512-f5j5b/Gf71L+dbqxIpQ4Z2WlmI/mPJ0fOkGGmFgtb6sAu97EPczzbS3/tJKxmcYDj55OX6ssqwDAWOHIYDRDGA==" }, + "@types/parse-json": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@types/parse-json/-/parse-json-4.0.0.tgz", + "integrity": "sha512-//oorEZjL6sbPcKUaCdIGlIUeH26mgzimjBB77G6XRgnDl/L5wOnpyBGRe/Mmf5CVW3PwEBE1NjiMZ/ssFh4wA==", + "dev": true + }, "@types/prop-types": { "version": "15.7.3", "resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.3.tgz", @@ -5244,6 +5250,12 @@ "integrity": "sha1-3dgA2gxmEnOTzKWVDqloo6rxJTs=", "dev": true }, + "compare-versions": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/compare-versions/-/compare-versions-3.6.0.tgz", + "integrity": "sha512-W6Af2Iw1z4CB7q4uU4hv646dW9GQuBM+YpC0UvUCWSD8w90SJjp+ujJuXaEMtAXBtSqGfMPuFOVn4/+FlaqfBA==", + "dev": true + }, "component-emitter": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/component-emitter/-/component-emitter-1.3.0.tgz", @@ -5462,6 +5474,39 @@ "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" }, + "cosmiconfig": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/cosmiconfig/-/cosmiconfig-7.0.0.tgz", + "integrity": "sha512-pondGvTuVYDk++upghXJabWzL6Kxu6f26ljFw64Swq9v6sQPUL3EUlVDV56diOjpCayKihL6hVe8exIACU4XcA==", + "dev": true, + "requires": { + "@types/parse-json": "^4.0.0", + "import-fresh": "^3.2.1", + "parse-json": "^5.0.0", + "path-type": "^4.0.0", + "yaml": "^1.10.0" + }, + "dependencies": { + "parse-json": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-5.2.0.tgz", + "integrity": "sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==", + "dev": true, + "requires": { + "@babel/code-frame": "^7.0.0", + "error-ex": "^1.3.1", + "json-parse-even-better-errors": "^2.3.0", + "lines-and-columns": "^1.1.6" + } + }, + "path-type": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz", + "integrity": "sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==", + "dev": true + } + } + }, "create-ecdh": { "version": "4.0.3", "resolved": "https://registry.npmjs.org/create-ecdh/-/create-ecdh-4.0.3.tgz", @@ -7728,6 +7773,15 @@ "locate-path": "^3.0.0" } }, + "find-versions": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/find-versions/-/find-versions-4.0.0.tgz", + "integrity": "sha512-wgpWy002tA+wgmO27buH/9KzyEOQnKsG/R0yrcjPT9BOFm0zRBVQbZ95nRGXWMywS8YR5knRbpohio0bcJABxQ==", + "dev": true, + "requires": { + "semver-regex": "^3.1.2" + } + }, "findup-sync": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/findup-sync/-/findup-sync-3.0.0.tgz", @@ -8656,6 +8710,133 @@ "integrity": "sha1-7AbBDgo0wPL68Zn3/X/Hj//QPHM=", "dev": true }, + "husky": { + "version": "4.3.8", + "resolved": "https://registry.npmjs.org/husky/-/husky-4.3.8.tgz", + "integrity": "sha512-LCqqsB0PzJQ/AlCgfrfzRe3e3+NvmefAdKQhRYpxS4u6clblBoDdzzvHi8fmxKRzvMxPY/1WZWzomPZww0Anow==", + "dev": true, + "requires": { + "chalk": "^4.0.0", + "ci-info": "^2.0.0", + "compare-versions": "^3.6.0", + "cosmiconfig": "^7.0.0", + "find-versions": "^4.0.0", + "opencollective-postinstall": "^2.0.2", + "pkg-dir": "^5.0.0", + "please-upgrade-node": "^3.2.0", + "slash": "^3.0.0", + "which-pm-runs": "^1.0.0" + }, + "dependencies": { + "ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "requires": { + "color-convert": "^2.0.1" + } + }, + "chalk": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.0.tgz", + "integrity": "sha512-qwx12AxXe2Q5xQ43Ac//I6v5aXTipYrSESdOgzrN+9XjgEpyjpKuvSGaN4qE93f7TQTlerQQ8S+EQ0EyDoVL1A==", + "dev": true, + "requires": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + } + }, + "color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "requires": { + "color-name": "~1.1.4" + } + }, + "color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true + }, + "find-up": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", + "integrity": "sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==", + "dev": true, + "requires": { + "locate-path": "^6.0.0", + "path-exists": "^4.0.0" + } + }, + "has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true + }, + "locate-path": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", + "integrity": "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==", + "dev": true, + "requires": { + "p-locate": "^5.0.0" + } + }, + "p-limit": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "dev": true, + "requires": { + "yocto-queue": "^0.1.0" + } + }, + "p-locate": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-5.0.0.tgz", + "integrity": "sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==", + "dev": true, + "requires": { + "p-limit": "^3.0.2" + } + }, + "path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", + "dev": true + }, + "pkg-dir": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-5.0.0.tgz", + "integrity": "sha512-NPE8TDbzl/3YQYY7CSS228s3g2ollTFnc+Qi3tqmqJp9Vg2ovUpixcJEo2HJScN2Ez+kEaal6y70c0ehqJBJeA==", + "dev": true, + "requires": { + "find-up": "^5.0.0" + } + }, + "slash": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz", + "integrity": "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==", + "dev": true + }, + "supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "requires": { + "has-flag": "^4.0.0" + } + } + } + }, "hyphen": { "version": "1.6.2", "resolved": "https://registry.npmjs.org/hyphen/-/hyphen-1.6.2.tgz", @@ -11293,6 +11474,12 @@ "mimic-fn": "^2.1.0" } }, + "opencollective-postinstall": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/opencollective-postinstall/-/opencollective-postinstall-2.0.3.tgz", + "integrity": "sha512-8AV/sCtuzUeTo8gQK5qDZzARrulB3egtLzFgteqB2tcT4Mw7B8Kt7JcDHmltjz6FOAHsvTevk70gZEbhM4ZS9Q==", + "dev": true + }, "opn": { "version": "5.5.0", "resolved": "https://registry.npmjs.org/opn/-/opn-5.5.0.tgz", @@ -11664,6 +11851,15 @@ "find-up": "^3.0.0" } }, + "please-upgrade-node": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/please-upgrade-node/-/please-upgrade-node-3.2.0.tgz", + "integrity": "sha512-gQR3WpIgNIKwBMVLkpMUeR3e1/E1y42bqDQZfql+kDeXd8COYfM8PQA4X6y7a8u9Ua9FHmsrrmirW2vHs45hWg==", + "dev": true, + "requires": { + "semver-compare": "^1.0.0" + } + }, "pn": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/pn/-/pn-1.1.0.tgz", @@ -12994,6 +13190,18 @@ "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==" }, + "semver-compare": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/semver-compare/-/semver-compare-1.0.0.tgz", + "integrity": "sha1-De4hahyUGrN+nvsXiPavxf9VN/w=", + "dev": true + }, + "semver-regex": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/semver-regex/-/semver-regex-3.1.2.tgz", + "integrity": "sha512-bXWyL6EAKOJa81XG1OZ/Yyuq+oT0b2YLlxx7c+mrdYPaPbnj6WgVULXhinMIeZGufuUBu/eVRqXEhiv4imfwxA==", + "dev": true + }, "send": { "version": "0.17.1", "resolved": "https://registry.npmjs.org/send/-/send-0.17.1.tgz", @@ -15523,6 +15731,12 @@ "resolved": "https://registry.npmjs.org/which-module/-/which-module-2.0.0.tgz", "integrity": "sha1-2e8H3Od7mQK4o6j6SzHD4/fm6Ho=" }, + "which-pm-runs": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/which-pm-runs/-/which-pm-runs-1.0.0.tgz", + "integrity": "sha1-Zws6+8VS4LVd9rd4DKdGFfI60cs=", + "dev": true + }, "wide-align": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/wide-align/-/wide-align-1.1.3.tgz", @@ -15656,6 +15870,12 @@ "integrity": "sha1-HBH5IY8HYImkfdUS+TxmmaaoHVI=", "dev": true }, + "yaml": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-1.10.0.tgz", + "integrity": "sha512-yr2icI4glYaNG+KWONODapy2/jDdMSDnrONSjblABjD9B4Z5LgiircSt8m8sRZFNi08kG9Sm0uSHtEmP3zaEGg==", + "dev": true + }, "yargs": { "version": "13.3.2", "resolved": "https://registry.npmjs.org/yargs/-/yargs-13.3.2.tgz", @@ -15682,6 +15902,12 @@ "decamelize": "^1.2.0" } }, + "yocto-queue": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", + "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", + "dev": true + }, "yoga-layout-prebuilt": { "version": "1.9.6", "resolved": "https://registry.npmjs.org/yoga-layout-prebuilt/-/yoga-layout-prebuilt-1.9.6.tgz", diff --git a/client/package.json b/client/package.json index c1062f213..0a626597b 100644 --- a/client/package.json +++ b/client/package.json @@ -51,11 +51,17 @@ "dev": "webpack-dev-server --config webpack.dev.js", "build": "webpack --config webpack.prod.js", "lint": "eslint './**/*.js*'", + "lint:fix": "eslint --fix './**/*.js*'", "test": "jest --passWithNoTests", "predeploy": "npm run build", "check-env": "node ./utils/checkEnv", "postinstall": "sh -c \"are-you-es5 check -r . | tail -n 2 | head -n 1 > ./node_modules_non_es5 \"" }, + "husky": { + "hooks": { + "pre-commit": "npm run lint" + } + }, "devDependencies": { "@babel/core": "^7.8.4", "@babel/plugin-proposal-class-properties": "^7.7.0", @@ -77,6 +83,7 @@ "eslint-plugin-react-hooks": "^1.7.0", "extract-text-webpack-plugin": "^4.0.0-beta.0", "file-loader": "^4.2.0", + "husky": "^4.3.8", "mini-css-extract-plugin": "^0.9.0", "node-sass": "^4.12.0", "react-svg-loader": "^3.0.3", From 23f7ef44f936a1e9cc01647178e34f2f49412cd9 Mon Sep 17 00:00:00 2001 From: Matt Webster Date: Sun, 31 Jan 2021 22:28:28 -0800 Subject: [PATCH 07/12] removing pysupercluster and routes, tests --- server/api/Pipfile | 4 +- server/api/Pipfile.lock | 197 +++++------- .../code/lacity_data_api/models/clusters.py | 296 ------------------ .../api/code/lacity_data_api/routers/shim.py | 16 +- server/api/tests/integration/test_api_shim.py | 38 --- 5 files changed, 86 insertions(+), 465 deletions(-) delete mode 100644 server/api/code/lacity_data_api/models/clusters.py diff --git a/server/api/Pipfile b/server/api/Pipfile index c596ab139..c6a7def49 100644 --- a/server/api/Pipfile +++ b/server/api/Pipfile @@ -14,21 +14,21 @@ pytest = "*" numpy = "*" pandas = "*" psycopg2-binary = "*" -pysupercluster = "*" redis = "*" requests-async = "*" sendgrid = "*" SQLAlchemy = "*" gino = {extras = ["starlette"], version = "*"} fastapi = "*" +fastapi-utils = "*" alembic = "*" geopandas = "*" +geoalchemy2 = "*" ujson = "*" sentry-sdk = "*" aiocache = {extras = ["redis"], version = "*"} gunicorn = "*" uvicorn = {extras = ["standard"], version = "*"} -geoalchemy2 = "*" [requires] python_version = "3.7" diff --git a/server/api/Pipfile.lock b/server/api/Pipfile.lock index a4ddf3587..326eadbb6 100644 --- a/server/api/Pipfile.lock +++ b/server/api/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "6b3e6dcfaac67dd84aca5371577f441e5d8a8e40f620bc703078d7b464b59ea6" + "sha256": "109488af8f5163751bc4a2dd2b25f50b65c77e31c950215d1c6c4f008bfab23e" }, "pipfile-spec": 6, "requires": { @@ -36,11 +36,10 @@ }, "alembic": { "hashes": [ - "sha256:a4de8d3525a95a96d59342e14b95cab5956c25b0907dce1549bb4e3e7958f4c2", - "sha256:c057488cc8ac7c4d06025ea3907e1a4dd07af70376fa149cf6bd2bc11b43076f" + "sha256:04608b6904a6e6bd1af83e1a48f73f50ba214aeddef44b92d498df33818654a8" ], "index": "pypi", - "version": "==1.5.2" + "version": "==1.5.3" }, "async-timeout": { "hashes": [ @@ -126,21 +125,6 @@ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' and python_version < '4'", "version": "==0.7.1" }, - "contextvars": { - "hashes": [ - "sha256:f38c908aaa59c14335eeea12abea5f443646216c4e29380d7bf34d2018e2c39e" - ], - "markers": "python_version < '3.7'", - "version": "==2.4" - }, - "dataclasses": { - "hashes": [ - "sha256:0201d89fa866f68c8ebd9d08ee6ff50c0b255f8ec63a71c16fda7af82bb887bf", - "sha256:8479067f342acf957dc82ec415d355ab5edb7e7646b90dc6e2fd1d96ad084c97" - ], - "markers": "python_version < '3.7'", - "version": "==0.8" - }, "fastapi": { "hashes": [ "sha256:63c4592f5ef3edf30afa9a44fa7c6b7ccb20e0d3f68cd9eba07b44d552058dcb", @@ -149,6 +133,14 @@ "index": "pypi", "version": "==0.63.0" }, + "fastapi-utils": { + "hashes": [ + "sha256:0e6c7fc1870b80e681494957abf65d4f4f42f4c7f70005918e9181b22f1bd759", + "sha256:dd0be7dc7f03fa681b25487a206651d99f2330d5a567fb8ab6cb5f8a06a29360" + ], + "index": "pypi", + "version": "==0.2.1" + }, "fiona": { "hashes": [ "sha256:1c26f38bfbcd51e710c361f26db605ccf2b5f2a7967e0f4a88683cac3845c947", @@ -319,24 +311,6 @@ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.10" }, - "immutables": { - "hashes": [ - "sha256:1c11050c49e193a1ec9dda1747285333f6ba6a30bbeb2929000b9b1192097ec0", - "sha256:33ce2f977da7b5e0dddd93744862404bdb316ffe5853ec853e53141508fa2e6a", - "sha256:6c8eace4d98988c72bcb37c05e79aae756832738305ae9497670482a82db08bc", - "sha256:714aedbdeba4439d91cb5e5735cb10631fc47a7a69ea9cc8ecbac90322d50a4a", - "sha256:860666fab142401a5535bf65cbd607b46bc5ed25b9d1eb053ca8ed9a1a1a80d6", - "sha256:8797eed4042f4626b0bc04d9cf134208918eb0c937a8193a2c66df5041e62d2e", - "sha256:a0a1cc238b678455145bae291d8426f732f5255537ed6a5b7645949704c70a78", - "sha256:ab6c18b7b2b2abc83e0edc57b0a38bf0915b271582a1eb8c7bed1c20398f8040", - "sha256:c099212fd6504513a50e7369fe281007c820cf9d7bb22a336486c63d77d6f0b2", - "sha256:c453e12b95e1d6bb4909e8743f88b7f5c0c97b86a8bc0d73507091cb644e3c1e", - "sha256:ce01788878827c3f0331c254a4ad8d9721489a5e65cc43e19c80040b46e0d297", - "sha256:ef9da20ec0f1c5853b5c8f8e3d9e1e15b8d98c259de4b7515d789a606af8745e" - ], - "markers": "python_version >= '3.5'", - "version": "==0.14" - }, "importlib-metadata": { "hashes": [ "sha256:90bb658cdbbf6d1735b6341ce708fc7024a3e14e99ffdc5783edea9f9b077f83", @@ -359,8 +333,12 @@ "sha256:09c4b7f37d6c648cb13f9230d847adf22f8171b1ccc4d5682398e77f40309235", "sha256:1027c282dad077d0bae18be6794e6b6b8c91d58ed8a8d89a89d59693b9131db5", "sha256:13d3144e1e340870b25e7b10b98d779608c02016d5184cfb9927a9f10c689f42", + "sha256:195d7d2c4fbb0ee8139a6cf67194f3973a6b3042d742ebe0a9ed36d8b6f0c07f", + "sha256:22c178a091fc6630d0d045bdb5992d2dfe14e3259760e713c490da5323866c39", "sha256:24982cc2533820871eba85ba648cd53d8623687ff11cbb805be4ff7b4c971aff", "sha256:29872e92839765e546828bb7754a68c418d927cd064fd4708fab9fe9c8bb116b", + "sha256:2beec1e0de6924ea551859edb9e7679da6e4870d32cb766240ce17e0a0ba2014", + "sha256:3b8a6499709d29c2e2399569d96719a1b21dcd94410a586a18526b143ec8470f", "sha256:43a55c2930bbc139570ac2452adf3d70cdbb3cfe5912c71cdce1c2c6bbd9c5d1", "sha256:46c99d2de99945ec5cb54f23c8cd5689f6d7177305ebff350a58ce5f8de1669e", "sha256:500d4957e52ddc3351cabf489e79c91c17f6e0899158447047588650b5e69183", @@ -369,24 +347,39 @@ "sha256:62fe6c95e3ec8a7fad637b7f3d372c15ec1caa01ab47926cfdf7a75b40e0eac1", "sha256:6788b695d50a51edb699cb55e35487e430fa21f1ed838122d722e0ff0ac5ba15", "sha256:6dd73240d2af64df90aa7c4e7481e23825ea70af4b4922f8ede5b9e35f78a3b1", + "sha256:6f1e273a344928347c1290119b493a1f0303c52f5a5eae5f16d74f48c15d4a85", + "sha256:6fffc775d90dcc9aed1b89219549b329a9250d918fd0b8fa8d93d154918422e1", "sha256:717ba8fe3ae9cc0006d7c451f0bb265ee07739daf76355d06366154ee68d221e", "sha256:79855e1c5b8da654cf486b830bd42c06e8780cea587384cf6545b7d9ac013a0b", "sha256:7c1699dfe0cf8ff607dbdcc1e9b9af1755371f92a68f706051cc8c37d447c905", + "sha256:7fed13866cf14bba33e7176717346713881f56d9d2bcebab207f7a036f41b850", + "sha256:84dee80c15f1b560d55bcfe6d47b27d070b4681c699c572af2e3c7cc90a3b8e0", "sha256:88e5fcfb52ee7b911e8bb6d6aa2fd21fbecc674eadd44118a9cc3863f938e735", "sha256:8defac2f2ccd6805ebf65f5eeb132adcf2ab57aa11fdf4c0dd5169a004710e7d", + "sha256:98bae9582248d6cf62321dcb52aaf5d9adf0bad3b40582925ef7c7f0ed85fceb", "sha256:98c7086708b163d425c67c7a91bad6e466bb99d797aa64f965e9d25c12111a5e", "sha256:9add70b36c5666a2ed02b43b335fe19002ee5235efd4b8a89bfcf9005bebac0d", "sha256:9bf40443012702a1d2070043cb6291650a0841ece432556f784f004937f0f32c", + "sha256:a6a744282b7718a2a62d2ed9d993cad6f5f585605ad352c11de459f4108df0a1", + "sha256:acf08ac40292838b3cbbb06cfe9b2cb9ec78fce8baca31ddb87aaac2e2dc3bc2", "sha256:ade5e387d2ad0d7ebf59146cc00c8044acbd863725f887353a10df825fc8ae21", "sha256:b00c1de48212e4cc9603895652c5c410df699856a2853135b3967591e4beebc2", "sha256:b1282f8c00509d99fef04d8ba936b156d419be841854fe901d8ae224c59f0be5", + "sha256:b1dba4527182c95a0db8b6060cc98ac49b9e2f5e64320e2b56e47cb2831978c7", "sha256:b2051432115498d3562c084a49bba65d97cf251f5a331c64a12ee7e04dacc51b", + "sha256:b7d644ddb4dbd407d31ffb699f1d140bc35478da613b441c582aeb7c43838dd8", "sha256:ba59edeaa2fc6114428f1637ffff42da1e311e29382d81b339c1817d37ec93c6", + "sha256:bf5aa3cbcfdf57fa2ee9cd1822c862ef23037f5c832ad09cfea57fa846dec193", "sha256:c8716a48d94b06bb3b2524c2b77e055fb313aeb4ea620c8dd03a105574ba704f", + "sha256:caabedc8323f1e93231b52fc32bdcde6db817623d33e100708d9a68e1f53b26b", "sha256:cd5df75523866410809ca100dc9681e301e3c27567cf498077e8551b6d20e42f", "sha256:cdb132fc825c38e1aeec2c8aa9338310d29d337bebbd7baa06889d09a60a1fa2", + "sha256:d53bc011414228441014aa71dbec320c66468c1030aae3a6e29778a3382d96e5", + "sha256:d73a845f227b0bfe8a7455ee623525ee656a9e2e749e4742706d80a6065d5e2c", + "sha256:d9be0ba6c527163cbed5e0857c451fcd092ce83947944d6c14bc95441203f032", "sha256:e249096428b3ae81b08327a63a485ad0878de3fb939049038579ac0ef61e17e7", - "sha256:e8313f01ba26fbbe36c7be1966a7b7424942f670f38e666995b88d012765b9be" + "sha256:e8313f01ba26fbbe36c7be1966a7b7424942f670f38e666995b88d012765b9be", + "sha256:feb7b34d6325451ef96bc0e36e1a6c0c1c64bc1fbec4b854f4529e51887b1621" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.1.1" @@ -400,73 +393,56 @@ }, "numpy": { "hashes": [ - "sha256:012426a41bc9ab63bb158635aecccc7610e3eff5d31d1eb43bc099debc979d94", - "sha256:06fab248a088e439402141ea04f0fffb203723148f6ee791e9c75b3e9e82f080", - "sha256:0eef32ca3132a48e43f6a0f5a82cb508f22ce5a3d6f67a8329c81c8e226d3f6e", - "sha256:1ded4fce9cfaaf24e7a0ab51b7a87be9038ea1ace7f34b841fe3b6894c721d1c", - "sha256:2e55195bc1c6b705bfd8ad6f288b38b11b1af32f3c8289d6c50d47f950c12e76", - "sha256:2ea52bd92ab9f768cc64a4c3ef8f4b2580a17af0a5436f6126b08efbd1838371", - "sha256:36674959eed6957e61f11c912f71e78857a8d0604171dfd9ce9ad5cbf41c511c", - "sha256:384ec0463d1c2671170901994aeb6dce126de0a95ccc3976c43b0038a37329c2", - "sha256:39b70c19ec771805081578cc936bbe95336798b7edf4732ed102e7a43ec5c07a", - "sha256:400580cbd3cff6ffa6293df2278c75aef2d58d8d93d3c5614cd67981dae68ceb", - "sha256:43d4c81d5ffdff6bae58d66a3cd7f54a7acd9a0e7b18d97abb255defc09e3140", - "sha256:50a4a0ad0111cc1b71fa32dedd05fa239f7fb5a43a40663269bb5dc7877cfd28", - "sha256:603aa0706be710eea8884af807b1b3bc9fb2e49b9f4da439e76000f3b3c6ff0f", - "sha256:6149a185cece5ee78d1d196938b2a8f9d09f5a5ebfbba66969302a778d5ddd1d", - "sha256:759e4095edc3c1b3ac031f34d9459fa781777a93ccc633a472a5468587a190ff", - "sha256:7fb43004bce0ca31d8f13a6eb5e943fa73371381e53f7074ed21a4cb786c32f8", - "sha256:811daee36a58dc79cf3d8bdd4a490e4277d0e4b7d103a001a4e73ddb48e7e6aa", - "sha256:8b5e972b43c8fc27d56550b4120fe6257fdc15f9301914380b27f74856299fea", - "sha256:99abf4f353c3d1a0c7a5f27699482c987cf663b1eac20db59b8c7b061eabd7fc", - "sha256:a0d53e51a6cb6f0d9082decb7a4cb6dfb33055308c4c44f53103c073f649af73", - "sha256:a12ff4c8ddfee61f90a1633a4c4afd3f7bcb32b11c52026c92a12e1325922d0d", - "sha256:a4646724fba402aa7504cd48b4b50e783296b5e10a524c7a6da62e4a8ac9698d", - "sha256:a76f502430dd98d7546e1ea2250a7360c065a5fdea52b2dffe8ae7180909b6f4", - "sha256:a9d17f2be3b427fbb2bce61e596cf555d6f8a56c222bd2ca148baeeb5e5c783c", - "sha256:ab83f24d5c52d60dbc8cd0528759532736b56db58adaa7b5f1f76ad551416a1e", - "sha256:aeb9ed923be74e659984e321f609b9ba54a48354bfd168d21a2b072ed1e833ea", - "sha256:c843b3f50d1ab7361ca4f0b3639bf691569493a56808a0b0c54a051d260b7dbd", - "sha256:cae865b1cae1ec2663d8ea56ef6ff185bad091a5e33ebbadd98de2cfa3fa668f", - "sha256:cc6bd4fd593cb261332568485e20a0712883cf631f6f5e8e86a52caa8b2b50ff", - "sha256:cf2402002d3d9f91c8b01e66fbb436a4ed01c6498fffed0e4c7566da1d40ee1e", - "sha256:d051ec1c64b85ecc69531e1137bb9751c6830772ee5c1c426dbcfe98ef5788d7", - "sha256:d6631f2e867676b13026e2846180e2c13c1e11289d67da08d71cacb2cd93d4aa", - "sha256:dbd18bcf4889b720ba13a27ec2f2aac1981bd41203b3a3b27ba7a33f88ae4827", - "sha256:df609c82f18c5b9f6cb97271f03315ff0dbe481a2a02e56aeb1b1a985ce38e60" + "sha256:0d28a54afcf46f1f9ebd163e49ad6b49087f22986fefd01a23ca0c1cdda25ca6", + "sha256:1264c66129f5ef63187649dd43f1ca59532e8c098723643336a85131c0dcce3f", + "sha256:1abc02e30e3efd81a4571e00f8e62bf42e343c76698e0a3e11d9c2b3ee0d77a7", + "sha256:2445a96fbae23a4109c61be0f0af0f3bc273905dc5687a710850c1dfde0fc994", + "sha256:2bf0e68c92ef077fe766e53f8937d8ac341bdbca68ec128ae049b7d5c34e3206", + "sha256:33edfc0eb229f86f539493917b34035054313a11afbed48404aaf9f86bf4b0f6", + "sha256:3d8233c03f116d068d5365fed4477f2947c7229582dad81e5953088989294cec", + "sha256:4d592264d2a4f368afbb4288b5ceb646d4cbaf559c0249c096fbb0a149806b90", + "sha256:5ae765dd29c71a555f8102281f6fb15a3f4dbd35f6e7daf36af9df6d9dd716a5", + "sha256:894aaee60043a98b03f0ad992c810f62e3a15f98a701e1c0f58a4f4a0df13429", + "sha256:89bd70c9ad540febe6c28451ba225eb4e49d27f64728357f512c808002325dfa", + "sha256:93c2abea7bb69f47029b84ceac30ab46dfcfdb99b671ad850a333ff794a765e4", + "sha256:abdfa075e293d73638ece434708aa60b510dc6e70d805f57f481a0f550b25a9e", + "sha256:afeee581b50df20ef07b736e62ca612858f1fcdba96651d26ab44e3d567a4e6e", + "sha256:b51b9ef0624f4b01b846c981034c10d2e30db33f9f8be71e992f3900741f6f77", + "sha256:b66a6c15d793eda7cdad986e737775aa31b9306d588c14dd0277d2dda5546150", + "sha256:cb257bb0c0a3176c32782a63cfab2eace7eabfa2a3b2dfd85a13700617ccaf28", + "sha256:cf5d9dcbdbe523fa665c5309cce5f144648d94a7fddbf5a40f8e0d5c9f5b596d", + "sha256:d1bc331e1706fd1809a1bc8a31205329e5b30cf5ba50461c624da267e99f6ae6", + "sha256:db5e69d08756a2fa75a42b4e433880b6187768fe1bc73d21819def893e5128c6", + "sha256:e3db646af9f6a145f0c57202f4b55d4a33f975e395e78fb7b394644c17c1a3a6", + "sha256:e9c5fd330d2fedf06051bafb996252de9b032fcb2ec03eefc9a543e56efa66d4", + "sha256:eee454d3aa3955d0c0069a0f265fea47f1e1384c35a110a95efed358eb6e1562", + "sha256:f1e9424e9aa3834ea27cc12f9c6ea8ace5da18ee60a720bb3a85b2f733f41782" ], "index": "pypi", - "version": "==1.19.5" + "version": "==1.20.0" }, "pandas": { "hashes": [ - "sha256:0a643bae4283a37732ddfcecab3f62dd082996021b980f580903f4e8e01b3c5b", - "sha256:0de3ddb414d30798cbf56e642d82cac30a80223ad6fe484d66c0ce01a84d6f2f", - "sha256:19a2148a1d02791352e9fa637899a78e371a3516ac6da5c4edc718f60cbae648", - "sha256:21b5a2b033380adbdd36b3116faaf9a4663e375325831dac1b519a44f9e439bb", - "sha256:24c7f8d4aee71bfa6401faeba367dd654f696a77151a8a28bc2013f7ced4af98", - "sha256:26fa92d3ac743a149a31b21d6f4337b0594b6302ea5575b37af9ca9611e8981a", - "sha256:2860a97cbb25444ffc0088b457da0a79dc79f9c601238a3e0644312fcc14bf11", - "sha256:2b1c6cd28a0dfda75c7b5957363333f01d370936e4c6276b7b8e696dd500582a", - "sha256:2c2f7c670ea4e60318e4b7e474d56447cf0c7d83b3c2a5405a0dbb2600b9c48e", - "sha256:3be7a7a0ca71a2640e81d9276f526bca63505850add10206d0da2e8a0a325dae", - "sha256:4c62e94d5d49db116bef1bd5c2486723a292d79409fc9abd51adf9e05329101d", - "sha256:5008374ebb990dad9ed48b0f5d0038124c73748f5384cc8c46904dace27082d9", - "sha256:5447ea7af4005b0daf695a316a423b96374c9c73ffbd4533209c5ddc369e644b", - "sha256:573fba5b05bf2c69271a32e52399c8de599e4a15ab7cec47d3b9c904125ab788", - "sha256:5a780260afc88268a9d3ac3511d8f494fdcf637eece62fb9eb656a63d53eb7ca", - "sha256:70865f96bb38fec46f7ebd66d4b5cfd0aa6b842073f298d621385ae3898d28b5", - "sha256:731568be71fba1e13cae212c362f3d2ca8932e83cb1b85e3f1b4dd77d019254a", - "sha256:b61080750d19a0122469ab59b087380721d6b72a4e7d962e4d7e63e0c4504814", - "sha256:bf23a3b54d128b50f4f9d4675b3c1857a688cc6731a32f931837d72effb2698d", - "sha256:c16d59c15d946111d2716856dd5479221c9e4f2f5c7bc2d617f39d870031e086", - "sha256:c61c043aafb69329d0f961b19faa30b1dab709dd34c9388143fc55680059e55a", - "sha256:c94ff2780a1fd89f190390130d6d36173ca59fcfb3fe0ff596f9a56518191ccb", - "sha256:edda9bacc3843dfbeebaf7a701763e68e741b08fccb889c003b0a52f0ee95782", - "sha256:f10fc41ee3c75a474d3bdf68d396f10782d013d7f67db99c0efbfd0acb99701b" + "sha256:050ed2c9d825ef36738e018454e6d055c63d947c1d52010fbadd7584f09df5db", + "sha256:055647e7f4c5e66ba92c2a7dcae6c2c57898b605a3fb007745df61cc4015937f", + "sha256:23ac77a3a222d9304cb2a7934bb7b4805ff43d513add7a42d1a22dc7df14edd2", + "sha256:2de012a36cc507debd9c3351b4d757f828d5a784a5fc4e6766eafc2b56e4b0f5", + "sha256:30e9e8bc8c5c17c03d943e8d6f778313efff59e413b8dbdd8214c2ed9aa165f6", + "sha256:324e60bea729cf3b55c1bf9e88fe8b9932c26f8669d13b928e3c96b3a1453dff", + "sha256:37443199f451f8badfe0add666e43cdb817c59fa36bceedafd9c543a42f236ca", + "sha256:47ec0808a8357ab3890ce0eca39a63f79dcf941e2e7f494470fe1c9ec43f6091", + "sha256:496fcc29321e9a804d56d5aa5d7ec1320edfd1898eee2f451aa70171cf1d5a29", + "sha256:50e6c0a17ef7f831b5565fd0394dbf9bfd5d615ee4dd4bb60a3d8c9d2e872323", + "sha256:5527c5475d955c0bc9689c56865aaa2a7b13c504d6c44f0aadbf57b565af5ebd", + "sha256:57d5c7ac62925a8d2ab43ea442b297a56cc8452015e71e24f4aa7e4ed6be3d77", + "sha256:9d45f58b03af1fea4b48e44aa38a819a33dccb9821ef9e1d68f529995f8a632f", + "sha256:b26e2dabda73d347c7af3e6fed58483161c7b87a886a4e06d76ccfe55a044aa9", + "sha256:cfd237865d878da9b65cfee883da5e0067f5e2ff839e459466fb90565a77bda3", + "sha256:d7cca42dba13bfee369e2944ae31f6549a55831cba3117e17636955176004088", + "sha256:fe7de6fed43e7d086e3d947651ec89e55ddf00102f9dd5758763d56d182f0564" ], "index": "pypi", - "version": "==1.1.5" + "version": "==1.2.1" }, "psycopg2-binary": { "hashes": [ @@ -569,13 +545,6 @@ "markers": "python_version >= '3.6'", "version": "==3.0.0.post1" }, - "pysupercluster": { - "hashes": [ - "sha256:1e9739e9dad6126f346b5816dd6d9affc31d8482ef89fd30ff87f8412530726a" - ], - "index": "pypi", - "version": "==0.7.6" - }, "python-dateutil": { "hashes": [ "sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c", @@ -818,11 +787,11 @@ }, "urllib3": { "hashes": [ - "sha256:19188f96923873c92ccb987120ec4acaa12f0461fa9ce5d3d0772bc965a39e08", - "sha256:d8ff90d979214d7b4f8ce956e80f4028fc6860e4431f731ea4a8c08f23f99473" + "sha256:1b465e494e3e0d8939b50680403e3aedaa2bc434b7d5af64dfd3c958d7f5ae80", + "sha256:de3eedaad74a2683334e282005cd8d7f22f4d55fa690a2a1020a416cb0a47e73" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'", - "version": "==1.26.2" + "version": "==1.26.3" }, "uvicorn": { "extras": [ @@ -1010,11 +979,11 @@ }, "packaging": { "hashes": [ - "sha256:24e0da08660a87484d1602c30bb4902d74816b6985b93de36926f5bc95741858", - "sha256:78598185a7008a470d64526a8059de9aaa449238f280fc9eb6b13ba6c4109093" + "sha256:5b327ac1320dc863dca72f4514ecc086f31186744b84a230374cc1fd776feae5", + "sha256:67714da7f7bc052e064859c05c595155bd1ee9f69f76557e21f051443c20947a" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==20.8" + "version": "==20.9" }, "pluggy": { "hashes": [ @@ -1107,11 +1076,11 @@ }, "urllib3": { "hashes": [ - "sha256:19188f96923873c92ccb987120ec4acaa12f0461fa9ce5d3d0772bc965a39e08", - "sha256:d8ff90d979214d7b4f8ce956e80f4028fc6860e4431f731ea4a8c08f23f99473" + "sha256:1b465e494e3e0d8939b50680403e3aedaa2bc434b7d5af64dfd3c958d7f5ae80", + "sha256:de3eedaad74a2683334e282005cd8d7f22f4d55fa690a2a1020a416cb0a47e73" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'", - "version": "==1.26.2" + "version": "==1.26.3" }, "zipp": { "hashes": [ diff --git a/server/api/code/lacity_data_api/models/clusters.py b/server/api/code/lacity_data_api/models/clusters.py deleted file mode 100644 index fa95c40de..000000000 --- a/server/api/code/lacity_data_api/models/clusters.py +++ /dev/null @@ -1,296 +0,0 @@ -from typing import List -import datetime - -import numpy -import pysupercluster -from sqlalchemy import and_ - -from .service_request import ServiceRequest -from .region import Region -from .council import get_councils_dict -from .request_type import get_types_dict -from . import db, cache -from ..config import DEBUG, CACHE_MAX_RETRIES -from ..services import utilities - - -DEFAULT_CITY_ZOOM = 11 # a click on a city point zooms from 10 to 12 -DEFAULT_REGION_ZOOM = 12 # a click on a city point zooms from 10 to 12 -DEFAULT_COUNCIL_ZOOM = 13 # a click on a council point zooms to 14 -DEFAULT_LATITUDE = 34.0522 -DEFAULT_LONGITUDE = -118.2437 - - -class Cluster: - def __init__(self, - count: int, - expansion_zoom: int, - id: int, - latitude: float, - longitude: float): - self.count = count - self.expansion_zoom = expansion_zoom - self.id = id - self.latitude = latitude - self.longitude = longitude - - -async def get_clusters_for_regions( - start_date: datetime.date, - end_date: datetime.date, - type_ids: List[int], - council_ids: List[int], - zoom_current: int -) -> List[Cluster]: - """ - Cluster service request pins by council regions - - Args: - start_date (date): beginning of date range service was requested - end_date (date): end of date range service was requested - type_ids (List[int]): the request type ids to match on - council_ids (List[int]): the council ids to match on - - Returns: - cluster: a list of cluster objects - """ - - # TODO: CACHE 'region-reqs:start-end-types-councils' - result = await ( - db.select( - [ - ServiceRequest.region_id, - db.func.count() - ] - ).where( - and_( - ServiceRequest.created_date >= start_date, - ServiceRequest.created_date <= end_date, - ServiceRequest.type_id.in_(type_ids), - ServiceRequest.council_id.in_(council_ids), - ) - ).group_by( - ServiceRequest.region_id - ).gino.all() - ) - - cluster_list = [] - - for row in result: - region = await Region.get(row[0]) - cluster_list.append(Cluster( - count=row[1], - expansion_zoom=DEFAULT_REGION_ZOOM, - id=region.region_id, - latitude=region.latitude, - longitude=region.longitude - )) - - return cluster_list - - -async def get_clusters_for_councils( - start_date: datetime.date, - end_date: datetime.date, - type_ids: List[int], - council_ids: List[int], - zoom_current: int -) -> List[Cluster]: - """ - Cluster service request pins by council - - Args: - start_date (date): beginning of date range service was requested - end_date (date): end of date range service was requested - type_ids (List[int]): the request type ids to match on - council_ids (List[int]): the council ids to match on - - Returns: - cluster: a list of cluster objects - """ - - # TODO: CACHE 'council-reqs:start-end-types-councils' - result = await ( - db.select( - [ - ServiceRequest.council_id, - db.func.count() - ] - ).where( - and_( - ServiceRequest.created_date >= start_date, - ServiceRequest.created_date <= end_date, - ServiceRequest.type_id.in_(type_ids), - ServiceRequest.council_id.in_(council_ids), - ) - ).group_by( - ServiceRequest.council_id - ).gino.all() - ) - - # zoom_next = (zoom_current + 1) or DEFAULT_COUNCIL_ZOOM - cluster_list = [] - - # TODO: replace this with a caching solution - # returns dictionary with council id as key and name, lat, long - # council_result = await db.all(Council.query) - # councils = [ - # (i.council_id, [i.council_name, i.latitude, i.longitude]) - # for i in council_result - # ] - councils_dict = await get_councils_dict() - - for row in result: - council = councils_dict.get(row[0]) - cluster_list.append(Cluster( - count=row[1], - expansion_zoom=DEFAULT_COUNCIL_ZOOM, - id=row[0], - latitude=council[1], - longitude=council[2] - )) - - return cluster_list - - -async def get_points( - start_date: datetime.date, - end_date: datetime.date, - type_ids: List[int], - council_ids: List[int] -) -> List[List[int]]: - """ - Get filtered geospacial points for service requests for the entire city - - Args: - start_date (date): beginning of date range service was requested - end_date (date): end of date range service was requested - type_ids (List[int]): the request type ids to match - council_ids: (List[int]): the council ids to match - - Returns: - a list of latitude and logitude pairs of service request locations - """ - - result = await ( - db.select( - [ - ServiceRequest.latitude, - ServiceRequest.longitude - ] - ).where( - and_( - ServiceRequest.created_date >= start_date, - ServiceRequest.created_date <= end_date, - ServiceRequest.type_id.in_(type_ids), - ServiceRequest.council_id.in_(council_ids) - ) - ).gino.all() - ) - - return [[row.latitude, row.longitude] for row in result] - - -async def get_clusters_for_bounds( - start_date: datetime.date, - end_date: datetime.date, - type_ids: List[int], - council_ids: List[int], - zoom_current: int, - bounds -) -> List[Cluster]: - """ - Cluster pins for the entire city - - Args: - start_date (date): beginning of date range service was requested - end_date (date): end of date range service was requested - type_ids (List[int]): the request type ids to match on - - Returns: - a JSON object either representing a cluster or a pin for a request - """ - - cache_key = utilities.cache_key( - "pins", - { - "start_date": start_date, - "end_date": end_date, - "type_ids": type_ids, - "council_ids": council_ids - } - ) - - # Try accessing the result from cache the configured number of times - for _ in range(CACHE_MAX_RETRIES): - try: - result = await cache.get(cache_key) - except Exception: - # Output when timeout occurs - print("Redis get cache timeout error") - continue - else: - break - else: - print(f"Redis cache is unavailable after {CACHE_MAX_RETRIES} tries") - - if result is None: - # query database and set cache entry - result = await ( - db.select( - [ - ServiceRequest.request_id, - ServiceRequest.type_id, - ServiceRequest.latitude, - ServiceRequest.longitude - ] - ).where( - and_( - ServiceRequest.created_date >= start_date, - ServiceRequest.created_date <= end_date, - ServiceRequest.type_id.in_(type_ids), - ServiceRequest.council_id.in_(council_ids) - ) - ).gino.all() - ) - # Try setting the cache with results. OK if fails. - try: - await cache.set(cache_key, result) - except Exception: - print("Redis set cache timeout error") - else: - if DEBUG: - print(f"Cache hit for key ({cache_key})") - - return await cluster_points(result, bounds, zoom_current) - - -async def cluster_points(result, bounds, zoom_current): - # get points to cluster - points = [[row.longitude, row.latitude] for row in result] - - index = pysupercluster.SuperCluster( - numpy.array(points), - min_zoom=0, - max_zoom=17, - radius=200, - extent=512 - ) - - cluster_list = index.getClusters( - top_left=(bounds.west, bounds.north), - bottom_right=(bounds.east, bounds.south), - zoom=zoom_current - ) - - types_dict = await get_types_dict() - - for item in cluster_list: - # change single item clusters into points - if item['count'] == 1: - pin = result[item['id']] # cluster id matches the result row - item['srnumber'] = "1-" + str(pin.request_id) - item['requesttype'] = types_dict[pin.type_id] - del item['expansion_zoom'] - - return cluster_list diff --git a/server/api/code/lacity_data_api/routers/shim.py b/server/api/code/lacity_data_api/routers/shim.py index 093143f67..2093459f5 100644 --- a/server/api/code/lacity_data_api/routers/shim.py +++ b/server/api/code/lacity_data_api/routers/shim.py @@ -4,7 +4,7 @@ Filter, Feedback, Comparison ) from ..models import ( - clusters, request_type, service_request, council + request_type, service_request, council ) from ..services import ( email, github, reports @@ -65,20 +65,6 @@ async def get_open_requests(): } -@router.post("/map/heat") -async def get_heatmap(filter: Filter): - # convert type names to type ids - type_ids = await request_type.get_type_ids_by_str_list(filter.requestTypes) - - result = await clusters.get_points( - filter.startDate, - filter.endDate, - type_ids, - filter.ncList - ) - return result - - @router.post("/map/pins") async def get_pins(filter: Filter): # convert type names to type ids diff --git a/server/api/tests/integration/test_api_shim.py b/server/api/tests/integration/test_api_shim.py index 85fc58d9a..d9427de0a 100644 --- a/server/api/tests/integration/test_api_shim.py +++ b/server/api/tests/integration/test_api_shim.py @@ -9,44 +9,6 @@ """ -def test_map_heat(client): - # post old style filters (i.e. text request types) - url = "/map/heat" - response = client.post( - url, - json={ - "startDate": "01/01/2020", - "endDate": "01/02/2020", - "ncList": [ - 29, - 30, - 32, - 33, - 34, - 46, - 52, - 54, - 55, - 58, - 60, - 76, - 97, - 104, - 119, - 121, - 128 - ], - "requestTypes": [ - "Dead Animal Removal", - "Homeless Encampment", - "Graffiti Removal" - ] - } - ) - assert response.status_code == 200 - assert len(response.json()) == 621 - - def test_map_pins(client): # post old style filters (i.e. text request types) url = "/map/pins" From 5ca6939fa559b0bdc12540fe44bdae0b873df2bb Mon Sep 17 00:00:00 2001 From: Matt Webster Date: Sun, 31 Jan 2021 22:33:39 -0800 Subject: [PATCH 08/12] updating requirements --- server/api/requirements.txt | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/server/api/requirements.txt b/server/api/requirements.txt index b00fda6ce..5ff5de08d 100644 --- a/server/api/requirements.txt +++ b/server/api/requirements.txt @@ -1,6 +1,6 @@ aiocache==0.11.1 aioredis==1.3.1 -alembic==1.5.2 +alembic==1.5.3 async-timeout==3.0.1 asyncpg==0.21.0 attrs==20.3.0 @@ -9,7 +9,7 @@ chardet==3.0.4 click==7.1.2 click-plugins==1.1.1 cligj==0.7.1 -coverage==5.3.1 +coverage==5.4 fastapi==0.63.0 fastapi-utils==0.2.1 Fiona==1.8.18 @@ -27,16 +27,15 @@ http3==0.6.7 httptools==0.1.1 hyperframe==5.2.0 idna==2.10 -immutables==0.14 importlib-metadata==1.7.0 iniconfig==1.1.1 Mako==1.1.4 MarkupSafe==1.1.1 mccabe==0.6.1 munch==2.5.0 -numpy==1.19.5 -packaging==20.8 -pandas==1.1.5 +numpy==1.20.0 +packaging==20.9 +pandas==1.2.1 pluggy==0.13.1 psycopg2-binary==2.8.6 py==1.10.0 @@ -45,10 +44,9 @@ pydantic==1.7.3 pyflakes==2.2.0 pyparsing==2.4.7 pyproj==3.0.0.post1 -pysupercluster==0.7.6 -pytest==6.2.1 +pytest==6.2.2 pytest-asyncio==0.14.0 -pytest-cov==2.10.1 +pytest-cov==2.11.1 python-dateutil==2.8.1 python-dotenv==0.15.0 python-editor==1.0.4 @@ -69,7 +67,7 @@ starlette==0.13.6 toml==0.10.2 typing-extensions==3.7.4.3 ujson==4.0.2 -urllib3==1.26.2 +urllib3==1.26.3 uvicorn==0.13.3 uvloop==0.14.0 watchgod==0.6 From f1bebda7e41c808cb62c6ba60846a7316996b548 Mon Sep 17 00:00:00 2001 From: Matt Webster Date: Sun, 31 Jan 2021 22:39:29 -0800 Subject: [PATCH 09/12] setting debug to true --- server/.env.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/.env.example b/server/.env.example index 1e60ac58e..058f3e247 100644 --- a/server/.env.example +++ b/server/.env.example @@ -8,7 +8,7 @@ API_HOST_PORT=5000 DB_HOST_PORT=5432 # api config -DEBUG=False +DEBUG=True API_RESTART_POLICY=no APP_PROTOCOL=http APP_HOST=api From aa809a1dbd1f5869c0ab3d70b20b81ec84dd51e6 Mon Sep 17 00:00:00 2001 From: Matt Webster Date: Sun, 31 Jan 2021 22:45:46 -0800 Subject: [PATCH 10/12] lint is last step --- .github/workflows/build_test_backend.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_test_backend.yml b/.github/workflows/build_test_backend.yml index 34a6c2442..a8b9853f8 100644 --- a/.github/workflows/build_test_backend.yml +++ b/.github/workflows/build_test_backend.yml @@ -13,13 +13,13 @@ jobs: run: | cp .env.example .env docker-compose up --no-start api - - name: Lint API - run: docker-compose run api flake8 - name: Seed database and run API run: | docker-compose run -e TESTING=True api alembic upgrade head docker-compose up -d api - name: Run unit tests run: docker-compose run api pytest + - name: Lint API + run: docker-compose run api flake8 # - name: Run Postman tests # run: chmod +x postman/test.sh && postman/test.sh From 236d8bf6bf63a7a59977f7981456bde1249d479d Mon Sep 17 00:00:00 2001 From: Matt Webster Date: Sun, 31 Jan 2021 22:53:09 -0800 Subject: [PATCH 11/12] debug alembic --- .github/workflows/build_test_backend.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_test_backend.yml b/.github/workflows/build_test_backend.yml index a8b9853f8..4864e2444 100644 --- a/.github/workflows/build_test_backend.yml +++ b/.github/workflows/build_test_backend.yml @@ -15,7 +15,7 @@ jobs: docker-compose up --no-start api - name: Seed database and run API run: | - docker-compose run -e TESTING=True api alembic upgrade head + docker-compose run -e TESTING=True DEBUG=True api alembic upgrade head docker-compose up -d api - name: Run unit tests run: docker-compose run api pytest From c62e81651ab6e0f35fd24e48f48b1a378a3a5b0c Mon Sep 17 00:00:00 2001 From: Matt Webster Date: Sun, 31 Jan 2021 22:59:30 -0800 Subject: [PATCH 12/12] fix run command --- .github/workflows/build_test_backend.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_test_backend.yml b/.github/workflows/build_test_backend.yml index 4864e2444..05fbe9218 100644 --- a/.github/workflows/build_test_backend.yml +++ b/.github/workflows/build_test_backend.yml @@ -15,7 +15,7 @@ jobs: docker-compose up --no-start api - name: Seed database and run API run: | - docker-compose run -e TESTING=True DEBUG=True api alembic upgrade head + docker-compose run -e TESTING=True -e DEBUG=True api alembic upgrade head docker-compose up -d api - name: Run unit tests run: docker-compose run api pytest