Merge pull request #304 from hackforla/dev

Push latest changes into master
hackforla · Feb 27, 2020 · 5dd0d5b · 5dd0d5b
2 parents 445bf8f + 40da08a
commit 5dd0d5b
Show file tree

Hide file tree

Showing 29 changed files with 384 additions and 90 deletions.
diff --git a/.github/workflows/Publish_Backend_Package.yml b/.github/workflows/Publish_Backend_Package.yml
@@ -0,0 +1,27 @@
+name: PublishBackendImage
+on:
+  push:
+    branches:
+      - master
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [3.7]
+    steps:
+    - uses: actions/checkout@v1
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v1
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Build and Publish to Registry
+      uses: elgohr/Publish-Docker-Github-Action@master
+      with:
+        name: hackforla/311-data/backend
+        username: ${{ secrets.DOCKER_USERNAME }}
+        password: ${{ secrets.DOCKER_PASSWORD }}
+        registry: docker.pkg.github.com
+        dockerfile: server/Dockerfile
+        context: server
diff --git a/.github/workflows/Publish_Frontend_Package.yml b/.github/workflows/Publish_Frontend_Package.yml
@@ -0,0 +1,26 @@
+name: PublishFrontendImage
+on:
+  push:
+    branches:
+      - master
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        node-version: [10.x]
+    steps:
+    - uses: actions/checkout@v1
+    - name: Use Node.js ${{ matrix.node-version }}
+      uses: actions/setup-node@v1
+      with:
+        node-version: ${{ matrix.node-version }}
+    - name: Build and Publish to Registry
+      uses: elgohr/Publish-Docker-Github-Action@master
+      with:
+        name: hackforla/311-data/frontend
+        username: ${{ secrets.DOCKER_USERNAME }}
+        password: ${{ secrets.DOCKER_PASSWORD }}
+        registry: docker.pkg.github.com
+        dockerfile: ./Dockerfile
diff --git a/Orchestration/docker-compose-example.yml b/Orchestration/docker-compose-example.yml
@@ -4,18 +4,18 @@ services:
   backend:
    build:
      context: ../server
-   image: 311-data-back:0.0.1
+   image: docker.pkg.github.com/hackforla/311-data/backend:0.0.1
    restart: always
    container_name: "311-backend"
    environment:
-     DB_CONNECTION_STRING: postgres://REDACTED:REDACTED@db:5432/postgres
+     DB_CONNECTION_STRING: postgresql://REDACTED:REDACTED@db:5432/postgres
    ports:
      - 5000:5000
 
   frontend:
    build:
      context: ..
-   image: 311-data-front:0.0.1
+   image: docker.pkg.github.com/hackforla/311-data/frontend:0.0.1
    restart: always
    container_name: "311-frontend"
    ports:

diff --git a/onboard.sh b/onboard.sh
@@ -75,16 +75,6 @@ else
   echo "WARN If errors persist, follow this to remove lfs \n\n\thttps://github.com/git-lfs/git-lfs/issues/3026#issuecomment-451598434"
 fi
 
-# Check for postgres tools
-if ! [ -x "$(command -v pg_config)" ]; then
-  echo "👺Error: pg_config is not installed." >&2
-  echo "👺👺👺 The backend required psycopg2 which requires a manual install of pg_config and libpq headers."
-  echo "👺👺👺 To install those, please refer to \n\n\thttp://initd.org/psycopg/docs/install.html"
-  exit 1
-fi
-echo "👍 Found pg_config installation"
-
-
 # Check for python3
 if ! [ -x "$(command -v python3)" ]; then
   echo "👺Error: python3 is not installed." >&2

diff --git a/package.json b/package.json
@@ -25,6 +25,7 @@
     "react-leaflet-choropleth": "^2.0.0",
     "react-redux": "^7.1.3",
     "react-test-renderer": "^16.12.0",
+    "react-tooltip": "^4.0.3",
     "react-vis": "^1.11.7",
     "redux": "^4.0.4",
     "redux-devtools-extension": "^2.13.8",

diff --git a/public/index.html b/public/index.html
@@ -8,6 +8,7 @@
     <link rel="stylesheet" href="https://unpkg.com/[email protected]/dist/leaflet.css"
           integrity="sha512-xwE/Az9zrjBIphAcBb3F6JVqxf46+CDLwfLMHloNu6KEQCAWi6HcDUbeOfBIptF7tcCzusKFjFw2yuvEpDL9wQ=="
           crossorigin=""/>
+    <link href="https://fonts.googleapis.com/css?family=Open+Sans:700|Roboto&display=swap" rel="stylesheet">
     <title>311 Data</title>
   </head>
   <body class="has-navbar-fixed-bottom">

diff --git a/server/Dockerfile b/server/Dockerfile
@@ -1,16 +1,12 @@
-FROM python:3.7-alpine
+FROM python:3.7-slim
 
-RUN echo "http://dl-8.alpinelinux.org/alpine/edge/community" >> /etc/apk/repositories
+RUN apt-get update && apt-get install -yq \
+    python3 python3-dev gcc \
+    gfortran musl-dev
 
-RUN apk add --no-cache --allow-untrusted --repository http://dl-3.alpinelinux.org/alpine/edge/testing hdf5 hdf5-dev
-RUN apk --no-cache --update-cache add gcc musl-dev gfortran python python-dev py-pip build-base wget freetype-dev libpng-dev postgresql-dev openblas-dev \
-  && pip install --no-cache-dir cython numpy
-RUN ln -s /usr/include/locale.h /usr/include/xlocale.h
+COPY requirements.txt .
 
-
-COPY requirements.txt /
-
-RUN pip install --no-cache-dir -r /requirements.txt
+RUN pip install --no-cache-dir -r requirements.txt
 
 COPY src/ /app
 

diff --git a/server/requirements.txt b/server/requirements.txt
@@ -1,8 +1,7 @@
 aiofiles==0.4.0
 attrs==19.3.0
-certifi==2019.9.11
+certifi==2019.11.28
 chardet==3.0.4
-Click==7.0
 entrypoints==0.3
 flake8==3.7.9
 h11==0.8.1
@@ -11,37 +10,35 @@ hpack==3.0.0
 httpcore==0.3.0
 httptools==0.0.13
 hyperframe==5.2.0
-idna==2.8
+idna==2.9
 importlib-metadata==1.4.0
-itsdangerous==1.1.0
-Jinja2==2.10.3
-MarkupSafe==1.1.1
 mccabe==0.6.1
 more-itertools==8.1.0
 multidict==4.5.2
-numpy==1.18.0
+numpy==1.18.1
 packaging==20.0
-pandas==0.25.3
+pandas==1.0.1
 pluggy==0.13.1
-psycopg2==2.8.4
+psycopg2-binary==2.8.4
 py==1.8.1
 pycodestyle==2.5.0
 pyflakes==2.1.1
 pyparsing==2.4.6
 pytest==5.3.3
 python-dateutil==2.8.1
 pytz==2019.3
-requests==2.22.0
+requests==2.23.0
 requests-async==0.5.0
 rfc3986==1.3.2
 sanic==19.9.0
-six==1.13.0
+Sanic-Cors==0.10.0.post3
+Sanic-Plugins-Framework==0.9.2
+six==1.14.0
 sodapy==2.0.0
-SQLAlchemy==1.3.11
+SQLAlchemy==1.3.13
 ujson==1.35
-urllib3==1.25.7
+urllib3==1.25.8
 uvloop==0.14.0
 wcwidth==0.1.8
 websockets==8.1
-Werkzeug==0.16.0
 zipp==1.0.0
diff --git a/server/src/app.py b/server/src/app.py
@@ -1,6 +1,7 @@
 import os
 from sanic import Sanic
 from sanic.response import json
+from sanic_cors import CORS
 from configparser import ConfigParser
 from threading import Timer
 from datetime import datetime
@@ -13,6 +14,7 @@
 from services.sqlIngest import DataHandler
 
 app = Sanic(__name__)
+CORS(app)
 
 
 def configure_app():
@@ -80,8 +82,7 @@ async def ingest(request):
     if not all(year in ALLOWED_YEARS for year in years):
         return json({"error":
                     f"'years' param values must be one of {ALLOWED_YEARS}"})
-    loader = DataHandler()
-    loader.loadConfig(configFilePath='./settings.cfg')
+    loader = DataHandler(app.config['Settings'])
     loader.populateFullDatabase(yearRange=years)
     return_data = {'response': 'ingest ok'}
     return json(return_data)
@@ -125,7 +126,8 @@ async def test_multiple_workers(request):
 
 if __name__ == '__main__':
     configure_app()
+    worker_count = max(cpu_count()//2, 1)
     app.run(host=app.config['Settings']['Server']['HOST'],
             port=int(app.config['Settings']['Server']['PORT']),
-            workers=cpu_count()//2,
+            workers=worker_count,
             debug=app.config['Settings']['Server']['DEBUG'])
diff --git a/server/src/services/dataService.py b/server/src/services/dataService.py
@@ -6,9 +6,10 @@ class DataService(object):
     def includeMeta(func):
         def inner1(*args, **kwargs):
             dataResponse = func(*args, **kwargs)
+            if 'Error' in dataResponse:
+                return dataResponse
 
             withMeta = {'lastPulled': 'NOW', 'data': dataResponse}
-            print(withMeta)
             return withMeta
 
         return inner1
@@ -23,7 +24,10 @@ def __init__(self, config=None, tableName="ingest_staging_table"):
         self.engine = db.create_engine(self.dbString)
 
     @includeMeta
-    def query(self, queryItems=[], queryfilters=[], limit=None):
+    def query(self, queryItems=None, queryfilters=[], limit=None):
+        if not queryItems or not isinstance(queryItems, list):
+            return {'Error': 'Missing query items'}
+
         items = ', '.join(queryItems)
         query = 'SELECT {} FROM {}'.format(items, self.table)
         if queryfilters:

diff --git a/server/src/services/sqlIngest.py b/server/src/services/sqlIngest.py
@@ -4,8 +4,12 @@
 import pandas as pd
 import sqlalchemy as db
 from sodapy import Socrata
-from .databaseOrm import tableFields, insertFields, readFields  # Contains db specs and field definitions
 from configparser import ConfigParser
+if __name__ == '__main__':
+    # Contains db specs and field definitions
+    from databaseOrm import tableFields, insertFields, readFields
+else:
+    from .databaseOrm import tableFields, insertFields, readFields
 
 
 class DataHandler:
@@ -14,29 +18,15 @@ def __init__(self, config=None, configFilePath=None, separator=','):
         self.config = config
         self.dbString = None if not self.config \
             else self.config['Database']['DB_CONNECTION_STRING']
+        self.token = None if config['Socrata']['TOKEN'] == 'None' \
+            else config['Socrata']['TOKEN']
         self.filePath = None
         self.configFilePath = configFilePath
         self.separator = separator
         self.fields = tableFields
         self.insertParams = insertFields
         self.readParams = readFields
-        self.dialect = None
-
-    def loadConfig(self, configFilePath):
-        '''Load and parse config data'''
-        if self.config:
-            print('Config already exists at %s. Nothing to load.' %
-                  self.configFilePath)
-            return
-        print('Loading config file %s' % configFilePath)
-        self.configFilePath = configFilePath
-        config = ConfigParser()
-        config.read(configFilePath)
-        self.config = config
-        self.dbString = config['Database']['DB_CONNECTION_STRING']
         self.dialect = self.dbString.split(':')[0]
-        self.token = None if config['Socrata']['TOKEN'] == 'None' \
-            else config['Socrata']['TOKEN']
 
     def loadData(self, fileName="2018_mini"):
         '''Load dataset into pandas object'''
@@ -242,8 +232,9 @@ def fix_nan_vals(resultDict):
 
 if __name__ == "__main__":
     '''Class DataHandler workflow from initial load to SQL population'''
-    loader = DataHandler()
-    loader.loadConfig(configFilePath='../settings.cfg')
+    config = ConfigParser()
+    config.read('../settings.cfg')
+    loader = DataHandler(config)
     loader.fetchSocrataFull()
     loader.cleanData()
     loader.ingestData('ingest_staging_table')
diff --git a/server/src/settings.example.cfg b/server/src/settings.example.cfg
@@ -4,7 +4,7 @@ HOST = 0.0.0.0
 PORT = 5000
 
 [Database]
-DB_CONNECTION_STRING = mysql://REDACTED:REDACTED@localhost:5432/public
+DB_CONNECTION_STRING = postgresql://REDACTED:REDACTED@localhost:5432/postgres
 DATA_DIRECTORY = static
 
 [Api]

diff --git a/server/test/test_db_service.py b/server/test/test_db_service.py
@@ -0,0 +1,35 @@
+from src.services.dataService import DataService
+
+TESTCONFIG = {
+    "Database": {
+        "DB_CONNECTION_STRING": "postgresql://testingString/postgresql"
+    }
+}
+
+
+def test_serviceExists():
+    # Arrange
+    # Act
+    data_worker = DataService(TESTCONFIG)
+    # Assert
+    assert isinstance(data_worker, DataService)
+
+
+def test_emptyQuery():
+    # Arrange
+    queryItems = []
+    data_worker = DataService(TESTCONFIG)
+    # Act
+    result = data_worker.query(queryItems)
+    # Assert
+    assert result['Error'] is not None
+
+
+def test_nullQuery():
+    # Arrange
+    queryItems = None
+    data_worker = DataService(TESTCONFIG)
+    # Act
+    result = data_worker.query(queryItems)
+    # Assert
+    assert result['Error'] is not None
diff --git a/src/App.jsx b/src/App.jsx
@@ -4,6 +4,7 @@ import { connect } from 'react-redux';
 import Header from './components/main/header/Header';
 import Body from './components/main/body/Body';
 import Footer from './components/main/footer/Footer';
+import Tooltip from './components/main/tooltip/Tooltip';
 
 const App = () => {
   useEffect(() => {
@@ -15,6 +16,7 @@ const App = () => {
       <Header />
       <Body />
       <Footer />
+      <Tooltip />
     </div>
   );
 };

diff --git a/src/components/common/HoverOverInfo.jsx b/src/components/common/HoverOverInfo.jsx
@@ -0,0 +1,28 @@
+import React from 'react';
+import PropTypes from 'proptypes';
+
+const HoverOverInfo = ({
+  title,
+  text,
+  position,
+  children
+}) => (
+  <span
+    data-for="react-tooltip"
+    data-tip={JSON.stringify({ title, text })}
+    data-place={position}>
+    { children }
+  </span>
+);
+
+export default HoverOverInfo;
+
+HoverOverInfo.propTypes = {
+  title: PropTypes.string,
+  text: PropTypes.string,
+  position: PropTypes.oneOf(['top', 'bottom', 'left', 'right']),
+};
+
+HoverOverInfo.defaultProps = {
+  position: 'right'
+};