Adding async redis caching

.github/workflows/Continuous_Integration_Backend.yml

@@ -1,9 +1,9 @@
 name: CI_Backend
 
-on:
-  pull_request:
-    paths:
-      - 'server/**'
+on: [push]
+  # pull_request:
+  #   paths:
+  #     - 'server/**'
 
 defaults:
   run:
@@ -23,8 +23,8 @@ jobs:
           docker-compose up --no-start api
 
       - name: Seed DB and Run Api
-        run: |          
-            docker-compose run api python bin/db_seed.py --file data/requests.csv.gz
+        run: |
+            docker-compose run -e TESTING=True api alembic upgrade head
             docker-compose up -d api
 
       - name: Linting

.gitignore

@@ -23,3 +23,6 @@ yarn-error.log*
 # config
 config.js
 .env
+
+# builds
+dist

e2e/cypress/integration/e2e_tests.spec.js

@@ -3,7 +3,7 @@ describe('311 Data', () => {
     const {_} = Cypress
 
     context('Explore My Council\'s 311 Data', () => {
-        it('Map: Last Week, Dead Animal [DAN]', () => {
+        it('Map: Year-to-date, Dead Animal [DAN]', () => {
 
             cy.server()
             cy.route({
@@ -19,9 +19,9 @@ describe('311 Data', () => {
             cy.get('.menu-tabs > :nth-child(1)').click()
             cy.get('.active').click()
             cy.get('.dropdown-trigger > .button').click()
-            cy.get('[value="LAST_WEEK"]').click()
+            cy.get('[value="YEAR_TO_DATE"]').click()
             cy.get(':nth-child(2) > .select-group-content > .select-item > .select-item-box').click()
-            cy.get('#type-selector-container > div.columns.is-0 > div:nth-child(1) > div:nth-child(2) > span > input').click({force: true})
+            cy.get('#checkbox-request-checkbox-all').click({force: true})
             cy.get('#btn-sidebar-submit-button').click()
 
             cy.wait('@getFilterList').should((xhr) => {
@@ -41,7 +41,7 @@ describe('311 Data', () => {
             })
         })
 
-        it('Data Visualization: Last Week, Dead Animal [DAN]', () => {
+        it('Data Visualization: Year-to-date, Dead Animal [DAN]', () => {
 
             cy.server()
             cy.route({
@@ -56,9 +56,9 @@ describe('311 Data', () => {
             cy.get('.navbar-end > :nth-child(1)').click()
             cy.get('.menu-tabs > :nth-child(2)').click()
             cy.get('.dropdown-trigger > .button').click()
-            cy.get('[value="LAST_WEEK"]').click()
+            cy.get('[value="YEAR_TO_DATE"]').click()
             cy.get(':nth-child(2) > .select-group-content > .select-item > .select-item-box').click()
-            cy.get('#type-selector-container > div.columns.is-0 > div:nth-child(1) > div:nth-child(2) > span > input').click({force: true})
+            cy.get('#checkbox-request-checkbox-all').click({force: true})
             cy.get('#btn-sidebar-submit-button').click()
 
             cy.wait('@getFilterList').should((xhr) => {
@@ -81,7 +81,7 @@ describe('311 Data', () => {
             cy.visit('/')
             cy.get('#navbar > div > div:nth-child(2)').click()
             cy.get('#date-selector-dropdown > .dropdown-trigger > .button').click()
-            cy.get('[value="LAST_WEEK"]').click()
+            cy.get('[value="YEAR_TO_DATE"]').click()
             cy.get('.district-selector > :nth-child(1)').click()
             cy.get('#district-selector-dropdown > .dropdown-trigger > .button').click()
             cy.get('[value="nc"]').click()

e2e/cypress/support/commands.js

@@ -28,7 +28,11 @@ import '@testing-library/cypress/add-commands'
 //import 'cypress-mailosaur'
 
 Cypress.Commands.add('format_number_with_K', (value) => {
-    return ((value / 1000).toFixed(1)) + "K"
+    if (value > 9999) {
+        return ((value / 1000).toFixed(0)) + "K"
+    } else {
+        return ((value / 1000).toFixed(1)) + "K"
+    }
 })
 
 Cypress.Commands.add('format_number_with_comma', (value) => {

server/.env.example

@@ -1,58 +1,39 @@
 
-############################### API SETTINGS #############################
-
-# Server
-HOST=0.0.0.0
-PORT=5000
-DEBUG=0
-ACCESS_LOG=1
-AUTO_RELOAD=1
-WORKERS=1
-TMP_DIR=./__tmp__
-UPDATE_ON_START=0
-
-# Database
-DATABASE_URL=postgresql://311_user:311_pass@db:5432/311_db
-DATABASE_LOG_QUERIES=0
-
-# Redis
-REDIS_ENABLED=1
-REDIS_URL=redis://redis:6379
-REDIS_TTL_SECONDS=3600
-
-# Picklebase
-PICKLEBASE_ENABLED=0
-PICKLEBASE_BATCH_SIZE=400000
-
-# Picklecache
-PICKLECACHE_ENABLED=0
-PICKLECACHE_TTL_SECONDS=3600
-
-# Socrata
-SOCRATA_TOKEN=
-SOCRATA_BATCH_SIZE=50000
-
-# Github
+######################### DOCKER-COMPOSE SETTINGS ########################
+
+COMPOSE_PROJECT_NAME=311_data
+
+# host/public configs
+API_HOST_PORT=5000
+DB_HOST_PORT=5433
+
+# api config
+DEBUG=True
+API_RESTART_POLICY=no
+APP_PORT=5000
+API_LEGACY_MODE=False
+
+# db connection config
+DB_HOST=db
+DB_PORT=5432
+DB_USER=311_user
+DB_PASS=311_pass
+DB_NAME=311_db_test
+DB_ECHO=False
+
+############################ 3RD PARTY TOOLS ##############################
+
+# GitHub
 GITHUB_TOKEN=
 GITHUB_ISSUES_URL=https://api.github.com/repos/hackforla/311-data-support/issues
 GITHUB_PROJECT_URL=
 GITHUB_SHA=DEVELOPMENT
-
-# Slack
-SLACK_WEBHOOK_URL=
-SLACK_ERROR_CODES=[400, 500]
+GITHUB_CODE_VERSION=0.1.1
 
 # Sendgrid
-SENDGRID_API_KEY=
+SENDGRID_API_KEY=33333333
 
-######################### DOCKER-COMPOSE SETTINGS ########################
-
-COMPOSE_PROJECT_NAME=311_data
-DB_USER=311_user
-DB_PASS=311_pass
-DB_NAME=311_db
-DB_HOST_PORT=5433
-API_HOST_PORT=5000
-API_RESTART_POLICY=no
+# Sentry
+SENTRY_URL="https://[email protected]/5439266"
 
-################################# OVERRIDES ##############################
+############################################################################

server/.gitignore

@@ -1,7 +1,9 @@
-*.env
+.env
 
 /api/__tmp__
 /jmeter/out
 
 __pycache__
 .pytest_cache
+
+__working__

server/api/.dockerignore

@@ -0,0 +1,15 @@
+# Ignore
+*
+
+# Don't ignore
+!requirements.txt
+
+!Dockerfile
+!README.md
+!alembic
+!alembic.ini
+!bin
+!code
+!setup.cfg
+!src
+!tests

server/api/Dockerfile

@@ -1,23 +1,23 @@
-FROM python:3.7-slim
+FROM python:3.7-slim-buster
 
-RUN apt-get update && \
-    apt-get install -yq \
-      python3 \
-      python3-dev \
-      gcc \
-      g++ \
-      gfortran \
-      musl-dev && \
-    pip install --upgrade pip
+RUN apt-get update \
+    && apt-get install -y \
+    gcc \
+    g++ \
+    && apt-get clean
 
-COPY requirements.txt /home/api/
+WORKDIR /home/api
+
+COPY requirements.txt .
 
 RUN pip install --no-cache-dir -r /home/api/requirements.txt
 
-COPY setup.cfg /home/api/
-COPY /bin /home/api/bin/
-COPY .env* /src /home/api/src/
+ENV APP_HOST=0.0.0.0
+# need add the src folder to sys.path for compatibility
+ENV PYTHONPATH=.:/home/api/code:/home/api/src
 
-WORKDIR /home/api
+EXPOSE 5000
+
+COPY . .
 
-CMD python bin/api_check.py && python bin/api_start.py
+CMD gunicorn lacity_data_api.asgi:app -w 4 -k uvicorn.workers.UvicornWorker -b 0.0.0.0:5000

server/api/Pipfile

@@ -7,23 +7,29 @@ verify_ssl = true
 pytest-asyncio = "*"
 pytest = "*"
 flake8 = "*"
+requests = "*"
+pytest-cov = "*"
 
 [packages]
 numpy = "*"
 pandas = "*"
 psycopg2-binary = "*"
 pysupercluster = "*"
-python-dotenv = "*"
 redis = "*"
 requests-async = "*"
-sanic = "*"
-sanic-plugins-framework = "*"
-sanic-cors = "*"
-sanic-compress = "*"
 sendgrid = "*"
 sodapy = "*"
 SQLAlchemy = "*"
 tabulate = "*"
+gino = {extras = ["starlette"], version = "*"}
+fastapi = "*"
+uvicorn = "*"
+alembic = "*"
+geopandas = "*"
+ujson = "*"
+sentry-sdk = "*"
+aiocache = {extras = ["redis"], version = "*"}
+gunicorn = "*"
 
 [requires]
 python_version = "3.7"