New API testing

.github/workflows/Continuous_Integration_Backend.yml

@@ -1,9 +1,9 @@
 name: CI_Backend
 
-on:
-  pull_request:
-    paths:
-      - 'server/**'
+on: [push]
+  # pull_request:
+  #   paths:
+  #     - 'server/**'
 
 defaults:
   run:
@@ -23,8 +23,8 @@ jobs:
           docker-compose up --no-start api
 
       - name: Seed DB and Run Api
-        run: |          
-            docker-compose run api python bin/db_seed.py --file data/requests.csv.gz
+        run: |
+            docker-compose run -e TESTING=True api alembic upgrade head
             docker-compose up -d api
 
       - name: Linting

.github/workflows/Daily_Backend_Update_Dev.yml

@@ -0,0 +1,26 @@
+name: Daily_Backend_Update_Dev
+
+on:
+  schedule:
+    - cron: '0 7 * * *'
+
+jobs:
+  dev_update:
+    runs-on: ubuntu-latest
+    steps:
+      - name: run update script on DEV environment
+        uses: appleboy/ssh-action@master
+        env:
+          PREFECT__CONTEXT__SECRETS__SLACK_HOOK: ${{ secrets.SLACK_HOOK_DEV }}
+        with:
+          username: ec2-user
+          host: ${{ secrets.AWS_SSH_HOST_DEV }}
+          key: ${{ secrets.AWS_SSH_PEM_KEY }}
+          envs: PREFECT__CONTEXT__SECRETS__SLACK_HOOK
+          script: |
+            cd 311-data/server
+            docker-compose run -e PREFECT__CONTEXT__SECRETS__SLACK_HOOK prefect python flow.py
+      - name: clear API cache with curl
+        uses: wei/curl@v1
+        with:
+          args: -X POST ${{ secrets.API_URL_DEV }}/status/reset-cache

.github/workflows/Daily_Backend_Update_Prod.yml

@@ -0,0 +1,21 @@
+name: Daily_Backend_Update_Prod
+
+on:
+  schedule:
+    - cron: '10 7 * * *'
+
+jobs:
+  prod_update:
+    runs-on: ubuntu-latest
+    steps:
+      - name: run update script on AWS prod
+        uses: appleboy/ssh-action@master
+        with:
+          username: ec2-user
+          host: ${{ secrets.AWS_SSH_HOST_PROD }}
+          key: ${{ secrets.AWS_SSH_PEM_KEY }}
+          command_timeout: 60m
+          script: |
+            set -e
+            cd 311-data/server
+            docker-compose run --rm api python bin/db_update.py

.github/workflows/create_release.yml

@@ -0,0 +1,44 @@
+name: Create Release
+on:
+  push:
+    # Sequence of patterns matched against refs/tags
+    branches:
+      - 'release/v*' # Push events to matching v*, i.e. v1.0, v20.15.10    
+jobs:
+  build:
+    name: Create Release and Docker Images
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: Branch name
+        uses: nelonoel/branch-name@v1
+      - name: Create Release
+        id: create_release
+        uses: actions/create-release@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # This token is provided by Actions, you do not need to create your own token
+        with:
+          tag_name: ${{ env.BRANCH_NAME }}
+          release_name: Release ${{ env.BRANCH_NAME }}
+          body_path: server/docs/release_notes.md
+          draft: false
+          prerelease: false
+      - name: Build and Push API to GitHub Packages
+        uses: docker/build-push-action@v1
+        with:
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+          path: server/api
+          registry: docker.pkg.github.com
+          repository: hackforla/311-data/api
+          tags: latest, ${{ env.BRANCH_NAME }}
+      - name: Build and Push Prefect to GitHub Packages
+        uses: docker/build-push-action@v1
+        with:
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+          path: server/prefect
+          registry: docker.pkg.github.com
+          repository: hackforla/311-data/prefect
+          tags: latest, ${{ env.BRANCH_NAME }}

.gitignore

@@ -23,3 +23,6 @@ yarn-error.log*
 # config
 config.js
 .env
+
+# builds
+dist

e2e/cypress/integration/e2e_tests.spec.js

@@ -3,7 +3,7 @@ describe('311 Data', () => {
     const {_} = Cypress
 
     context('Explore My Council\'s 311 Data', () => {
-        it('Map: Last Week, Dead Animal [DAN]', () => {
+        it('Map: Year-to-date, Dead Animal [DAN]', () => {
 
             cy.server()
             cy.route({
@@ -19,9 +19,9 @@ describe('311 Data', () => {
             cy.get('.menu-tabs > :nth-child(1)').click()
             cy.get('.active').click()
             cy.get('.dropdown-trigger > .button').click()
-            cy.get('[value="LAST_WEEK"]').click()
+            cy.get('[value="YEAR_TO_DATE"]').click()
             cy.get(':nth-child(2) > .select-group-content > .select-item > .select-item-box').click()
-            cy.get('#type-selector-container > div.columns.is-gapless > div:nth-child(1) > div:nth-child(2) > span > input').click({force: true})
+            cy.get('#checkbox-request-checkbox-all').click({force: true})
             cy.get('#btn-sidebar-submit-button').click()
 
             cy.wait('@getFilterList').should((xhr) => {
@@ -41,7 +41,7 @@ describe('311 Data', () => {
             })
         })
 
-        it('Data Visualization: Last Week, Dead Animal [DAN]', () => {
+        it('Data Visualization: Year-to-date, Dead Animal [DAN]', () => {
 
             cy.server()
             cy.route({
@@ -56,9 +56,9 @@ describe('311 Data', () => {
             cy.get('.navbar-end > :nth-child(1)').click()
             cy.get('.menu-tabs > :nth-child(2)').click()
             cy.get('.dropdown-trigger > .button').click()
-            cy.get('[value="LAST_WEEK"]').click()
+            cy.get('[value="YEAR_TO_DATE"]').click()
             cy.get(':nth-child(2) > .select-group-content > .select-item > .select-item-box').click()
-            cy.get('#type-selector-container > div.columns.is-gapless > div:nth-child(1) > div:nth-child(2) > span > input').click({force: true})
+            cy.get('#checkbox-request-checkbox-all').click({force: true})
             cy.get('#btn-sidebar-submit-button').click()
 
             cy.wait('@getFilterList').should((xhr) => {
@@ -81,7 +81,7 @@ describe('311 Data', () => {
             cy.visit('/')
             cy.get('#navbar > div > div:nth-child(2)').click()
             cy.get('#date-selector-dropdown > .dropdown-trigger > .button').click()
-            cy.get('[value="LAST_WEEK"]').click()
+            cy.get('[value="YEAR_TO_DATE"]').click()
             cy.get('.district-selector > :nth-child(1)').click()
             cy.get('#district-selector-dropdown > .dropdown-trigger > .button').click()
             cy.get('[value="nc"]').click()

e2e/cypress/support/commands.js

@@ -28,7 +28,11 @@ import '@testing-library/cypress/add-commands'
 //import 'cypress-mailosaur'
 
 Cypress.Commands.add('format_number_with_K', (value) => {
-    return ((value / 1000).toFixed(1)) + "K"
+    if (value > 9999) {
+        return ((value / 1000).toFixed(0)) + "K"
+    } else {
+        return ((value / 1000).toFixed(1)) + "K"
+    }
 })
 
 Cypress.Commands.add('format_number_with_comma', (value) => {

misc/workflows/End_To_End_Tests.yml

@@ -1,3 +1,4 @@
+
 # DISABLED UNTIL: (1) we get more test coverage, and
 # (2) the tests run against the code that's being pushed
 # instead of the code already on dev

server/.env.example

@@ -1,58 +1,44 @@
 
-############################### API SETTINGS #############################
-
-# Server
-HOST=0.0.0.0
-PORT=5000
-DEBUG=0
-ACCESS_LOG=1
-AUTO_RELOAD=1
-WORKERS=1
-TMP_DIR=./__tmp__
-UPDATE_ON_START=0
-
-# Database
-DATABASE_URL=postgresql://311_user:311_pass@db:5432/311_db
-DATABASE_LOG_QUERIES=0
-
-# Redis
-REDIS_ENABLED=1
-REDIS_URL=redis://redis:6379
-REDIS_TTL_SECONDS=3600
-
-# Picklebase
-PICKLEBASE_ENABLED=0
-PICKLEBASE_BATCH_SIZE=400000
-
-# Picklecache
-PICKLECACHE_ENABLED=0
-PICKLECACHE_TTL_SECONDS=3600
-
-# Socrata
-SOCRATA_TOKEN=
-SOCRATA_BATCH_SIZE=50000
-
-# Github
+######################### DOCKER-COMPOSE SETTINGS ########################
+
+COMPOSE_PROJECT_NAME=311_data
+
+# host/public configs 80/5432 in prod
+API_HOST_PORT=5000
+DB_HOST_PORT=5433
+
+# api config
+DEBUG=True
+API_RESTART_POLICY=no
+APP_PORT=5000
+
+# db connection config
+DB_HOST=db
+DB_PORT=5432
+DB_USER=311_user
+DB_PASS=311_pass
+DB_NAME=311_db_test
+DB_ECHO=False
+
+############################ PREFECT CONFIG ##############################
+
+# to populate a new database for development uncomment:
+# PREFECT__MODE=full
+# PREFECT__DATA__YEARS=2020,2019
+
+############################ 3RD PARTY TOOLS ##############################
+
+# GitHub
 GITHUB_TOKEN=
 GITHUB_ISSUES_URL=https://api.github.com/repos/hackforla/311-data-support/issues
 GITHUB_PROJECT_URL=
 GITHUB_SHA=DEVELOPMENT
-
-# Slack
-SLACK_WEBHOOK_URL=
-SLACK_ERROR_CODES=[400, 500]
+GITHUB_CODE_VERSION=2.0.0-alpha
 
 # Sendgrid
 SENDGRID_API_KEY=
 
-######################### DOCKER-COMPOSE SETTINGS ########################
-
-COMPOSE_PROJECT_NAME=311_data
-DB_USER=311_user
-DB_PASS=311_pass
-DB_NAME=311_db
-DB_HOST_PORT=5433
-API_HOST_PORT=5000
-API_RESTART_POLICY=no
+# Sentry
+SENTRY_URL=https://[email protected]/5439266
 
-################################# OVERRIDES ##############################
+############################################################################

server/.gitignore

@@ -1,7 +1,9 @@
-*.env
+.env
 
-/api/__tmp__
+__tmp__
 /jmeter/out
 
 __pycache__
 .pytest_cache
+
+__working__

server/api/.dockerignore

@@ -0,0 +1,15 @@
+# Ignore
+*
+
+# Don't ignore
+!requirements.txt
+
+!Dockerfile
+!README.md
+!alembic
+!alembic.ini
+!bin
+!code
+!setup.cfg
+!src
+!tests