backend refactor

.github/workflows/Continuous_Integration_Backend.yml

@@ -17,13 +17,10 @@ jobs:
       run: |
         sudo apt install libpq-dev python3-dev
         python -m pip install --upgrade pip
-        pip install -r server/requirements.txt
-        cp server/src/settings.example.cfg server/src/settings.cfg
+        pip install -r server/api/requirements.txt
+        cp server/.env.example server/api/.env
     - name: Lint with flake8
-      run: |
-        # stop the build if there are Python syntax errors or undefined names
-        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
-        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
-        flake8 . --count --max-complexity=12 --max-line-length=127 --statistics
-    - name: Test with pytest
-      run: pytest server
+      run: flake8 server
+    # disabled until we have a test DB to connect to
+    # - name: Test with pytest
+    #   run: pytest server

.github/workflows/Publish_Backend_Package.yml

@@ -18,15 +18,15 @@ jobs:
         python-version: ${{ matrix.python-version }}
     - name: Setup settings
       run: |
-        cp server/src/settings.example.cfg server/src/settings.cfg
+        cp server/.env.example server/api/.env
     - name: Build and Publish to Registry
       uses: elgohr/Publish-Docker-Github-Action@master
       with:
         name: hackforla/311-data/backend
         username: ${{ secrets.DOCKER_USERNAME }}
         password: ${{ secrets.DOCKER_PASSWORD }}
         registry: docker.pkg.github.com
-        dockerfile: server/Dockerfile
+        dockerfile: server/api/Dockerfile
         context: server
         tags: "latest, ${{github.sha}}"
     - name: Login to heroku
@@ -37,7 +37,7 @@ jobs:
       env:
         HEROKU_API_KEY: ${{ secrets.HEROKU_API_KEY }}
       run: |
-        docker tag docker.pkg.github.com/hackforla/311-data/backend:${{github.sha}} registry.heroku.com/hackforla-311/web
+        docker tag docker.pkg.github.com/hackforla/311-data/api:${{github.sha}} registry.heroku.com/hackforla-311/web
         docker push registry.heroku.com/hackforla-311/web
     - name: Release
       env:

.github/workflows/dev-server-CI.yml

@@ -0,0 +1,25 @@
+name: dev-server-CI
+
+on:
+  push:
+    branches:
+      - dev
+
+jobs:
+  start:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Pull, Build, Restart
+        uses: peterkimzz/[email protected]
+        id: ssm
+        with:
+          aws-region: us-east-1
+          aws-access-key-id: ${{ secrets.AWS_CI_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_CI_SECRET_ACCESS_KEY }}
+          instance-ids: ${{ secrets.AWS_DEV_INSTANCE_ID }}
+          working-directory: /home/ec2-user/311-data/server
+          command: |
+            git pull
+            docker-compose build api
+            docker-compose up --no-deps -d api

.gitignore

@@ -38,14 +38,14 @@ config.js
 .env
 settings.cfg
 
-docker-compose.yml
 Orchestration/docker-compose.yml
 
 __pycache__/
 
 # csv files
 /dataAnalysis/datasets
 /server/src/static
+__tmp__
 
 # checkpoints
 /dataAnalysis/.ipynb_checkpoints

server/.env.example

@@ -0,0 +1,50 @@
+
+############################### API SETTINGS #############################
+
+# Server
+PORT=5000
+DEBUG=0
+ACCESS_LOG=1
+AUTO_RELOAD=1
+WORKERS=1
+TMP_DIR=./__tmp__
+
+# Database
+DATABASE_URL=postgresql://311_user:311_pass@db:5432/311_db
+DATABASE_LOG_QUERIES=0
+
+# Redis
+REDIS_ENABLED=1
+REDIS_URL=redis://redis:6379
+REDIS_TTL_SECONDS=3600
+
+# Picklebase
+PICKLEBASE_ENABLED=0
+PICKLEBASE_BATCH_SIZE=400000
+
+# Picklecache
+PICKLECACHE_ENABLED=0
+
+# Socrata
+SOCRATA_TOKEN=
+SOCRATA_BATCH_SIZE=50000
+
+# Github
+GITHUB_TOKEN=
+GITHUB_ISSUES_URL=https://api.github.com/repos/hackforla/311-data-support/issues
+GITHUB_PROJECT_URL=
+GITHUB_SHA=DEVELOPMENT
+
+# Slack
+SLACK_WEBHOOK_URL=
+SLACK_ERROR_CODES=[400, 500]
+
+######################### DOCKER-COMPOSE SETTINGS ########################
+
+COMPOSE_PROJECT_NAME=311_data
+DB_USER=311_user
+DB_PASS=311_pass
+DB_NAME=311_db
+DB_HOST_PORT=5433
+API_HOST_PORT=5000
+API_RESTART_POLICY=no

server/README.md

@@ -0,0 +1,102 @@
+## Getting Started
+
+### TL;DR
+  - install docker and docker compose on your machine
+  - run `chmod +x install.sh && ./install.sh` from this directory
+  - put a [Socrata](https://dev.socrata.com/) api token in your `.env` file
+  - run `docker-compose run api python bin/db_seed.py --years 2019,2020`
+
+### Step 1: install docker and docker-compose
+Docker and Docker-Compose are the only required dependencies for running the server. You can find installation instructions [here](https://docs.docker.com/compose/install/) for Docker and [here](https://docs.docker.com/compose/install/) for Docker-Compose.
+
+Once you're done, you can check that everything is working:
+```
+docker --version              # confirms docker is installed
+docker-compose --version      # confirms docker-compose is installed
+docker info                   # confirms the docker daemon is running
+```
+
+### Step 2: run the install script
+With docker running on your machine, `cd` into this directory and run:
+```
+chmod +x install.sh && ./install.sh
+```
+This will download/build a bunch of docker images, create your `.env` file, set up your database, and then fire everything up. If all goes well, at the end you should have running api server backed by Postgres and Redis. You can then tour the running services by hitting these URLs in a browser:
+- http://localhost:5000 -- the api -- should say "you hit the index". This means the api is running.
+- http://localhost:8080 -- a postgres GUI. Login with the following:
+  - System: **PostgreSQL**
+  - Server: **db**
+  - Username: **311_user**
+  - Password: **311_pass**
+  - Database: **311_db**
+- http://localhost:5001 -- a redis GUI. Login with the following:
+  - Host: **redis**
+  - Port: **6379**
+  - Database ID: **0**
+
+You can shut down all the services by hitting `Ctrl-C`. And run `docker-compose up` to bring everything back up again.
+
+### Step 3: seed your database
+Right now the server is functional and all endpoints should be working. But for most purposes you'll need some data in your database. The data comes from [Socrata](https://dev.socrata.com/), a public api that hosts many datasets for LA and other cities. To add data, you'll need to get a Socrata token and run one more command.
+
+- #### 3a: add a Socrata token to your .env file (possible optional)
+Socrata threatens to throttle you if you don't have an api token. We're not sure they actually do that, but the api does seem to run more slowly without a token. So get a token from another team member, or [register](https://opendata.socrata.com/login) with Socrata and they'll give you one. Then add it to the Socrata section in your `.env` file:
+```
+SOCRATA_TOKEN=thetoken
+```
+- #### 3b: run the seed script
+It takes a while to seed the database, so we recommend starting with 2 years of data from Socrata. Run the following command to get data for 2019 and 2020, which is plenty for most dev purposes. ETA **20 to 30 minutes**. (If you've still got the backend services running, you can run this command in a separate terminal window.)
+```
+docker-compose run api python bin/db_seed.py --years 2019,2020
+```
+If you decide later that you need more data, just run the command again with the year(s) you want to add. Socrata goes back to 2015.
+
+(NOTE: run the above command with `--help` instead of `--years` for more info on the options. And if you ever want to reset your database and start over, run `docker-compose run api python bin/db_reset.py`.)
+
+
+## Development
+
+### Optional Dependencies
+
+  - #### Postman
+
+
+### Useful commands
+```
+docker-compose up                     # start the backend services
+docker-compose up --build             # start the backend services after rebuilding containers
+
+docker-compose run api bash           # log in to api shell
+docker-compose run api flake8         # lint your python code
+docker-compose run api pytest         # run unit tests against python code
+
+docker-compose run redis redis-cli    # run the redis cli  
+```
+
+### Using the python interpreter
+```
+docker-compose run api bash
+cd src
+python
+
+Python 3.7.7 (default, May 20 2020, 21:10:21)
+[GCC 8.3.0] on linux
+Type "help", "copyright", "credits" or "license" for more information.
+>>> import db
+>>> db.version()                      # print version
+>>> db.info.years()                   # list years currently in db
+>>> db.info.rows()                    # row counts for years in db
+>>> db.requests.add_years([2018])     # add 2018 to DB
+>>> db.requests.drop_years([2018])    # drop 2018
+>>> db.reset()                        # wipe the DB and recreate tables/views
+```
+
+### Dev workflow
+
+
+
+## Uninstall
+Run this command to remove all docker containers, images, volumes, and networks that are specific to this project. It will leave in place generic docker assets (like the `postgres` and `python` images) that you may be using for other purposes.
+```
+docker-compose down --rmi local --volumes
+```

server/api/Dockerfile

@@ -0,0 +1,22 @@
+FROM python:3.7-slim
+
+RUN apt-get update && \
+    apt-get install -yq \
+      python3 \
+      python3-dev \
+      gcc \
+      g++ \
+      gfortran \
+      musl-dev && \
+    pip install --upgrade pip
+
+COPY requirements.txt /home/api/
+
+RUN pip install --no-cache-dir -r /home/api/requirements.txt
+
+COPY /bin /home/api/bin/
+COPY .env* /src /home/api/src/
+
+WORKDIR /home/api
+
+CMD python bin/api_check.py && python bin/api_start.py

server/api/bin/api_check.py

@@ -0,0 +1,95 @@
+import sys
+from os.path import join, dirname
+sys.path.append(join(dirname(__file__), '../src'))
+
+
+def check_db_version():
+    import db
+    from utils.log import log, log_colors
+
+    setup_message = """
+        Your database is not set up. Please run:
+
+        docker-compose run api python bin/db_reset.py
+    """
+
+    migrate_message = """
+        You're running an old version of the database. Please run
+        this command to get the latest:
+
+        docker-compose run api python bin/db_migrate.py
+    """
+
+    version = db.version()
+    latest_version = 0  # will come from the migrate module
+
+    if version == -1:
+        log(setup_message, color=log_colors.FAIL, dedent=True)
+        sys.exit(1)
+
+    elif version < latest_version:
+        log(migrate_message, color=log_colors.FAIL, dedent=True)
+        sys.exit(1)
+
+    else:
+        log('DB structure OK')
+
+
+def check_db_update():
+    import db
+    from datetime import datetime
+    from utils.log import log, log_colors
+
+    last_updated = db.info.last_updated()
+    time_since_update = datetime.utcnow() - last_updated
+    log('DB last updated: {}'.format(last_updated.isoformat()))
+
+    if time_since_update.days > 7:
+        update_message = """
+            Your database hasn't been updated in a while. If you'd like
+            to get the latest Socrata data, open a new terminal and run:
+
+            docker-compose run api python bin/db_update.py"""
+
+        log(update_message, color=log_colors.WARNING, dedent=True)
+
+
+def show_db_contents():
+    import db
+    from tabulate import tabulate
+    from settings import Socrata
+
+    years = sorted(Socrata.DATASET_IDS.keys())
+    info_rows = db.info.rows()['byYear']
+    rows = [info_rows.get(year, 0) for year in years]
+
+    print(tabulate({
+        'year': years,
+        'requests': rows,
+    }, tablefmt='psql', headers='keys'))
+
+
+if __name__ == '__main__':
+    from utils.log import log_heading
+    import time
+
+    time.sleep(1)
+
+    log_heading('checks')
+    check_db_version()
+    check_db_update()
+
+    # log_heading('database contents')
+    # show_db_contents()
+
+    import pb
+    if not pb.enabled:
+        pb.clear_data()
+    elif not pb.available():
+        pb.populate()
+
+    from utils.settings import log_settings
+    log_heading('settings')
+    log_settings()
+
+    log_heading('starting server')

server/api/bin/api_start.py

@@ -0,0 +1,8 @@
+import sys
+from os.path import join, dirname
+sys.path.append(join(dirname(__file__), '../src'))
+
+
+if __name__ == '__main__':
+    import app
+    app.start()

server/api/bin/db_migrate.py

@@ -0,0 +1,7 @@
+import sys
+from os.path import join, dirname
+sys.path.append(join(dirname(__file__), '../src'))
+
+
+if __name__ == '__main__':
+    print('to be implemented')