From 2177400bed072a80b886cca97ca1524316de6a11 Mon Sep 17 00:00:00 2001 From: Matt Webster Date: Wed, 2 Dec 2020 19:16:46 -0800 Subject: [PATCH 1/2] adding docker tools for client side devs --- client/docker/README.md | 40 ++++++++++++++++++++++++ client/docker/api.env | 28 +++++++++++++++++ client/docker/docker-compose.yml | 52 ++++++++++++++++++++++++++++++++ client/docker/init_api.sh | 6 ++++ 4 files changed, 126 insertions(+) create mode 100644 client/docker/README.md create mode 100644 client/docker/api.env create mode 100644 client/docker/docker-compose.yml create mode 100755 client/docker/init_api.sh diff --git a/client/docker/README.md b/client/docker/README.md new file mode 100644 index 000000000..5b532dbe7 --- /dev/null +++ b/client/docker/README.md @@ -0,0 +1,40 @@ +# Install 311 Server locally for development + +This doc in intended for client side developers working on the 311 Data project who want an easy way to run a local copy of the server. It downloads images for whatever code is in the ```dev``` branch in GitHub. + +## Introduction and Prerequisites + +These instructions assume only that Docker is installed. They also assume the steps are being followed on a Mac, but it should be easy enough to follow for other platforms. + +All of the 311 Server components (e.g. Postgres, Redis) are downloaded and run as Docker containers. This wouldn't be the best configuration for a production system, but it is more than adequate for development. + +The 4 included images will need about 1 GB of space with an additional 1 GB or so needed for the data. + +## Initialization Script + +```bash +cp api.env .env # copy the sample variables to .env +docker-compose up -d # downloads and starts all images +docker-compose run api alembic upgrade head # install the database schema +docker-compose run prefect python flow.py # loads data to database +open http://localhost:5000/docs # open the docs page (on a Mac) +``` + +## Installation Instructions + +The easiest way to install the 311 Server is with the included initialization script. Open a terminal to the client/docker directory and execute the following commands. + +```bash +chmod +x init_api.sh # make the init bash script executable +./init_api.sh # execute the script +``` + +The process should take about 5 minutes to run from start to finish. + +## Updating the data + +In order to have current data for reports the prefect container should be run periodically. It will get any data added or changed since the last time it was run. + +```bash +docker-compose run prefect python flow.py # loads new data to database +``` diff --git a/client/docker/api.env b/client/docker/api.env new file mode 100644 index 000000000..dfc9a7aa8 --- /dev/null +++ b/client/docker/api.env @@ -0,0 +1,28 @@ + +######################### DOCKER-COMPOSE SETTINGS ######################## + +COMPOSE_PROJECT_NAME=311_api + +# host/public configs +API_HOST_PORT=5000 +DB_HOST_PORT=5432 + +# api config +DEBUG=True +API_RESTART_POLICY=no +APP_PROTOCOL=http +APP_HOST=api +APP_PORT=5000 + +# db connection config +DB_HOST=db +DB_PORT=5432 +DB_USER=311_user +DB_PASS=311_pass +DB_NAME=311_db +DB_ECHO=False + +############################ PREFECT CONFIG ############################## + +# to populate a new database for development +PREFECT__DATA__YEARS=2020,2019 diff --git a/client/docker/docker-compose.yml b/client/docker/docker-compose.yml new file mode 100644 index 000000000..8e4672666 --- /dev/null +++ b/client/docker/docker-compose.yml @@ -0,0 +1,52 @@ +version: '3.8' + +services: + db: + container_name: 311-postgres + image: postgres:12 + restart: always + env_file: .env + environment: + POSTGRES_USER: ${DB_USER} + POSTGRES_PASSWORD: ${DB_PASS} + POSTGRES_DB: ${DB_NAME} + ports: + - target: 5432 + published: ${DB_HOST_PORT} + volumes: + - backend_data:/var/lib/postgresql/data + + redis: + container_name: 311-redis + image: redis:6 + restart: always + ports: + - "6379:6379" + + api: + container_name: 311-api + image: la311data/311_data_api:dev + restart: ${API_RESTART_POLICY} + env_file: .env + environment: + DATABASE_URL: postgresql://${DB_USER}:${DB_PASS}@db:5432/${DB_NAME} + CACHE_ENDPOINT: redis + ports: + - target: ${APP_PORT} + published: ${API_HOST_PORT} + depends_on: + - db + - redis + + prefect: + container_name: 311-prefect + image: la311data/311_data_prefect:dev + env_file: .env + environment: + PREFECT__CONTEXT__SECRETS__DSN: postgresql://${DB_USER}:${DB_PASS}@${DB_HOST}:5432/${DB_NAME} + PREFECT__API_URL: ${APP_PROTOCOL}://${APP_HOST}:${APP_PORT} + depends_on: + - db + +volumes: + backend_data: diff --git a/client/docker/init_api.sh b/client/docker/init_api.sh new file mode 100755 index 000000000..51fb186d5 --- /dev/null +++ b/client/docker/init_api.sh @@ -0,0 +1,6 @@ +#!/bin/bash +cp api.env .env +docker-compose up -d # downloads and starts all images (in background) +docker-compose run api alembic upgrade head # install the database schema +docker-compose run prefect python flow.py # loads data to database +open http://localhost:5000/docs # open the docs page (on a Mac) From a10e46f2b2e5af9b07c354724b19d1a43a4d0196 Mon Sep 17 00:00:00 2001 From: Matt Webster <10199792+mattyweb@users.noreply.github.com> Date: Thu, 3 Dec 2020 21:35:28 -0800 Subject: [PATCH 2/2] Made doc changes Making changes to the doc and merging to get code on dev. --- client/docker/README.md | 33 ++++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/client/docker/README.md b/client/docker/README.md index 5b532dbe7..63e2f8f0d 100644 --- a/client/docker/README.md +++ b/client/docker/README.md @@ -10,16 +10,6 @@ All of the 311 Server components (e.g. Postgres, Redis) are downloaded and run a The 4 included images will need about 1 GB of space with an additional 1 GB or so needed for the data. -## Initialization Script - -```bash -cp api.env .env # copy the sample variables to .env -docker-compose up -d # downloads and starts all images -docker-compose run api alembic upgrade head # install the database schema -docker-compose run prefect python flow.py # loads data to database -open http://localhost:5000/docs # open the docs page (on a Mac) -``` - ## Installation Instructions The easiest way to install the 311 Server is with the included initialization script. Open a terminal to the client/docker directory and execute the following commands. @@ -31,10 +21,31 @@ chmod +x init_api.sh # make the init bash script executable The process should take about 5 minutes to run from start to finish. -## Updating the data +## Updating data and code In order to have current data for reports the prefect container should be run periodically. It will get any data added or changed since the last time it was run. ```bash docker-compose run prefect python flow.py # loads new data to database ``` + +When there are changes to the code in the /server/api folder on the dev branch, new images are automatically created and uploaded to Docker Hub. In order to get these images run the following commands from the client/docker directory. + +```bash +docker-compose pull # downloads new images +docker-compose up +``` + +## Stopping and Removing images + +When you don't need the images during development you can stop them (but not remove them) with the ```down``` command. + +```bash +docker-compose down +``` + +To completely delete the images and associated volumes run the ```rm``` command. + +```bash +docker-compose rm +```