Merge pull request #301 from CSCfi/feature/update-dev-build

This commit adds the use of .env file for `docker-compose up` so that developer's own environment variables would not be accidentally pushed to the repository. Also makes it easy to handle your own development environment. ### Changes Made Docker-compose and docker-compose-tls files changed to use variables from .env file. Adds .env.example file to copy it to user's local .env file. Updates README with instructions. Updates .gitignore to ignore .env file Updates github workflow commands on docker-compose with env file flag Additional changes: Update README with instructions on Python requirements. Update pre-commit script to execute tests even if .tox folder does not exist.
CSCfi · Nov 30, 2021 · 0e80b7e · 0e80b7e
2 parents 36ce69b + 5bf4236
commit 0e80b7e
Show file tree

Hide file tree

Showing 7 changed files with 107 additions and 48 deletions.
diff --git a/.env.example b/.env.example
@@ -0,0 +1,33 @@
+      # authentication
+      AAI_CLIENT_SECRET=secret_must_be_long
+      AAI_CLIENT_ID=aud2
+      ISS_URL=http://mockauth:8000
+      AUTH_URL=http://localhost:8000/authorize
+      OIDC_URL=http://mockauth:8000
+      AUTH_REFERER=http://mockauth:8000
+      JWK_URL=http://mockauth:8000/keyset
+
+      # app urls
+      BASE_URL=http://localhost:5430
+      # change to http://frontend:3000 if started using docker-compose for frontend
+      REDIRECT_URL=http://localhost:3000
+
+      # logging
+      LOG_LEVEL=DEBUG
+
+      # database
+      MONGO_HOST=database:27017
+      MONGO_DATABASE=default
+      MONGO_AUTHDB=admin
+      MONGO_INITDB_ROOT_PASSWORD=admin
+      MONGO_INITDB_ROOT_USERNAME=admin
+      MONGO_SSL=true
+      MONGO_SSL_CA=/tls/cacert
+      MONGO_SSL_CLIENT_KEY=/tls/key
+      MONGO_SSL_CLIENT_CERT=/tls/cert
+
+      # doi
+      DOI_API=http://mockdoi:8001/dois
+      DOI_PREFIX=10.xxxx
+      DOI_USER=user
+      DOI_KEY=key
diff --git a/.github/workflows/int.yml b/.github/workflows/int.yml
@@ -26,7 +26,7 @@ jobs:
         
     - name: Start Services
       run: |
-        docker-compose up -d --build
+        docker-compose --env-file .env.example up -d --build 
         sleep 30
   
     - name: Run Integration test
@@ -55,7 +55,7 @@ jobs:
 
     - name: Start services with TLS enabled
       run: |
-        docker-compose -f docker-compose-tls.yml up -d
+        docker-compose -f docker-compose-tls.yml --env-file .env.example up -d
         sleep 30
 
     - name: Run Integration test

diff --git a/.gitignore b/.gitignore
@@ -93,6 +93,7 @@ venv/
 ENV/
 env.bak/
 venv.bak/
+.env
 
 # Spyder project settings
 .spyderproject

diff --git a/README.md b/README.md
@@ -11,14 +11,18 @@ Service also validates submitted metadata objects against EGA XSD metadata model
 
 ## Install and run
 
-Requirements:
+### Requirements:
 - Python 3.8+
 - MongoDB
 - Docker + docker-compose
 
-For quick testing, launch both server and database with Docker by running `docker-compose up --build` (add `-d` flag to run containers in background). Server can then be found from `http://localhost:5430`.
+### For quick testing:
+- copy the contents of .env.example file to .env file
+- launch both server and database with Docker by running `docker-compose up --build` (add `-d` flag to run containers in background).
 
-For more detailed setup, do following:
+Server can then be found from `http://localhost:5430`.
+
+### For more detailed setup, do following:
 - Install project by running: `pip install .` in project root
 - Setup mongodb and env variables via desired way, details:
   - Server expects to find mongodb instance running, specified with following environment variables:
@@ -61,6 +65,25 @@ To start using the VS Code devcontainer:
 
 Git hooks are activated inside the local development environment which will run tox tests before pushing. To ignore them for fast updates use the flag `--no-verify`.
 
+### Keeping Python requirements up to date
+
+1. Install `pip-tools`:
+    * `pip install pip-tools`
+    * if using docker-compose pip-tools are installed automatically
+
+2. Add new packages to `requirements.in` or `requirements-dev.in`
+
+3. Update `.txt` file for the changed requirements file:
+    * `pip-compile requirements.in`
+    * `pip-compile requirements-dev.in`
+
+4. If you want to update all dependencies to their newest versions, run:
+    * `pip-compile --upgrade requirements.in`
+
+5. To install Python requirements run:
+    * `pip-sync requirements.txt`
+
+
 ## Build and deploy
 
 Production version can be built and run with following docker commands:

diff --git a/docker-compose-tls.yml b/docker-compose-tls.yml
@@ -16,34 +16,36 @@ services:
       - mockdoi
     restart: on-failure
     environment:
-      - "MONGO_HOST=database:27017"
-      - "MONGO_SSL=true"
-      - "MONGO_SSL_CA=/tls/cacert"
-      - "MONGO_SSL_CLIENT_KEY=/tls/key"
-      - "MONGO_SSL_CLIENT_CERT=/tls/cert"
-      - "AAI_CLIENT_SECRET=secret"
-      - "AAI_CLIENT_ID=aud2"
-      - "ISS_URL=http://mockauth:8000"
-      - "AUTH_URL=http://localhost:8000/authorize"
-      - "OIDC_URL=http://mockauth:8000"
+      - "MONGO_HOST=${MONGO_HOST}"
+      - "MONGO_SSL=${MONGO_SSL}"
+      - "MONGO_SSL_CA=${MONGO_SSL_CA}"
+      - "MONGO_SSL_CLIENT_KEY=${MONGO_SSL_CLIENT_KEY}"
+      - "MONGO_SSL_CLIENT_CERT=${MONGO_SSL_CLIENT_CERT}"
+      - "AAI_CLIENT_SECRET=${AAI_CLIENT_SECRET}"
+      - "AAI_CLIENT_ID=${AAI_CLIENT_ID}"
+      - "ISS_URL=${ISS_URL}"
+      - "AUTH_URL=${AUTH_URL}"
+      - "OIDC_URL=${OIDC_URL}"
+      - "AUTH_REFERER=${AUTH_REFERER}"
+      - "JWK_URL=${JWK_URL}"
+      - "BASE_URL=${BASE_URL}"
       # Enable this for working with front-end on localhost
-      # or change to http://frontend:3000 if started using docker-compose
-      # - "REDIRECT_URL=http://localhost:3000"
-      - "AUTH_REFERER=http://mockauth:8000"
-      - "JWK_URL=http://mockauth:8000/keyset"
-      - "LOG_LEVEL=DEBUG"
-      - "DOI_API=http://mockdoi:8001/dois"
-      - "DOI_PREFIX=10.xxxx"
-      - "DOI_USER=user"
-      - "DOI_KEY=key"
+      # - "REDIRECT_URL=${REDIRECT_URL}"
+      - "LOG_LEVEL=${LOG_LEVEL}"
+      - "MONGO_DATABASE=${MONGO_DATABASE}"
+      - "MONGO_AUTHDB=${MONGO_AUTHDB}"
+      - "DOI_API=${DOI_API}"
+      - "DOI_PREFIX=${DOI_PREFIX}"
+      - "DOI_USER=${DOI_USER}"
+      - "DOI_KEY=${DOI_KEY}"
   database:
     image: "mongo"
     container_name: "metadata_submitter_database_dev"
     command: "mongod --tlsMode=requireTLS --tlsCertificateKeyFile=/tls/combined2 --tlsCAFile=/tls/cacert"
     restart: on-failure
     environment:
-      - "MONGO_INITDB_ROOT_USERNAME=admin"
-      - "MONGO_INITDB_ROOT_PASSWORD=admin"
+      - "MONGO_INITDB_ROOT_USERNAME=${MONGO_INITDB_ROOT_USERNAME}"
+      - "MONGO_INITDB_ROOT_PASSWORD=${MONGO_INITDB_ROOT_PASSWORD}"
     volumes:
       - data:/data/db
       - ./config:/tls

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -15,31 +15,31 @@ services:
       - mockdoi
     restart: on-failure
     environment:
-      - "MONGO_HOST=database:27017"
-      - "AAI_CLIENT_SECRET=secret"
-      - "AAI_CLIENT_ID=aud2"
-      - "ISS_URL=http://mockauth:8000"
-      - "AUTH_URL=http://localhost:8000/authorize"
-      - "OIDC_URL=http://mockauth:8000"
+      - "MONGO_HOST=${MONGO_HOST}"
+      - "AAI_CLIENT_SECRET=${AAI_CLIENT_SECRET}"
+      - "AAI_CLIENT_ID=${AAI_CLIENT_ID}"
+      - "ISS_URL=${ISS_URL}"
+      - "AUTH_URL=${AUTH_URL}"
+      - "OIDC_URL=${OIDC_URL}"
+      - "AUTH_REFERER=${AUTH_REFERER}"
+      - "JWK_URL=${JWK_URL}"
+      - "BASE_URL=${BASE_URL}"
       # Enable this for working with front-end on localhost
-      # or change to http://frontend:3000 if started using docker-compose
-      # - "REDIRECT_URL=http://localhost:3000"
-      - "AUTH_REFERER=http://mockauth:8000"
-      - "JWK_URL=http://mockauth:8000/keyset"
-      - "LOG_LEVEL=DEBUG"
-      - "MONGO_DATABASE=default"
-      - "MONGO_AUTHDB=admin"
-      - "DOI_API=http://mockdoi:8001/dois"
-      - "DOI_PREFIX=10.xxxx"
-      - "DOI_USER=user"
-      - "DOI_KEY=key"
+      # - "REDIRECT_URL=${REDIRECT_URL}"
+      - "LOG_LEVEL=${LOG_LEVEL}"
+      - "MONGO_DATABASE=${MONGO_DATABASE}"
+      - "MONGO_AUTHDB=${MONGO_AUTHDB}"
+      - "DOI_API=${DOI_API}"
+      - "DOI_PREFIX=${DOI_PREFIX}"
+      - "DOI_USER=${DOI_USER}"
+      - "DOI_KEY=${DOI_KEY}"
   database:
     image: "mongo"
     container_name: "metadata_submitter_database_dev"
     restart: on-failure
     environment:
-      - "MONGO_INITDB_ROOT_USERNAME=admin"
-      - "MONGO_INITDB_ROOT_PASSWORD=admin"
+      - "MONGO_INITDB_ROOT_USERNAME=${MONGO_INITDB_ROOT_USERNAME}"
+      - "MONGO_INITDB_ROOT_PASSWORD=${MONGO_INITDB_ROOT_PASSWORD}"
     volumes:
       - data:/data/db
     expose:
@@ -59,7 +59,7 @@ services:
       - 8000:8000
     volumes:
       - ./tests/integration/mock_auth.py:/mock_auth.py
-    entrypoint: ["python", "/mock_auth.py", "0.0.0.0", "8000"]
+    entrypoint: [ "python", "/mock_auth.py", "0.0.0.0", "8000" ]
   mockdoi:
     build:
       dockerfile: Dockerfile-dev
@@ -73,6 +73,6 @@ services:
       - 8001:8001
     volumes:
       - ./tests/integration/mock_doi_api.py:/mock_doi_api.py
-    entrypoint: ["python", "/mock_doi_api.py", "0.0.0.0", "8001"]
+    entrypoint: [ "python", "/mock_doi_api.py", "0.0.0.0", "8001" ]
 volumes:
   data:
diff --git a/scripts/pre-commit.sh b/scripts/pre-commit.sh
@@ -3,7 +3,7 @@
 # Comment out pre-commit hooks you don't want to use
 
 echo "Running tox as a pre-commit hook"
-cd $(git rev-parse --show-toplevel) && rm -r .tox && tox -p auto
+cd $(git rev-parse --show-toplevel) && rm -r .tox; tox -p auto
 
 if [ $? -ne 0 ]; then
     echo "=============================="