Feature/ted 90

.github/workflows/unit-tests-hermes.yml

@@ -0,0 +1,42 @@
+name: Tests on server
+
+on:
+  workflow_dispatch:
+  push:
+    branches: [ feature/* ]
+  pull_request:
+    branches: [ master ]
+env:
+  VAULT_TOKEN: ${{ secrets.VAULT_TOKEN }}
+  VAULT_ADDR: ${{ secrets.VAULT_ADDR }}
+  ENVIRONMENT: staging
+jobs:
+  build:
+    name: pyTest
+    runs-on: self-hosted
+
+    steps:
+      - uses: actions/checkout@v1
+
+      - name: Make envfile
+        run: make staging-dotenv-file
+
+      - name: Start staging infra
+        run: make start-project-services
+
+      - name: Run unit tests
+        run: make test-all
+
+#      - name: Clean Mongo DB
+#        run: make clean-mongo-db
+#      - name: Upload coverage to Codecov
+#        uses: codecov/codecov-action@v1
+#        with:
+#          token: ${{ secrets.CODECOV_TOKEN }}
+#          file: coverage.xml
+#          env_vars: OS,PYTHON
+#          name: codecov-umbrella
+#          fail_ci_if_error: true
+
+#      - name: start-infra
+#        run: make stop-project-staging-services

.gitignore

@@ -100,3 +100,4 @@ venv_activation.sh
 infra/traefik/letsencrypt/acme.json
 *.log
 infra/airflow/logs/scheduler/latest
+/.airflow/

Makefile

@@ -5,14 +5,13 @@ END_BUILD_PRINT = \e[0m
 CURRENT_UID := $(shell id -u)
 export CURRENT_UID
 # These are constants used for make targets so we can start prod and staging services on the same machine
-STAGING := staging
-PRODUCTION := prod
-PROD_ENV_FILE := .env
-STAGING_ENV_FILE := $(PROD_ENV_FILE).staging
+ENV_FILE := .env
 
 # include .env files if they exist
 -include .env
--include .env.staging
+
+PROJECT_PATH = $(shell pwd)
+AIRFLOW_INFRA_FOLDER ?= ${PROJECT_PATH}/.airflow
 
 #-----------------------------------------------------------------------------
 # Dev commands
@@ -50,27 +49,27 @@ test-all:
 	@ tox
 
 build-externals:
-	@ echo "$(BUILD_PRINT)Creating the necessary volumes, networks and folders and setting the special rights"
+	@ echo -e "$(BUILD_PRINT)Creating the necessary volumes, networks and folders and setting the special rights"
 	@ docker network create proxy-net || true
 
 #-----------------------------------------------------------------------------
 # SERVER SERVICES
 #-----------------------------------------------------------------------------
 start-traefik: build-externals
-	@ echo "$(BUILD_PRINT)Starting the Traefik services"
-	@ docker-compose --file ./infra/traefik/docker-compose.yml --env-file ${PROD_ENV_FILE} up -d
+	@ echo -e "$(BUILD_PRINT)Starting the Traefik services $(END_BUILD_PRINT)"
+	@ docker-compose -p common --file ./infra/traefik/docker-compose.yml --env-file ${ENV_FILE} up -d
 
 stop-traefik:
-	@ echo "$(BUILD_PRINT)Stopping the Traefik services"
-	@ docker-compose --file ./infra/traefik/docker-compose.yml --env-file ${PROD_ENV_FILE} down
+	@ echo -e "$(BUILD_PRINT)Stopping the Traefik services $(END_BUILD_PRINT)"
+	@ docker-compose -p common --file ./infra/traefik/docker-compose.yml --env-file ${ENV_FILE} down
 
 start-portainer: build-externals
-	@ echo "$(BUILD_PRINT)Starting the Portainer services"
-	@ docker-compose --file ./infra/portainer/docker-compose.yml --env-file ${PROD_ENV_FILE} up -d
+	@ echo -e "$(BUILD_PRINT)Starting the Portainer services $(END_BUILD_PRINT)"
+	@ docker-compose -p common --file ./infra/portainer/docker-compose.yml --env-file ${ENV_FILE} up -d
 
 stop-portainer:
-	@ echo "$(BUILD_PRINT)Stopping the Portainer services"
-	@ docker-compose --file ./infra/portainer/docker-compose.yml --env-file ${PROD_ENV_FILE} down
+	@ echo -e "$(BUILD_PRINT)Stopping the Portainer services $(END_BUILD_PRINT)"
+	@ docker-compose -p common --file ./infra/portainer/docker-compose.yml --env-file ${ENV_FILE} down
 
 start-server-services: | start-traefik start-portainer
 stop-server-services: | stop-traefik stop-portainer
@@ -79,130 +78,151 @@ stop-server-services: | stop-traefik stop-portainer
 # PROJECT SERVICES
 #-----------------------------------------------------------------------------
 create-env-airflow:
-	@ echo "$(BUILD_PRINT) Create Airflow env"
-	@ mkdir -p infra/airflow/logs infra/airflow/plugins
-	@ cd infra/airflow/ && ln -s -f ../../dags && ln -s -f ../../ted_sws
-	@ echo -e "AIRFLOW_UID=$(CURRENT_UID)" >infra/airflow/.env
-
+	@ echo -e "$(BUILD_PRINT) Create Airflow env $(END_BUILD_PRINT)"
+	@ echo -e "$(BUILD_PRINT) ${AIRFLOW_INFRA_FOLDER} ${ENVIRONMENT} $(END_BUILD_PRINT)"
+	@ mkdir -p ${AIRFLOW_INFRA_FOLDER}/logs ${AIRFLOW_INFRA_FOLDER}/plugins ${AIRFLOW_INFRA_FOLDER}/.env
+	@ ln -s -f ${PROJECT_PATH}/dags ${AIRFLOW_INFRA_FOLDER}/dags
+	@ ln -s -f ${PROJECT_PATH}/ted_sws ${AIRFLOW_INFRA_FOLDER}/ted_sws
+	@ chmod 777 ${AIRFLOW_INFRA_FOLDER}/logs ${AIRFLOW_INFRA_FOLDER}/plugins ${AIRFLOW_INFRA_FOLDER}/.env
+
+build-airflow: guard-ENVIRONMENT create-env-airflow build-externals
+	@ echo -e "$(BUILD_PRINT) Build Airflow services $(END_BUILD_PRINT)"
+	@ docker-compose -p ${ENVIRONMENT} --file ./infra/airflow/docker-compose.yaml --env-file ${ENV_FILE} build --no-cache --force-rm
+	@ docker-compose -p ${ENVIRONMENT} --file ./infra/airflow/docker-compose.yaml --env-file ${ENV_FILE} up -d --force-recreate
 
-build-airflow: create-env-airflow build-externals
-	@ echo "$(BUILD_PRINT) Build Airflow services"
-	@ docker-compose -p ${PRODUCTION} --file ./infra/airflow/docker-compose.yaml --env-file ${PROD_ENV_FILE} build --no-cache --force-rm
-	@ docker-compose -p ${PRODUCTION} --file ./infra/airflow/docker-compose.yaml --env-file ${PROD_ENV_FILE} up -d --force-recreate
 start-airflow: build-externals
-	@ echo "$(BUILD_PRINT)Starting Airflow servies"
-	@ docker-compose -p ${PRODUCTION} --file ./infra/airflow/docker-compose.yaml --env-file ${PROD_ENV_FILE} up -d
+	@ echo -e "$(BUILD_PRINT)Starting Airflow servies $(END_BUILD_PRINT)"
+	@ docker-compose -p ${ENVIRONMENT} --file ./infra/airflow/docker-compose.yaml --env-file ${ENV_FILE} up -d
+
 stop-airflow:
-	@ echo "$(BUILD_PRINT)Stoping Airflow services"
-	@ docker-compose -p ${PRODUCTION} --file ./infra/airflow/docker-compose.yaml --env-file ${PROD_ENV_FILE} down
+	@ echo -e "$(BUILD_PRINT)Stoping Airflow services $(END_BUILD_PRINT)"
+	@ docker-compose -p ${ENVIRONMENT} --file ./infra/airflow/docker-compose.yaml --env-file ${ENV_FILE} down
+
+#	------------------------
 start-allegro-graph: build-externals
-	@ echo "$(BUILD_PRINT)Starting Allegro-Graph servies"
-	@ docker-compose -p ${PRODUCTION} --file ./infra/allegro-graph/docker-compose.yml --env-file ${PROD_ENV_FILE} up -d
+	@ echo -e "$(BUILD_PRINT)Starting Allegro-Graph servies $(END_BUILD_PRINT)"
+	@ docker-compose -p ${ENVIRONMENT} --file ./infra/allegro-graph/docker-compose.yml --env-file ${ENV_FILE} up -d
+
 stop-allegro-graph:
-	@ echo "$(BUILD_PRINT)Stoping Allegro-Graph services"
-	@ docker-compose -p ${PRODUCTION} --file ./infra/allegro-graph/docker-compose.yml --env-file ${PROD_ENV_FILE} down
+	@ echo -e "$(BUILD_PRINT)Stoping Allegro-Graph services $(END_BUILD_PRINT)"
+	@ docker-compose -p ${ENVIRONMENT} --file ./infra/allegro-graph/docker-compose.yml --env-file ${ENV_FILE} down
+
+#	------------------------
 build-elasticsearch: build-externals
-	@ echo "$(BUILD_PRINT) Build Elasticsearch services"
-	@ docker-compose -p ${PRODUCTION} --file ./infra/elasticsearch/docker-compose.yml --env-file ${PROD_ENV_FILE} build --no-cache --force-rm
-	@ docker-compose -p ${PRODUCTION} --file ./infra/elasticsearch/docker-compose.yml --env-file ${PROD_ENV_FILE} up -d --force-recreate
+	@ echo -e "$(BUILD_PRINT) Build Elasticsearch services $(END_BUILD_PRINT)"
+	@ docker-compose -p ${ENVIRONMENT} --file ./infra/elasticsearch/docker-compose.yml --env-file ${ENV_FILE} build --no-cache --force-rm
+	@ docker-compose -p ${ENVIRONMENT} --file ./infra/elasticsearch/docker-compose.yml --env-file ${ENV_FILE} up -d --force-recreate
+
 start-elasticsearch: build-externals
-	@ echo "$(BUILD_PRINT)Starting the Elasticsearch services"
-	@ docker-compose -p ${PRODUCTION} --file ./infra/elasticsearch/docker-compose.yml --env-file ${PROD_ENV_FILE} up -d
+	@ echo -e "$(BUILD_PRINT)Starting the Elasticsearch services $(END_BUILD_PRINT)"
+	@ docker-compose -p ${ENVIRONMENT} --file ./infra/elasticsearch/docker-compose.yml --env-file ${ENV_FILE} up -d
+
 stop-elasticsearch:
-	@ echo "$(BUILD_PRINT)Stopping the Elasticsearch services"
-	@ docker-compose -p ${PRODUCTION} --file ./infra/elasticsearch/docker-compose.yml --env-file ${PROD_ENV_FILE} down
+	@ echo -e "$(BUILD_PRINT)Stopping the Elasticsearch services $(END_BUILD_PRINT)"
+	@ docker-compose -p ${ENVIRONMENT} --file ./infra/elasticsearch/docker-compose.yml --env-file ${ENV_FILE} down
 
 
 start-minio: build-externals
-	@ echo "$(BUILD_PRINT)Starting the Minio services"
-	@ docker-compose -p ${PRODUCTION} --file ./infra/minio/docker-compose.yml --env-file ${PROD_ENV_FILE} up -d
+	@ echo -e "$(BUILD_PRINT)Starting the Minio services $(END_BUILD_PRINT)"
+	@ docker-compose -p ${ENVIRONMENT} --file ./infra/minio/docker-compose.yml --env-file ${ENV_FILE} up -d
 
 stop-minio:
-	@ echo "$(BUILD_PRINT)Stopping the Minio services"
-	@ docker-compose -p ${PRODUCTION} --file ./infra/minio/docker-compose.yml --env-file ${PROD_ENV_FILE} down
+	@ echo -e "$(BUILD_PRINT)Stopping the Minio services $(END_BUILD_PRINT)"
+	@ docker-compose -p ${ENVIRONMENT} --file ./infra/minio/docker-compose.yml --env-file ${ENV_FILE} down
 
 
 start-mongo: build-externals
-	@ echo "$(BUILD_PRINT)Starting the Minio services"
-	@ docker-compose -p ${PRODUCTION} --file ./infra/mongo/docker-compose.yml --env-file ${PROD_ENV_FILE} up -d
+	@ echo -e "$(BUILD_PRINT)Starting the Mongo services $(END_BUILD_PRINT)"
+	@ docker-compose -p ${ENVIRONMENT} --file ./infra/mongo/docker-compose.yml --env-file ${ENV_FILE} up -d
 
 stop-mongo:
-	@ echo "$(BUILD_PRINT)Stopping the Minio services"
-	@ docker-compose -p ${PRODUCTION} --file ./infra/mongo/docker-compose.yml --env-file ${PROD_ENV_FILE} down
-
-
-start-project-prod-services: | start-airflow start-elasticsearch start-allegro-graph start-minio start-mongo
-stop-project-prod-services: | stop-airflow stop-elasticsearch stop-allegro-graph stop-minio stop-mongo
-
+	@ echo -e "$(BUILD_PRINT)Stopping the Mongo services $(END_BUILD_PRINT)"
+	@ docker-compose -p ${ENVIRONMENT} --file ./infra/mongo/docker-compose.yml --env-file ${ENV_FILE} down
 
-start-airflow-staging: build-externals
-	@ echo "$(BUILD_PRINT)Starting Airflow servies"
-	@ docker-compose -p ${STAGING} --file ./infra/airflow/docker-compose.yaml --env-file ${STAGING_ENV_FILE} up -d
 
-stop-airflow-staging:
-	@ echo "$(BUILD_PRINT)Stoping Airflow services"
-	@ docker-compose -p ${STAGING} --file ./infra/airflow/docker-compose.yaml --env-file ${STAGING_ENV_FILE} down
+start-project-services: | start-airflow start-mongo
+stop-project-services: | stop-airflow stop-mongo
 
-start-allegro-graph-staging: build-externals
-	@ echo "$(BUILD_PRINT)Starting Allegro-Graph servies"
-	@ docker-compose -p ${STAGING} --file ./infra/allegro-graph/docker-compose.yaml --env-file ${STAGING_ENV_FILE} up -d
-
-stop-allegro-graph-staging:
-	@ echo "$(BUILD_PRINT)Stoping Allegro-Graph services"
-	@ docker-compose -p ${STAGING} --file ./infra/allegro-graph/docker-compose.yaml --env-file ${STAGING_ENV_FILE} down
-
-start-elasticsearch-staging: build-externals
-	@ echo "$(BUILD_PRINT)Starting the Elasticsearch services"
-	@ docker-compose -p ${STAGING} --file ./infra/elasticsearch/docker-compose.yml --env-file ${STAGING_ENV_FILE} up -d
-
-stop-elasticsearch-staging:
-	@ echo "$(BUILD_PRINT)Stopping the Elasticsearch services"
-	@ docker-compose -p ${STAGING} --file ./infra/elasticsearch/docker-compose.yml --env-file ${STAGING_ENV_FILE} down
-
-
-start-minio-staging: build-externals
-	@ echo "$(BUILD_PRINT)Starting the Minio services"
-	@ docker-compose -p ${STAGING} --file ./infra/minio/docker-compose.yml --env-file ${STAGING_ENV_FILE} up -d
-
-stop-minio-staging:
-	@ echo "$(BUILD_PRINT)Stopping the Minio services"
-	@ docker-compose -p ${STAGING} --file ./infra/minio/docker-compose.yml --env-file ${STAGING_ENV_FILE} down
-
-start-mongo-staging: build-externals
-	@ echo "$(BUILD_PRINT)Starting the Minio services"
-	@ docker-compose -p ${STAGING} --file ./infra/mongo/docker-compose.yml --env-file ${STAGING_ENV_FILE} up -d
-
-stop-mongo-staging:
-	@ echo "$(BUILD_PRINT)Stopping the Minio services"
-	@ docker-compose -p ${STAGING} --file ./infra/mongo/docker-compose.yml --env-file ${STAGING_ENV_FILE} down
+#-----------------------------------------------------------------------------
+# VAULT SERVICES
+#-----------------------------------------------------------------------------
+# Testing whether an env variable is set or not
+guard-%:
+	@ if [ "${${*}}" = "" ]; then \
+        echo -e "$(BUILD_PRINT)Environment variable $* not set $(END_BUILD_PRINT)"; \
+        exit 1; \
+	fi
+
+# Testing that vault is installed
+vault-installed: #; @which vault1 > /dev/null
+	@ if ! hash vault 2>/dev/null; then \
+        echo -e "$(BUILD_PRINT)Vault is not installed, refer to https://www.vaultproject.io/downloads $(END_BUILD_PRINT)"; \
+        exit 1; \
+	fi
+# Get secrets in dotenv format
+staging-dotenv-file: guard-VAULT_ADDR guard-VAULT_TOKEN vault-installed
+	@ echo -e "$(BUILD_PRINT)Creating .env.staging file $(END_BUILD_PRINT)"
+	@ echo VAULT_ADDR=${VAULT_ADDR} > .env
+	@ echo VAULT_TOKEN=${VAULT_TOKEN} >> .env
+	@ echo DOMAIN=ted-data.eu >> .env
+	@ echo ENVIRONMENT=staging >> .env
+	@ echo SUBDOMAIN=staging. >> .env
+	@ echo AIRFLOW_INFRA_FOLDER=~/airflow-infra/staging >> .env
+	@ vault kv get -format="json" ted-staging/airflow | jq -r ".data.data | keys[] as \$$k | \"\(\$$k)=\(.[\$$k])\"" >> .env
+	@ vault kv get -format="json" ted-staging/mongo-db | jq -r ".data.data | keys[] as \$$k | \"\(\$$k)=\(.[\$$k])\"" >> .env
+
+dev-dotenv-file: guard-VAULT_ADDR guard-VAULT_TOKEN vault-installed
+	@ echo -e "$(BUILD_PRINT)Create .env file $(END_BUILD_PRINT)"
+	@ echo VAULT_ADDR=${VAULT_ADDR} > .env
+	@ echo VAULT_TOKEN=${VAULT_TOKEN} >> .env
+	@ echo DOMAIN=localhost >> .env
+	@ echo ENVIRONMENT=dev >> .env
+	@ echo SUBDOMAIN= >> .env
+	@ echo AIRFLOW_INFRA_FOLDER=${AIRFLOW_INFRA_FOLDER} >> .env
+	@ vault kv get -format="json" ted-dev/airflow | jq -r ".data.data | keys[] as \$$k | \"\(\$$k)=\(.[\$$k])\"" >> .env
+	@ vault kv get -format="json" ted-dev/mongo-db | jq -r ".data.data | keys[] as \$$k | \"\(\$$k)=\(.[\$$k])\"" >> .env
+
+
+prod-dotenv-file: guard-VAULT_ADDR guard-VAULT_TOKEN vault-installed
+	@ echo -e "$(BUILD_PRINT)Create .env file $(END_BUILD_PRINT)"
+	@ echo VAULT_ADDR=${VAULT_ADDR} > .env
+	@ echo VAULT_TOKEN=${VAULT_TOKEN} >> .env
+	@ echo DOMAIN=ted-data.eu >> .env
+	@ echo ENVIRONMENT=prod >> .env
+	@ echo SUBDOMAIN= >> .env
+	@ echo AIRFLOW_INFRA_FOLDER=~/airflow-infra/prod >> .env
+	@ vault kv get -format="json" ted-prod/airflow | jq -r ".data.data | keys[] as \$$k | \"\(\$$k)=\(.[\$$k])\"" >> .env
+	@ vault kv get -format="json" ted-prod/mongo-db | jq -r ".data.data | keys[] as \$$k | \"\(\$$k)=\(.[\$$k])\"" >> .env
+
+
+#clean-mongo-db:
+#	@ export PYTHONPATH=$(PWD) && python ./tests/clean_mongo_db.py
 
-start-project-staging-services: | start-airflow-staging start-elasticsearch-staging start-allegro-graph-staging start-minio-staging start-mongo-staging
-stop-project-staging-services: | stop-airflow-staging stop-elasticsearch-staging stop-allegro-graph-staging stop-minio-staging stop-mongo-staging
 
 #build-open-semantic-search:
-#	@ echo "Build open-semantic-search"
+#	@ echo -e "Build open-semantic-search"
 #	@ cd infra && rm -rf open-semantic-search
 #	@ cd infra && git clone --recurse-submodules --remote-submodules https://github.com/opensemanticsearch/open-semantic-search.git
 #	@ cd infra/open-semantic-search/ && ./build-deb
-#	@ echo "Patch open-semantic-search configs"
+#	@ echo -e "Patch open-semantic-search configs"
 #	@ cat infra/docker-compose-configs/open-semantic-search-compose-patch.yml > infra/open-semantic-search/docker-compose.yml
 #	@ cd infra/open-semantic-search/ && docker-compose rm -fsv
 #	@ cd infra/open-semantic-search/ && docker-compose build
 #
 #start-open-semantic-search:
-#	@ echo "Start open-semantic-search"
+#	@ echo -e "Start open-semantic-search"
 #	@ cd infra/open-semantic-search/ && docker-compose up -d
 #
 #
 #stop-open-semantic-search:
-#	@ echo "Stop open-semantic-search"
+#	@ echo -e "Stop open-semantic-search"
 #	@ cd infra/open-semantic-search/ && docker-compose down
 #
 #
 #start-silk-service:
-#	@ echo "Start silk service"
+#	@ echo -e "Start silk service"
 #	@ cd infra/silk/ && docker-compose up -d
 #
 #stop-silk-service:
-#	@ echo "Stop silk service"
+#	@ echo -e "Stop silk service"
 #	@ cd infra/silk/ && docker-compose down

infra/airflow/Dockerfile

@@ -1,4 +1,4 @@
-FROM apache/airflow:2.2.3-python3.8
+FROM apache/airflow:2.2.4-python3.8
 
 # quick sudo
 USER root
@@ -14,4 +14,4 @@ COPY requirements-airflow.txt /opt/airflow
 WORKDIR /opt/airflow
 
 RUN pip install --upgrade pip
-RUN pip install --no-cache-dir -r requirements-airflow.txt --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.2.3/constraints-no-providers-3.8.txt"
+RUN pip install --no-cache-dir -r requirements-airflow.txt --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.2.4/constraints-no-providers-3.8.txt"