Merge pull request #25 from geoadmin/develop

First Release

.dockerignore

@@ -2,20 +2,27 @@
 .git
 .gitignore
 .gitmodules
-**/*.md
+**/*.orig
 
 # Docker
 docker-compose.yml
 docker-compose.yml.in
 .docker
+Dockerfile
 
 # Python
-*.pyc
-__pycache__
+**/*.pyc
+**/__pycache__
 
 # Files not used within docker
-./tests/*
+./tests
 Makefile
 .venv
 lib-makefiles/*
 buildspec.yml
+minio.env
+.env.*
+.volumes
+.*
+service_launcher.py
+logs

.env.default

@@ -0,0 +1,9 @@
+AWS_ACCESS_KEY_ID=dummy-123
+AWS_SECRET_ACCESS_KEY=dummy-123
+AWS_S3_BUCKET_NAME=test-bucket
+AWS_S3_REGION_NAME=eu-central-1
+AWS_S3_ENDPOINT_URL=http://localhost:9090
+AWS_DB_REGION_NAME=eu-central-1
+AWS_DB_TABLE_NAME=test-db
+AWS_DB_ENDPOINT_URL=http://localhost:8080
+KML_STORAGE_HOST_URL=http://localhost:9090/test-bucket

.env.testing

@@ -0,0 +1,9 @@
+AWS_ACCESS_KEY_ID=dummy-123
+AWS_SECRET_ACCESS_KEY=dummy-123
+AWS_S3_BUCKET_NAME=test-bucket
+AWS_S3_REGION_NAME=wonderland
+# for some reason currently mocking the DynamoDB only works with a real region.
+# Fake regions will cause a crash when mocking.
+AWS_DB_REGION_NAME=us-east-1
+AWS_DB_TABLE_NAME=test-db
+ALLOWED_DOMAINS=.*\.geo\.admin\.ch

.github/workflows/pr-labeler.yml

@@ -0,0 +1,11 @@
+name: PR Auto Labeler
+
+on:
+  pull_request:
+    types:
+      - opened
+      - edited
+
+jobs:
+  pr-labeler:
+    uses: geoadmin/.github/.github/workflows/pr-labeler.yml@master

.github/workflows/semver.yml

@@ -0,0 +1,11 @@
+name: Bump SemVer Version
+
+on:
+  push:
+    branches:
+      - master
+      - develop
+
+jobs:
+  release:
+    uses: geoadmin/.github/.github/workflows/semver-release.yml@master

.gitignore

@@ -58,3 +58,10 @@ nose2-junit.xml
 
 # visual studio code config
 .vscode
+
+# .env files
+.env.local
+
+# files for local testing
+.volumes
+logs

.isort.cfg

@@ -2,5 +2,7 @@
 known_third_party=pytest
 known_defusedxml=defusedxml
 known_flask=flask
+known_boto3=boto3
+known_botocore=botocore
 force_single_line=True
-sections=FUTURE,STDLIB,THIRDPARTY,DEFUSEDXML,FLASK,FIRSTPARTY,LOCALFOLDER
+sections=FUTURE,STDLIB,THIRDPARTY,DEFUSEDXML,FLASK,BOTO3,BOTOCORE,FIRSTPARTY,LOCALFOLDER

.pylintrc

@@ -286,6 +286,7 @@ good-names=i,
            ex,
            fd,
            Run,
+           db,
            _
 
 # Good variable names regexes, separated by a comma. If names match any regex,
@@ -377,10 +378,10 @@ ignore-comments=yes
 ignore-docstrings=yes
 
 # Ignore imports when computing similarities.
-ignore-imports=no
+ignore-imports=yes
 
 # Minimum lines number of a similarity.
-min-similarity-lines=4
+min-similarity-lines=10
 
 
 [VARIABLES]

Dockerfile

@@ -1,7 +1,6 @@
 # Buster slim python 3.7 base image.
 FROM python:3.7-slim-buster
-ENV HTTP_PORT 8080
-RUN groupadd -r geoadmin && useradd -r -s /bin/false -g geoadmin geoadmin
+RUN groupadd -r geoadmin && useradd -u 1000 -r -s /bin/false -g geoadmin geoadmin
 
 
 # HERE : install relevant packages
@@ -20,6 +19,7 @@ ARG GIT_BRANCH=unknown
 ARG GIT_DIRTY=""
 ARG VERSION=unknown
 ARG AUTHOR=unknown
+ARG HTTP_PORT=5000
 LABEL git.hash=$GIT_HASH
 LABEL git.branch=$GIT_BRANCH
 LABEL git.dirty="$GIT_DIRTY"

Makefile

@@ -6,14 +6,25 @@ SERVICE_NAME := service-kml
 
 CURRENT_DIR := $(shell pwd)
 
+# Docker metadata
+GIT_HASH = `git rev-parse HEAD`
+GIT_HASH_SHORT = `git rev-parse --short HEAD`
+GIT_BRANCH = `git symbolic-ref HEAD --short 2>/dev/null`
+GIT_DIRTY = `git status --porcelain`
+GIT_TAG = `git describe --tags || echo "no version info"`
+AUTHOR = $(USER)
+
+
 # Test report configuration
 TEST_REPORT_DIR ?= $(CURRENT_DIR)/tests/report
 TEST_REPORT_FILE ?= nose2-junit.xml
 
 # general targets timestamps
 TIMESTAMPS = .timestamps
-REQUIREMENTS_TIMESTAMP = $(TIMESTAMPS)/.requirements.timestamp
-DEV_REQUIREMENTS_TIMESTAMP = $(TIMESTAMPS)/.dev-requirements.timestamps
+VOLUMES_MINIO = .volumes/minio
+LOGS_DIR = $(PWD)/logs
+REQUIREMENTS := $(TIMESTAMPS) $(VOLUMES_MINIO) $(LOGS_DIR) $(PIP_FILE) $(PIP_FILE_LOCK)
+
 
 # Docker variables
 DOCKER_REGISTRY = 974517877189.dkr.ecr.eu-central-1.amazonaws.com
@@ -30,7 +41,9 @@ PIP_FILE = Pipfile
 PIP_FILE_LOCK = Pipfile.lock
 
 # default configuration
+ENV_FILE ?= .env.default
 HTTP_PORT ?= 5000
+ROUTE_PREFIX ?= /api/kml
 
 # Commands
 PIPENV_RUN := pipenv run
@@ -42,12 +55,6 @@ ISORT := $(PIPENV_RUN) isort
 NOSE := $(PIPENV_RUN) nose2
 PYLINT := $(PIPENV_RUN) pylint
 
-# Docker metadata
-GIT_HASH := `git rev-parse HEAD`
-GIT_BRANCH := `git symbolic-ref HEAD --short 2>/dev/null`
-GIT_DIRTY := `git status --porcelain`
-GIT_TAG := `git describe --tags || echo "no version info"`
-AUTHOR := $(USER)
 
 
 all: help
@@ -75,39 +82,52 @@ help:
 	@echo "- dockerbuild        Build the project localy (with tag := $(DOCKER_IMG_LOCAL_TAG)) using the gunicorn WSGI server inside a container"
 	@echo "- dockerpush         Build and push the project localy (with tag := $(DOCKER_IMG_LOCAL_TAG))"
 	@echo "- dockerrun          Run the project using the gunicorn WSGI server inside a container (exposed port: 5000)"
-	@echo "- shutdown           Stop the aforementioned container"
 	@echo -e " \033[1mCLEANING TARGETS\033[0m "
 	@echo "- clean              Clean genereated files"
 	@echo "- clean_venv         Clean python venv"
+	@echo "- clean_logs         Clean logs"
+	@echo "- clean_vol_minio    Clean $(VOLUMES_MINIO)"
 
 
 # Build targets. Calling setup is all that is needed for the local files to be installed as needed.
 
 .PHONY: dev
-dev: $(DEV_REQUIREMENTS_TIMESTAMP)
+dev: $(REQUIREMENTS)
+	pipenv install --dev
 	pipenv shell
 
 
 .PHONY: setup
-setup: $(REQUIREMENTS_TIMESTAMP)
+setup: $(REQUIREMENTS)
+	pipenv install
 	pipenv shell
 
+
 .PHONY: ci
-ci: $(TIMESTAMPS) $(PIP_FILE) $(PIP_FILE_LOCK)
+ci: $(REQUIREMENTS)
 	# Create virtual env with all packages for development using the Pipfile.lock
 	pipenv sync --dev
 
 
 # linting target, calls upon yapf to make sure your code is easier to read and respects some conventions.
 
 .PHONY: format
-format: $(DEV_REQUIREMENTS_TIMESTAMP)
+format:
 	$(YAPF) -p -i --style .style.yapf $(PYTHON_FILES)
 	$(ISORT) $(PYTHON_FILES)
 
 
+.PHONY: ci-check-format
+ci-check-format: format
+	@if [[ -n `git status --porcelain` ]]; then \
+	 	>&2 echo "ERROR: the following files are not formatted correctly:"; \
+		>&2 git status --porcelain; \
+		exit 1; \
+	fi
+
+
 .PHONY: lint
-lint: $(DEV_REQUIREMENTS_TIMESTAMP)
+lint:
 	$(PYLINT) $(PYTHON_FILES)
 
 
@@ -118,21 +138,21 @@ format-lint: format lint
 # Test target
 
 .PHONY: test
-test: $(DEV_REQUIREMENTS_TIMESTAMP)
+test:
 	mkdir -p $(TEST_REPORT_DIR)
-	$(NOSE) -c tests/unittest.cfg --verbose --junit-xml-path $(TEST_REPORT_DIR)/$(TEST_REPORT_FILE) -s tests/
+	ENV_FILE=.env.testing $(NOSE) -c tests/unittest.cfg --verbose --junit-xml-path $(TEST_REPORT_DIR)/$(TEST_REPORT_FILE) -s tests/
 
 
 # Serve targets. Using these will run the application on your local machine. You can either serve with a wsgi front (like it would be within the container), or without.
 
 .PHONY: serve
-serve: $(REQUIREMENTS_TIMESTAMP)
-	FLASK_APP=service_launcher FLASK_DEBUG=1 $(FLASK) run --host=0.0.0.0 --port=$(HTTP_PORT)
+serve: clean_logs $(LOGS_DIR)
+	ENV_FILE=$(ENV_FILE) LOGS_DIR=$(LOGS_DIR) FLASK_APP=service_launcher.py FLASK_DEBUG=1 $(FLASK) run --host=0.0.0.0 --port=$(HTTP_PORT)
 
 
 .PHONY: gunicornserve
-gunicornserve: $(REQUIREMENTS_TIMESTAMP)
-	$(PYTHON) wsgi.py
+gunicornserve: clean_logs $(LOGS_DIR)
+	SCRIPT_NAME=$(ROUTE_PREFIX) ENV_FILE=$(ENV_FILE) LOGS_DIR=$(LOGS_DIR) $(PYTHON) wsgi.py
 
 
 # Docker related functions.
@@ -141,13 +161,15 @@ gunicornserve: $(REQUIREMENTS_TIMESTAMP)
 dockerlogin:
 	aws --profile swisstopo-bgdi-builder ecr get-login-password --region $(AWS_DEFAULT_REGION) | docker login --username AWS --password-stdin $(DOCKER_REGISTRY)
 
+
 .PHONY: dockerbuild
-dockerbuild:
+dockerbuild: $(VOLUMES_MINIO)
 	docker build \
 		--build-arg GIT_HASH="$(GIT_HASH)" \
 		--build-arg GIT_BRANCH="$(GIT_BRANCH)" \
 		--build-arg GIT_DIRTY="$(GIT_DIRTY)" \
 		--build-arg VERSION="$(GIT_TAG)" \
+		--build-arg HTTP_PORT="$(HTTP_PORT)" \
 		--build-arg AUTHOR="$(AUTHOR)" -t $(DOCKER_IMG_LOCAL_TAG) .
 
 
@@ -157,22 +179,33 @@ dockerpush: dockerbuild
 
 
 .PHONY: dockerrun
-dockerrun: dockerbuild
-	HTTP_PORT=$(HTTP_PORT) docker-compose up
+dockerrun: clean_logs dockerbuild $(LOGS_DIR)
+	docker run \
+		-it --rm --net=host \
+		--env-file=${PWD}/${ENV_FILE} \
+		--env LOGS_DIR=/logs \
+		--env SCRIPT_NAME=$(ROUTE_PREFIX) \
+		--mount type=bind,source="${LOGS_DIR}",target=/logs \
+		$(DOCKER_IMG_LOCAL_TAG)
 
 
-.PHONY: shutdown
-shutdown:
-	HTTP_PORT=$(HTTP_PORT) SERVICE_NAME=$(SERVICE_NAME) docker-compose down
+ # Clean targets
 
+.PHONY: clean_logs
+clean_logs:
+	rm -rf $(LOGS_DIR)
 
 .PHONY: clean_venv
 clean_venv:
 	pipenv --rm
 
+.PHONY: clean_vol_minio
+clean_volumes_minio:
+	rm -rf $(VOLUMES_MINIO)
+
 
 .PHONY: clean
-clean: clean_venv
+clean: clean_venv clean_logs clean_vol_minio
 	@# clean python cache files
 	find . -name __pycache__ -type d -print0 | xargs -I {} -0 rm -rf "{}"
 	rm -rf $(PYTHON_LOCAL_DIR)
@@ -185,11 +218,10 @@ clean: clean_venv
 $(TIMESTAMPS):
 	mkdir -p $(TIMESTAMPS)
 
-$(REQUIREMENTS_TIMESTAMP): $(TIMESTAMPS) $(PIP_FILE) $(PIP_FILE_LOCK)
-	pipenv install
-	@touch $(REQUIREMENTS_TIMESTAMP)
 
+$(VOLUMES_MINIO):
+	mkdir -p $(VOLUMES_MINIO)
 
-$(DEV_REQUIREMENTS_TIMESTAMP): $(TIMESTAMPS) $(PIP_FILE) $(PIP_FILE_LOCK)
-	pipenv install --dev
-	@touch $(DEV_REQUIREMENTS_TIMESTAMP)
+
+$(LOGS_DIR):
+	mkdir -p -m=777 $(LOGS_DIR)

Pipfile

@@ -9,9 +9,12 @@ PyYAML = ">=5.4"
 gevent = "~=20.9.0"
 gunicorn = "~=19.9.0"
 Flask = "~=1.1.1"
-Pillow = ">=8.1.2"
 logging-utilities = "~=1.2.1"
 defusedxml = "~=0.7.1"
+boto3 = "~=1.17.60"
+botocore = "~=1.20.98"
+moto = {extras = [ "s3",], version = "*"}
+python-dotenv = "~=0.17.0"
 
 [dev-packages]
 yapf = "*"