Merge pull request #66 from geoadmin/develop

New Release v2.0.0 - #major

.pylintrc

@@ -279,6 +279,10 @@ good-names=i,
            Run,
            db,
            assertKml,
+           assertKmlMetadata,
+           assertKmlFile,
+           assertKmlInDb,
+           assertKmlDbData,
            _
 
 # Good variable names regexes, separated by a comma. If names match any regex,

Makefile

@@ -145,6 +145,7 @@ serve: clean_logs $(LOGS_DIR)
 
 .PHONY: gunicornserve
 gunicornserve: clean_logs $(LOGS_DIR)
+	mkdir -p /tmp/gunicorn_workers
 	SCRIPT_NAME=$(ROUTE_PREFIX) ENV_FILE=$(ENV_FILE) LOGS_DIR=$(LOGS_DIR) $(PYTHON) wsgi.py
 
 

README.md

@@ -216,3 +216,4 @@ The service is configured by Environment Variable:
 | SCRIPT_NAME | `''` | If the service is behind a reverse proxy and not served at the root, the route prefix must be set in `SCRIPT_NAME`. |
 | CACHE_CONTROL | `no-cache, no-store, must-revalidate` | Cache Control header value of the GET endpoint(s) |
 | CACHE_CONTROL_4XX | `public, max-age=3600` | Cache Control header for 4XX responses |
+| GUNICORN_WORKER_TMP_DIR | `/tmp/gunicorn_workers` | Gunicorn worker tmp directory. :warning: This directory should be on **TMPFS** for better performance. |

app/helpers/dynamodb.py

@@ -43,28 +43,29 @@ def __init__(self, table_name, bucket_name, endpoint_url, table_region):
         self.endpoint = endpoint_url
 
     def save_item(
-        self, kml_id, kml_admin_id, file_key, file_length, timestamp, empty=False, author=''
+        self, kml_id, kml_admin_id, file_key, file_length, timestamp, author, author_version, empty
     ):
         logger.debug('Saving dynamodb item with primary key %s', kml_id)
+        db_item = {
+            'kml_id': kml_id,
+            'admin_id': kml_admin_id,
+            'created': timestamp,
+            'updated': timestamp,
+            'bucket': self.bucket_name,
+            'file_key': file_key,
+            'empty': empty,
+            'length': file_length,
+            'encoding': KML_FILE_CONTENT_ENCODING,
+            'content_type': KML_FILE_CONTENT_TYPE,
+            'author': author,
+            'author_version': author_version
+        }
         try:
-            self.table.put_item(
-                Item={
-                    'kml_id': kml_id,
-                    'admin_id': kml_admin_id,
-                    'created': timestamp,
-                    'updated': timestamp,
-                    'bucket': self.bucket_name,
-                    'file_key': file_key,
-                    'empty': empty,
-                    'length': file_length,
-                    'encoding': KML_FILE_CONTENT_ENCODING,
-                    'content_type': KML_FILE_CONTENT_TYPE,
-                    'author': author
-                }
-            )
+            self.table.put_item(Item=db_item)
         except EndpointConnectionError as error:
             logger.exception('Failed to connect to DynamoDB: %s', error)
             abort(502, 'Backend DB connection error, please consult logs')
+        return db_item
 
     def get_item(self, kml_id):
         logger.debug('Get dynamodb item with primary key %s', kml_id)
@@ -106,27 +107,33 @@ def get_item_by_admin_id(self, admin_id):
 
         return items[0]
 
-    def update_item(self, kml_id, file_length, timestamp, empty):
+    def update_item(self, kml_id, db_item, file_length, timestamp, empty, author_version=None):
         logger.debug('Updating dynamodb item with primary key %s', kml_id)
+        db_item['updated'] = timestamp
+        db_item['empty'] = empty
+        db_item['length'] = file_length
+        attribute_updates = {
+            'updated': {
+                'Value': timestamp, 'Action': 'PUT'
+            },
+            'empty': {
+                'Value': empty, 'Action': 'PUT'
+            },
+            'length': {
+                'Value': file_length, 'Action': 'PUT'
+            }
+        }
+        if author_version is not None:
+            attribute_updates['author_version'] = {'Value': author_version, 'Action': 'PUT'}
+            db_item['author_version'] = author_version
         try:
-            self.table.update_item(
-                Key={'kml_id': kml_id},
-                AttributeUpdates={
-                    'updated': {
-                        'Value': timestamp, 'Action': 'PUT'
-                    },
-                    'empty': {
-                        'Value': empty, 'Action': 'PUT'
-                    },
-                    'length': {
-                        'Value': file_length, 'Action': 'PUT'
-                    }
-                }
-            )
+            self.table.update_item(Key={'kml_id': kml_id}, AttributeUpdates=attribute_updates)
         except EndpointConnectionError as error:
             logger.exception('Failed to connect to DynamoDB: %s', error)
             abort(502, 'Backend DB connection error, please consult logs')
 
+        return db_item
+
     def delete_item(self, kml_id):
         logger.debug('Deleting dynamodb item with primary key %s', kml_id)
         try:

app/helpers/utils.py

@@ -16,7 +16,9 @@
 from flask import jsonify
 from flask import make_response
 from flask import request
+from flask.helpers import url_for
 
+from app.settings import DEFAULT_AUTHOR_VERSION
 from app.settings import KML_FILE_CONTENT_TYPE
 from app.settings import KML_MAX_SIZE
 from app.settings import KML_STORAGE_HOST_URL
@@ -179,6 +181,13 @@ def validate_kml_file():
     return gzip_string(kml_string), empty
 
 
+def validate_author():
+    author = request.form.get('author', None)
+    if author is None:
+        abort(400, "Missing author field")
+    return author
+
+
 def get_kml_file_link(file_key):
     if KML_STORAGE_HOST_URL:
         return f'{KML_STORAGE_HOST_URL}/{file_key}'
@@ -228,3 +237,24 @@ def decompress_if_gzipped(file_content):
             logger.error("Error when trying to decompress kml file: %s", error)
             raise error
     return ret
+
+
+def get_json_metadata(db_item, with_admin_id=False):
+    '''Return a json metadata output of a DB entry'''
+    metadata = {
+        'id': db_item['kml_id'],
+        'success': True,
+        'created': db_item['created'],
+        'updated': db_item['updated'],
+        'empty': db_item['empty'],
+        'author': db_item['author'],
+        'author_version': db_item.get('author_version', DEFAULT_AUTHOR_VERSION),
+        'links':
+            {
+                'self': url_for('get_kml_metadata', kml_id=db_item['kml_id'], _external=True),
+                'kml': get_kml_file_link(db_item['file_key']),
+            }
+    }
+    if with_admin_id:
+        metadata['admin_id'] = db_item['admin_id']
+    return metadata

app/routes.py

@@ -8,16 +8,17 @@
 from flask import jsonify
 from flask import make_response
 from flask import request
-from flask.helpers import url_for
 
 from app import app
 from app.helpers.dynamodb import get_db
 from app.helpers.s3 import get_storage
-from app.helpers.utils import get_kml_file_link
+from app.helpers.utils import get_json_metadata
+from app.helpers.utils import validate_author
 from app.helpers.utils import validate_content_length
 from app.helpers.utils import validate_content_type
 from app.helpers.utils import validate_kml_file
 from app.helpers.utils import validate_permissions
+from app.settings import DEFAULT_AUTHOR_VERSION
 from app.settings import SCRIPT_NAME
 from app.version import APP_VERSION
 
@@ -39,7 +40,9 @@ def create_kml():
     # Get the kml file data
     kml_string_gzip, empty = validate_kml_file()
     # Get the author
-    author = request.form.get('author', 'unknown')
+    author = validate_author()
+    # Get the client version
+    author_version = request.form.get('author_version', DEFAULT_AUTHOR_VERSION)
 
     kml_admin_id = urlsafe_b64encode(uuid4().bytes).decode('utf8').replace('=', '')
     kml_id = urlsafe_b64encode(uuid4().bytes).decode('utf8').replace('=', '')
@@ -50,75 +53,34 @@ def create_kml():
     storage.upload_object_to_bucket(file_key, kml_string_gzip)
 
     db = get_db()
-    db.save_item(kml_id, kml_admin_id, file_key, len(kml_string_gzip), timestamp, empty, author)
-
-    return make_response(
-        jsonify(
-            {
-                'id': kml_id,
-                'admin_id': kml_admin_id,
-                'success': True,
-                'created': timestamp,
-                'updated': timestamp,
-                'empty': empty,
-                'links':
-                    {
-                        'self': url_for('get_kml_metadata', kml_id=kml_id, _external=True),
-                        'kml': get_kml_file_link(file_key),
-                    }
-            }
-        ),
-        201
+    db_item = db.save_item(
+        kml_id,
+        kml_admin_id,
+        file_key,
+        len(kml_string_gzip),
+        timestamp,
+        author,
+        author_version,
+        empty
     )
 
+    return make_response(jsonify(get_json_metadata(db_item, with_admin_id=True)), 201)
+
 
 @app.route('/admin', methods=['GET'])
 def get_kml_metadata_by_admin_id():
     admin_id = request.args.get('admin_id')
     if not admin_id:
         logger.error("Query parameter admin_id is required: query=%s", request.args)
         abort(400, "Query parameter admin_id is required")
-    item = get_db().get_item_by_admin_id(admin_id)
-    return make_response(
-        jsonify(
-            {
-                'id': item['kml_id'],
-                'admin_id': admin_id,
-                'success': True,
-                'created': item['created'],
-                'updated': item['updated'],
-                'empty': item['empty'],
-                'links':
-                    {
-                        'self': url_for('get_kml_metadata', kml_id=item['kml_id'], _external=True),
-                        'kml': get_kml_file_link(item['file_key']),
-                    }
-            }
-        ),
-        200
-    )
+    db_item = get_db().get_item_by_admin_id(admin_id)
+    return make_response(jsonify(get_json_metadata(db_item, with_admin_id=True)), 200)
 
 
 @app.route('/admin/<kml_id>', methods=['GET'])
 def get_kml_metadata(kml_id):
-    item = get_db().get_item(kml_id)
-    return make_response(
-        jsonify(
-            {
-                'id': kml_id,
-                'success': True,
-                'created': item['created'],
-                'updated': item['updated'],
-                'empty': item['empty'],
-                'links':
-                    {
-                        'self': url_for('get_kml_metadata', kml_id=kml_id, _external=True),
-                        'kml': get_kml_file_link(item['file_key']),
-                    }
-            }
-        ),
-        200
-    )
+    db_item = get_db().get_item(kml_id)
+    return make_response(jsonify(get_json_metadata(db_item, with_admin_id=False)), 200)
 
 
 @app.route('/admin/<kml_id>', methods=['PUT'])
@@ -127,37 +89,25 @@ def get_kml_metadata(kml_id):
 def update_kml(kml_id):
     db = get_db()
 
-    item = db.get_item(kml_id)
-    admin_id = validate_permissions(item)
+    db_item = db.get_item(kml_id)
+    admin_id = validate_permissions(db_item)
+
+    # Get the client version
+    author_version = request.form.get('author_version', None)
 
     # Get the kml file data
     kml_string_gzip, empty = validate_kml_file()
 
     storage = get_storage()
-    storage.upload_object_to_bucket(item['file_key'], kml_string_gzip)
+    storage.upload_object_to_bucket(db_item['file_key'], kml_string_gzip)
 
     timestamp = datetime.utcnow().replace(tzinfo=timezone.utc).isoformat(timespec='milliseconds')
-    db.update_item(kml_id, len(kml_string_gzip), timestamp, empty)
-
-    return make_response(
-        jsonify(
-            {
-                'id': kml_id,
-                'admin_id': admin_id,
-                'success': True,
-                'created': item['created'],
-                'updated': timestamp,
-                'empty': empty,
-                'links':
-                    {
-                        'self': url_for('get_kml_metadata', kml_id=kml_id, _external=True),
-                        'kml': get_kml_file_link(item['file_key']),
-                    }
-            }
-        ),
-        200
+    db_item = db.update_item(
+        kml_id, db_item, len(kml_string_gzip), timestamp, empty, author_version
     )
 
+    return make_response(jsonify(get_json_metadata(db_item, with_admin_id=True)), 200)
+
 
 @app.route('/admin/<kml_id>', methods=['DELETE'])
 @validate_content_type("multipart/form-data")

app/settings.py

@@ -34,3 +34,5 @@
 
 CACHE_CONTROL = os.getenv('CACHE_CONTROL', 'no-cache, no-store, must-revalidate')
 CACHE_CONTROL_4XX = os.getenv('CACHE_CONTROL_4XX', 'public, max-age=3600')
+
+DEFAULT_AUTHOR_VERSION = '0.0.0'