Merge pull request #252 from CSCfi/develop

Release 0.11.0

.github/workflows/style.yml

@@ -28,3 +28,5 @@ jobs:
       run: tox -e flake8
     - name: Type hints check
       run: tox -e mypy
+    - name: Run bandit static code analysis
+      run: tox -e bandit

Dockerfile

@@ -9,7 +9,7 @@ ARG BRANCH=master
 RUN git clone -b ${BRANCH} https://github.com/CSCfi/metadata-submitter-frontend.git
 
 WORKDIR /metadata-submitter-frontend
-RUN npm install -g npm@7.20.0 \
+RUN npm install -g npm@7.21.0 \
     && npx --quiet pinst --disable \
     && npm install --production \
     && npm run build --production

docs/conf.py

@@ -11,7 +11,7 @@
 author = "CSC Developers"
 
 # The full version, including alpha/beta/rc tags
-release = "0.10.0"
+release = "0.11.0"
 
 
 # -- General configuration ---------------------------------------------------

docs/specification.yml

@@ -1290,6 +1290,8 @@ components:
         - published
         - metadataObjects
         - drafts
+        - doiInfo
+        - extraInfo
       additionalProperties: false
       properties:
         folderId:
@@ -1304,6 +1306,98 @@ components:
         published:
           type: boolean
           description: If folder is published or not
+        doiInfo:
+          type: object
+          required:
+            - creators
+            - titles
+            - subjects
+          properties:
+            creators:
+              type: array
+              items:
+                type: object
+                properties:
+                  name:
+                    type: string
+                    description: Full name
+                    example: "Last name, First name"
+                  nameType:
+                    type: string
+                    description: Type of name
+                    example: "Personal"
+                  givenName:
+                    type: string
+                    description: Official first name
+                    example: "First name"
+                  familyName:
+                    type: string
+                    description: Official last name
+                    example: "Last name"
+                  nameIdentifiers:
+                    type: array
+                    items:
+                      type: object
+                  affiliation:
+                    type: array
+                    items:
+                      type: object
+            subjects:
+              type: array
+              items:
+                type: object
+                properties:
+                  subject:
+                    type: string
+                    description: FOS defined subject name
+                    example: "FOS: field"
+                  subjectScheme:
+                    type: string
+                    description: Subject scheme name
+                    example: "Fields of Science and Technology (FOS)"
+        extraInfo:
+          type: object
+          required:
+            - identifier
+            - url
+            - resourceType
+            - publisher
+          properties:
+            identifier:
+              type: object
+              description: DOI id generated according to DOI recommendations
+              required:
+                - identifierType
+              properties:
+                identifierType:
+                  type: string
+                  example: "DOI"
+                doi:
+                  type: string
+                  description: Character string of DOI handle
+                  example: "prefix/suffix"
+                id:
+                  type: string
+                  description: Display url for DOI id
+                  example: "https://doi.org/prefix/suffix"
+            url:
+              type: string
+              description: URL of the digital location of the object
+            publisher:
+              type: string
+              description: Full name of publisher from Research Organization Registry
+            resourceType:
+              type: object
+              required:
+                - resourceTypeGeneral
+              properties:
+                resourceTypeGeneral:
+                  type: string
+                  description: Mandatory general type name
+                  example: "Dataset"
+                type:
+                  type: string
+                  description: Name of resource type
         metadataObjects:
           type: array
           items:

metadata_backend/__init__.py

@@ -1,5 +1,5 @@
 """Backend for submitting and validating XML Files containing ENA metadata."""
 
 __title__ = "metadata_backend"
-__version__ = VERSION = "0.10.0"
+__version__ = VERSION = "0.11.0"
 __author__ = "CSC Developers"

metadata_backend/api/handlers.py

@@ -13,6 +13,7 @@
 from motor.motor_asyncio import AsyncIOMotorClient
 from multidict import MultiDict, MultiDictProxy
 from xmlschema import XMLSchemaException
+from distutils.util import strtobool
 
 from .middlewares import decrypt_cookie, get_session
 
@@ -488,7 +489,7 @@ def _check_patch_folder(self, patch_ops: Any) -> None:
         """
         _required_paths = ["/name", "/description"]
         _required_values = ["schema", "accessionId"]
-        _arrays = ["/metadataObjects/-", "/drafts/-"]
+        _arrays = ["/metadataObjects/-", "/drafts/-", "/doiInfo"]
         _tags = re.compile("^/(metadataObjects|drafts)/[0-9]*/(tags)$")
 
         for op in patch_ops:
@@ -512,7 +513,7 @@ def _check_patch_folder(self, patch_ops: Any) -> None:
                     reason = f"{op['op']} on {op['path']}; replacing all objects is not allowed."
                     LOG.error(reason)
                     raise web.HTTPUnauthorized(reason=reason)
-                if op["path"] in _arrays:
+                if op["path"] in _arrays and op["path"] != "/doiInfo":
                     _ops = op["value"] if isinstance(op["value"], list) else [op["value"]]
                     for item in _ops:
                         if not all(key in item.keys() for key in _required_values):
@@ -547,11 +548,12 @@ async def get_folders(self, req: Request) -> Response:
         if "published" in req.query:
             pub_param = req.query.get("published", "").title()
             if pub_param in ["True", "False"]:
-                folder_query["published"] = {"$eq": eval(pub_param)}
+                folder_query["published"] = {"$eq": bool(strtobool(pub_param))}
             else:
                 reason = "'published' parameter must be either 'true' or 'false'"
                 LOG.error(reason)
                 raise web.HTTPBadRequest(reason=reason)
+
         folder_operator = FolderOperator(db_client)
         folders, total_folders = await folder_operator.query_folders(folder_query, page, per_page)
 
@@ -641,12 +643,19 @@ async def patch_folder(self, req: Request) -> Response:
         patch_ops = await self._get_data(req)
         self._check_patch_folder(patch_ops)
 
+        # Validate against folders schema if DOI is being added
+        for op in patch_ops:
+            if op["path"] == "/doiInfo":
+                curr_folder = await operator.read_folder(folder_id)
+                curr_folder["doiInfo"] = op["value"]
+                JSONValidator(curr_folder, "folders").validate
+
         await self._handle_check_ownedby_user(req, "folders", folder_id)
 
-        folder = await operator.update_folder(folder_id, patch_ops if isinstance(patch_ops, list) else [patch_ops])
+        upd_folder = await operator.update_folder(folder_id, patch_ops if isinstance(patch_ops, list) else [patch_ops])
 
-        body = json.dumps({"folderId": folder})
-        LOG.info(f"PATCH folder with ID {folder} was successful.")
+        body = json.dumps({"folderId": upd_folder})
+        LOG.info(f"PATCH folder with ID {upd_folder} was successful.")
         return web.Response(body=body, status=200, content_type="application/json")
 
     async def publish_folder(self, req: Request) -> Response:
@@ -780,17 +789,13 @@ async def get_user(self, req: Request) -> Response:
         if user_id != "current":
             LOG.info(f"User ID {user_id} was requested")
             raise web.HTTPUnauthorized(reason="Only current user retrieval is allowed")
-        db_client = req.app["db_client"]
-        operator = UserOperator(db_client)
 
         current_user = get_session(req)["user_info"]
-        user = await operator.read_user(current_user)
-        LOG.info(f"GET user with ID {user_id} was successful.")
 
         item_type = req.query.get("items", "").lower()
         if item_type:
             # Return only list of drafts or list of folder IDs owned by the user
-            result, link_headers = await self._get_user_items(req, user, item_type)
+            result, link_headers = await self._get_user_items(req, current_user, item_type)
             return web.Response(
                 body=json.dumps(result),
                 status=200,
@@ -799,6 +804,10 @@ async def get_user(self, req: Request) -> Response:
             )
         else:
             # Return whole user object if drafts or folders are not specified in query
+            db_client = req.app["db_client"]
+            operator = UserOperator(db_client)
+            user = await operator.read_user(current_user)
+            LOG.info(f"GET user with ID {user_id} was successful.")
             return web.Response(body=json.dumps(user), status=200, content_type="application/json")
 
     async def patch_user(self, req: Request) -> Response:
@@ -891,14 +900,14 @@ async def _get_user_items(self, req: Request, user: Dict, item_type: str) -> Tup
         page = self._get_page_param(req, "page", 1)
         per_page = self._get_page_param(req, "per_page", 5)
 
-        # Get the specific page of drafts
-        total_items = len(user[item_type])
-        if total_items <= per_page:
-            items = user[item_type]
-        else:
-            lower = (page - 1) * per_page
-            upper = page * per_page
-            items = user[item_type][lower:upper]
+        db_client = req.app["db_client"]
+        operator = UserOperator(db_client)
+        user_id = req.match_info["userId"]
+
+        query = {"userId": user}
+
+        items, total_items = await operator.filter_user(query, item_type, page, per_page)
+        LOG.info(f"GET user with ID {user_id} was successful.")
 
         result = {
             "page": {

metadata_backend/api/operators.py

@@ -4,6 +4,7 @@
 from datetime import datetime
 from typing import Any, Dict, List, Tuple, Union
 from uuid import uuid4
+import time
 
 from aiohttp import web
 from dateutil.relativedelta import relativedelta
@@ -658,6 +659,7 @@ async def create_folder(self, data: Dict) -> str:
         folder_id = self._generate_folder_id()
         data["folderId"] = folder_id
         data["published"] = False
+        data["dateCreated"] = int(time.time())
         data["metadataObjects"] = data["metadataObjects"] if "metadataObjects" in data else []
         data["drafts"] = data["drafts"] if "drafts" in data else []
         try:
@@ -675,25 +677,36 @@ async def create_folder(self, data: Dict) -> str:
             LOG.info(f"Inserting folder with id {folder_id} to database succeeded.")
             return folder_id
 
-    @auto_reconnect
-    async def query_folders(self, que: Dict, page_num: int, page_size: int) -> Tuple[List, int]:
+    async def query_folders(self, query: Dict, page_num: int, page_size: int) -> Tuple[List, int]:
         """Query database based on url query parameters.
 
-        :param que: Dict containing query information
+        :param query: Dict containing query information
         :param page_num: Page number
         :param page_size: Results per page
         :returns: Paginated query result
         """
-        _cursor = self.db_service.query("folder", que)
-        queried_folders = [folder async for folder in _cursor]
-        total_folders = len(queried_folders)
-        if total_folders <= page_size:
-            return queried_folders, total_folders
+        skips = page_size * (page_num - 1)
+        _query = [
+            {"$match": query},
+            {"$sort": {"dateCreated": -1}},
+            {"$skip": skips},
+            {"$limit": page_size},
+            {"$project": {"_id": 0}},
+        ]
+        data_raw = await self.db_service.do_aggregate("folder", _query)
+
+        if not data_raw:
+            data = []
         else:
-            lower = (page_num - 1) * page_size
-            upper = page_num * page_size
-            folders = queried_folders[lower:upper]
-            return folders, total_folders
+            data = [doc for doc in data_raw]
+
+        count_query = [{"$match": query}, {"$count": "total"}]
+        total_folders = await self.db_service.do_aggregate("folder", count_query)
+
+        if not total_folders:
+            total_folders = [{"total": 0}]
+
+        return data, total_folders[0]["total"]
 
     async def read_folder(self, folder_id: str) -> Dict:
         """Read object folder from database.
@@ -913,6 +926,45 @@ async def read_user(self, user_id: str) -> Dict:
             raise web.HTTPBadRequest(reason=reason)
         return user
 
+    async def filter_user(self, query: Dict, item_type: str, page_num: int, page_size: int) -> Tuple[List, int]:
+        """Query database based on url query parameters.
+
+        :param query: Dict containing query information
+        :param page_num: Page number
+        :param page_size: Results per page
+        :returns: Paginated query result
+        """
+        skips = page_size * (page_num - 1)
+        _query = [
+            {"$match": query},
+            {
+                "$project": {
+                    "_id": 0,
+                    item_type: {"$slice": [f"${item_type}", skips, page_size]},
+                }
+            },
+        ]
+        data = await self.db_service.do_aggregate("user", _query)
+
+        if not data:
+            data = [{item_type: []}]
+
+        count_query = [
+            {"$match": query},
+            {
+                "$project": {
+                    "_id": 0,
+                    "item": 1,
+                    "total": {
+                        "$cond": {"if": {"$isArray": f"${item_type}"}, "then": {"$size": f"${item_type}"}, "else": 0}
+                    },
+                }
+            },
+        ]
+        total_users = await self.db_service.do_aggregate("user", count_query)
+
+        return data[0][item_type], total_users[0]["total"]
+
     async def update_user(self, user_id: str, patch: List) -> str:
         """Update user object from database.
 
@@ -949,7 +1001,9 @@ async def assign_objects(self, user_id: str, collection: str, object_ids: List)
         """
         try:
             await self._check_user_exists(user_id)
-            assign_success = await self.db_service.append("user", user_id, {collection: {"$each": object_ids}})
+            assign_success = await self.db_service.append(
+                "user", user_id, {collection: {"$each": object_ids, "$position": 0}}
+            )
         except (ConnectionFailure, OperationFailure) as error:
             reason = f"Error happened while getting user: {error}"
             LOG.error(reason)

metadata_backend/database/db_service.py

@@ -198,7 +198,10 @@ async def append(self, collection: str, accession_id: str, data_to_be_addded: An
         """
         id_key = f"{collection}Id" if (collection in ["folder", "user"]) else "accessionId"
         find_by_id = {id_key: accession_id}
-        append_op = {"$addToSet": data_to_be_addded}
+        # push vs addtoSet
+        # push allows us to specify the postion but it does not check the items are unique
+        # addToSet cannot easily specify position
+        append_op = {"$push": data_to_be_addded}
         result = await self.database[collection].find_one_and_update(
             find_by_id, append_op, projection={"_id": False}, return_document=ReturnDocument.AFTER
         )