From 2b141183069e181c06693b1bc7500f941b425886 Mon Sep 17 00:00:00 2001
From: Stefan Negru <stefan@blankdots.com>
Date: Thu, 12 Aug 2021 21:42:40 +0300
Subject: [PATCH 01/25] create indexes to query more effectively

add datecreated to folder to easily sort by timestamp
---
 metadata_backend/conf/conf.py                 | 23 +++++++++++++++++--
 metadata_backend/helpers/schemas/folders.json |  4 ++++
 metadata_backend/server.py                    |  2 +-
 3 files changed, 26 insertions(+), 3 deletions(-)

diff --git a/metadata_backend/conf/conf.py b/metadata_backend/conf/conf.py
index ebe4e0b35..816985a79 100644
--- a/metadata_backend/conf/conf.py
+++ b/metadata_backend/conf/conf.py
@@ -94,13 +94,32 @@ def set_conf() -> Tuple[str, str]:
 connectTimeout = 15000
 
 
-def create_db_client() -> AsyncIOMotorClient:
+async def create_db_client() -> AsyncIOMotorClient:
     """Initialize database client for AioHTTP App.
 
     :returns: Coroutine-based Motor client for Mongo operations
     """
     LOG.debug("initialised DB client")
-    return AsyncIOMotorClient(url, connectTimeoutMS=connectTimeout, serverSelectionTimeoutMS=serverTimeout)
+    client = AsyncIOMotorClient(url, connectTimeoutMS=connectTimeout, serverSelectionTimeoutMS=serverTimeout)
+    await create_indexes(client, mongo_database)
+    return client
+
+
+async def create_indexes(client: AsyncIOMotorClient, mongo_database: str) -> None:
+    """Create Indexes for collections."""
+    db = client[mongo_database]
+    collections = await db.list_collection_names()
+
+    if "folder" not in collections:
+        await db.create_collection("folder")
+
+    db.folder.create_index([("dateCreated", -1)])
+    db.folder.create_index([("folderId", 1)], unique=True)
+
+    if "user" not in collections:
+        await db.create_collection("user")
+
+    db.user.create_index([("userId", 1)], unique=True)
 
 
 # 2) Load schema types and descriptions from json
diff --git a/metadata_backend/helpers/schemas/folders.json b/metadata_backend/helpers/schemas/folders.json
index a78126df6..509e03bde 100644
--- a/metadata_backend/helpers/schemas/folders.json
+++ b/metadata_backend/helpers/schemas/folders.json
@@ -18,6 +18,10 @@
       "type": "string",
       "title": "Folder Description"
     },
+    "dateCreated": {
+      "type": "int",
+      "title": "Unix time stamp of creation, used for indexing"
+    },
     "published": {
       "type": "boolean",
       "title": "Published Folder"
diff --git a/metadata_backend/server.py b/metadata_backend/server.py
index 49de9e973..8eb982c4a 100644
--- a/metadata_backend/server.py
+++ b/metadata_backend/server.py
@@ -119,7 +119,7 @@ async def init() -> web.Application:
         ]
         server.router.add_routes(frontend_routes)
         LOG.info("Frontend routes loaded")
-    server["db_client"] = create_db_client()
+    server["db_client"] = await create_db_client()
     server.on_shutdown.append(kill_sess_on_shutdown)
     LOG.info("Database client loaded")
     return server

From 0f59c451e6077291e55efc6346af3751fdaef928 Mon Sep 17 00:00:00 2001
From: Stefan Negru <stefan@blankdots.com>
Date: Thu, 12 Aug 2021 21:43:04 +0300
Subject: [PATCH 02/25] filter the data in mongo not after retrieving it

---
 metadata_backend/api/handlers.py  | 30 ++++++------
 metadata_backend/api/operators.py | 80 ++++++++++++++++++++++++++-----
 tests/test_handlers.py            | 16 ++++---
 tests/test_operators.py           |  8 ++--
 4 files changed, 97 insertions(+), 37 deletions(-)

diff --git a/metadata_backend/api/handlers.py b/metadata_backend/api/handlers.py
index e09a621d5..fb109c7a5 100644
--- a/metadata_backend/api/handlers.py
+++ b/metadata_backend/api/handlers.py
@@ -13,6 +13,7 @@
 from motor.motor_asyncio import AsyncIOMotorClient
 from multidict import MultiDict, MultiDictProxy
 from xmlschema import XMLSchemaException
+from distutils.util import strtobool
 
 from .middlewares import decrypt_cookie, get_session
 
@@ -547,11 +548,12 @@ async def get_folders(self, req: Request) -> Response:
         if "published" in req.query:
             pub_param = req.query.get("published", "").title()
             if pub_param in ["True", "False"]:
-                folder_query["published"] = {"$eq": eval(pub_param)}
+                folder_query["published"] = {"$eq": bool(strtobool(pub_param))}
             else:
                 reason = "'published' parameter must be either 'true' or 'false'"
                 LOG.error(reason)
                 raise web.HTTPBadRequest(reason=reason)
+
         folder_operator = FolderOperator(db_client)
         folders, total_folders = await folder_operator.query_folders(folder_query, page, per_page)
 
@@ -780,17 +782,13 @@ async def get_user(self, req: Request) -> Response:
         if user_id != "current":
             LOG.info(f"User ID {user_id} was requested")
             raise web.HTTPUnauthorized(reason="Only current user retrieval is allowed")
-        db_client = req.app["db_client"]
-        operator = UserOperator(db_client)
 
         current_user = get_session(req)["user_info"]
-        user = await operator.read_user(current_user)
-        LOG.info(f"GET user with ID {user_id} was successful.")
 
         item_type = req.query.get("items", "").lower()
         if item_type:
             # Return only list of drafts or list of folder IDs owned by the user
-            result, link_headers = await self._get_user_items(req, user, item_type)
+            result, link_headers = await self._get_user_items(req, current_user, item_type)
             return web.Response(
                 body=json.dumps(result),
                 status=200,
@@ -799,6 +797,10 @@ async def get_user(self, req: Request) -> Response:
             )
         else:
             # Return whole user object if drafts or folders are not specified in query
+            db_client = req.app["db_client"]
+            operator = UserOperator(db_client)
+            user = await operator.read_user(current_user)
+            LOG.info(f"GET user with ID {user_id} was successful.")
             return web.Response(body=json.dumps(user), status=200, content_type="application/json")
 
     async def patch_user(self, req: Request) -> Response:
@@ -891,14 +893,14 @@ async def _get_user_items(self, req: Request, user: Dict, item_type: str) -> Tup
         page = self._get_page_param(req, "page", 1)
         per_page = self._get_page_param(req, "per_page", 5)
 
-        # Get the specific page of drafts
-        total_items = len(user[item_type])
-        if total_items <= per_page:
-            items = user[item_type]
-        else:
-            lower = (page - 1) * per_page
-            upper = page * per_page
-            items = user[item_type][lower:upper]
+        db_client = req.app["db_client"]
+        operator = UserOperator(db_client)
+        user_id = req.match_info["userId"]
+
+        query = {"userId": user}
+
+        items, total_items = await operator.filter_user(query, item_type, page, per_page)
+        LOG.info(f"GET user with ID {user_id} was successful.")
 
         result = {
             "page": {
diff --git a/metadata_backend/api/operators.py b/metadata_backend/api/operators.py
index b5cea974c..9d5924b3a 100644
--- a/metadata_backend/api/operators.py
+++ b/metadata_backend/api/operators.py
@@ -4,6 +4,7 @@
 from datetime import datetime
 from typing import Any, Dict, List, Tuple, Union
 from uuid import uuid4
+import time
 
 from aiohttp import web
 from dateutil.relativedelta import relativedelta
@@ -658,6 +659,7 @@ async def create_folder(self, data: Dict) -> str:
         folder_id = self._generate_folder_id()
         data["folderId"] = folder_id
         data["published"] = False
+        data["dateCreated"] = int(time.time())
         data["metadataObjects"] = data["metadataObjects"] if "metadataObjects" in data else []
         data["drafts"] = data["drafts"] if "drafts" in data else []
         try:
@@ -675,25 +677,36 @@ async def create_folder(self, data: Dict) -> str:
             LOG.info(f"Inserting folder with id {folder_id} to database succeeded.")
             return folder_id
 
-    @auto_reconnect
-    async def query_folders(self, que: Dict, page_num: int, page_size: int) -> Tuple[List, int]:
+    async def query_folders(self, query: Dict, page_num: int, page_size: int) -> Tuple[List, int]:
         """Query database based on url query parameters.
 
-        :param que: Dict containing query information
+        :param query: Dict containing query information
         :param page_num: Page number
         :param page_size: Results per page
         :returns: Paginated query result
         """
-        _cursor = self.db_service.query("folder", que)
-        queried_folders = [folder async for folder in _cursor]
-        total_folders = len(queried_folders)
-        if total_folders <= page_size:
-            return queried_folders, total_folders
+        skips = page_size * (page_num - 1)
+        _query = [
+            {"$match": query},
+            {"$sort": {"dateCreated": -1}},
+            {"$skip": skips},
+            {"$limit": page_size},
+            {"$project": {"_id": 0}},
+        ]
+        data_raw = await self.db_service.do_aggregate("folder", _query)
+
+        if not data_raw:
+            data = []
         else:
-            lower = (page_num - 1) * page_size
-            upper = page_num * page_size
-            folders = queried_folders[lower:upper]
-            return folders, total_folders
+            data = [doc for doc in data_raw]
+
+        count_query = [{"$match": query}, {"$count": "total"}]
+        total_folders = await self.db_service.do_aggregate("folder", count_query)
+
+        if not total_folders:
+            total_folders = [{"total": 0}]
+
+        return data, total_folders[0]["total"]
 
     async def read_folder(self, folder_id: str) -> Dict:
         """Read object folder from database.
@@ -913,6 +926,45 @@ async def read_user(self, user_id: str) -> Dict:
             raise web.HTTPBadRequest(reason=reason)
         return user
 
+    async def filter_user(self, query: Dict, item_type: str, page_num: int, page_size: int) -> Tuple[List, int]:
+        """Query database based on url query parameters.
+
+        :param query: Dict containing query information
+        :param page_num: Page number
+        :param page_size: Results per page
+        :returns: Paginated query result
+        """
+        skips = page_size * (page_num - 1)
+        _query = [
+            {"$match": query},
+            {
+                "$project": {
+                    "_id": 0,
+                    item_type: {"$slice": [f"${item_type}", skips, page_size]},
+                }
+            },
+        ]
+        data = await self.db_service.do_aggregate("user", _query)
+
+        if not data:
+            data = [{item_type: []}]
+
+        count_query = [
+            {"$match": query},
+            {
+                "$project": {
+                    "_id": 0,
+                    "item": 1,
+                    "total": {
+                        "$cond": {"if": {"$isArray": f"${item_type}"}, "then": {"$size": f"${item_type}"}, "else": 0}
+                    },
+                }
+            },
+        ]
+        total_users = await self.db_service.do_aggregate("user", count_query)
+
+        return data[0][item_type], total_users[0]["total"]
+
     async def update_user(self, user_id: str, patch: List) -> str:
         """Update user object from database.
 
@@ -949,7 +1001,9 @@ async def assign_objects(self, user_id: str, collection: str, object_ids: List)
         """
         try:
             await self._check_user_exists(user_id)
-            assign_success = await self.db_service.append("user", user_id, {collection: {"$each": object_ids}})
+            assign_success = await self.db_service.append(
+                "user", user_id, {collection: {"$each": object_ids, "$position": 0}}
+            )
         except (ConnectionFailure, OperationFailure) as error:
             reason = f"Error happened while getting user: {error}"
             LOG.error(reason)
diff --git a/tests/test_handlers.py b/tests/test_handlers.py
index 94345a16b..1bbe85595 100644
--- a/tests/test_handlers.py
+++ b/tests/test_handlers.py
@@ -89,6 +89,7 @@ async def setUpAsync(self):
         useroperator_config = {
             "create_user.side_effect": self.fake_useroperator_create_user,
             "read_user.side_effect": self.fake_useroperator_read_user,
+            "filter_user.side_effect": self.fake_useroperator_filter_user,
             "check_user_has_doc.side_effect": self.fake_useroperator_user_has_folder,
         }
         self.patch_parser = patch(class_parser, spec=True)
@@ -197,6 +198,10 @@ async def fake_useroperator_read_user(self, user_id):
         """Fake read operation to return mocked user."""
         return self.test_user
 
+    async def fake_useroperator_filter_user(self, query, item_type, page, per_page):
+        """Fake read operation to return mocked user."""
+        return self.test_user[item_type], len(self.test_user[item_type])
+
     @unittest_run_loop
     async def test_submit_endpoint_submission_does_not_fail(self):
         """Test that submission with valid SUBMISSION.xml does not fail."""
@@ -716,7 +721,7 @@ async def test_get_user_drafts_with_no_drafts(self):
         """Test getting user drafts when user has no drafts."""
         response = await self.client.get("/users/current?items=drafts")
         self.assertEqual(response.status, 200)
-        self.MockedUserOperator().read_user.assert_called_once()
+        self.MockedUserOperator().filter_user.assert_called_once()
         json_resp = await response.json()
         result = {
             "page": {
@@ -734,10 +739,10 @@ async def test_get_user_drafts_with_1_draft(self):
         """Test getting user drafts when user has 1 draft."""
         user = self.test_user
         user["drafts"].append(self.metadata_json)
-        self.MockedUserOperator().read_user.return_value = user
+        self.MockedUserOperator().filter_user.return_value = (user["drafts"], 1)
         response = await self.client.get("/users/current?items=drafts")
         self.assertEqual(response.status, 200)
-        self.MockedUserOperator().read_user.assert_called_once()
+        self.MockedUserOperator().filter_user.assert_called_once()
         json_resp = await response.json()
         result = {
             "page": {
@@ -753,10 +758,10 @@ async def test_get_user_drafts_with_1_draft(self):
     @unittest_run_loop
     async def test_get_user_folder_list(self):
         """Test get user with folders url returns a folder ID."""
-        self.MockedUserOperator().read_user.return_value = self.test_user
+        self.MockedUserOperator().filter_user.return_value = (self.test_user["folders"], 1)
         response = await self.client.get("/users/current?items=folders")
         self.assertEqual(response.status, 200)
-        self.MockedUserOperator().read_user.assert_called_once()
+        self.MockedUserOperator().filter_user.assert_called_once()
         json_resp = await response.json()
         result = {
             "page": {
@@ -774,7 +779,6 @@ async def test_get_user_items_with_bad_param(self):
         """Test that error is raised if items parameter in query is not drafts or folders."""
         response = await self.client.get("/users/current?items=wrong_thing")
         self.assertEqual(response.status, 400)
-        self.MockedUserOperator().read_user.assert_called_once()
         json_resp = await response.json()
         self.assertEqual(
             json_resp["detail"], "wrong_thing is a faulty item parameter. Should be either folders or drafts"
diff --git a/tests/test_operators.py b/tests/test_operators.py
index 5424e6928..20ca13851 100644
--- a/tests/test_operators.py
+++ b/tests/test_operators.py
@@ -643,17 +643,17 @@ async def test_create_folder_db_create_fails(self):
     async def test_query_folders_empty_list(self):
         """Test query returns empty list."""
         operator = FolderOperator(self.client)
-        operator.db_service.query.return_value = AsyncIterator([])
+        operator.db_service.do_aggregate.side_effect = ([], [{"total": 0}])
         folders = await operator.query_folders({}, 1, 5)
-        operator.db_service.query.assert_called_once()
+        operator.db_service.do_aggregate.assert_called()
         self.assertEqual(folders, ([], 0))
 
     async def test_query_folders_1_item(self):
         """Test query returns a list with item."""
         operator = FolderOperator(self.client)
-        operator.db_service.query.return_value = AsyncIterator([{"name": "folder"}])
+        operator.db_service.do_aggregate.side_effect = ([{"name": "folder"}], [{"total": 1}])
         folders = await operator.query_folders({}, 1, 5)
-        operator.db_service.query.assert_called_once()
+        operator.db_service.do_aggregate.assert_called()
         self.assertEqual(folders, ([{"name": "folder"}], 1))
 
     async def test_check_object_folder_fails(self):

From 76aa1828604e37cce8c63c29993dc7819c507b4f Mon Sep 17 00:00:00 2001
From: Stefan Negru <stefan@blankdots.com>
Date: Thu, 12 Aug 2021 21:43:24 +0300
Subject: [PATCH 03/25] append using push so that we can add at the beginning

---
 metadata_backend/database/db_service.py | 5 ++++-
 tests/test_db_service.py                | 2 +-
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/metadata_backend/database/db_service.py b/metadata_backend/database/db_service.py
index d19c346f4..7e0f9977b 100644
--- a/metadata_backend/database/db_service.py
+++ b/metadata_backend/database/db_service.py
@@ -198,7 +198,10 @@ async def append(self, collection: str, accession_id: str, data_to_be_addded: An
         """
         id_key = f"{collection}Id" if (collection in ["folder", "user"]) else "accessionId"
         find_by_id = {id_key: accession_id}
-        append_op = {"$addToSet": data_to_be_addded}
+        # push vs addtoSet
+        # push allows us to specify the postion but it does not check the items are unique
+        # addToSet cannot easily specify position
+        append_op = {"$push": data_to_be_addded}
         result = await self.database[collection].find_one_and_update(
             find_by_id, append_op, projection={"_id": False}, return_document=ReturnDocument.AFTER
         )
diff --git a/tests/test_db_service.py b/tests/test_db_service.py
index 0153138f4..b88748710 100644
--- a/tests/test_db_service.py
+++ b/tests/test_db_service.py
@@ -197,7 +197,7 @@ async def test_append_data(self):
         success = await self.test_service.append("testcollection", self.id_stub, self.data_stub)
         self.collection.find_one_and_update.assert_called_once_with(
             {"accessionId": self.id_stub},
-            {"$addToSet": self.data_stub},
+            {"$push": self.data_stub},
             projection={"_id": False},
             return_document=True,
         )

From 33dd237c1684fcf1727b155e478bef6abb09b81c Mon Sep 17 00:00:00 2001
From: Stefan Negru <stefan@blankdots.com>
Date: Thu, 12 Aug 2021 22:07:00 +0300
Subject: [PATCH 04/25] add indexes at app start is problematic move them to db

---
 metadata_backend/conf/conf.py | 23 ++---------------------
 metadata_backend/server.py    |  2 +-
 2 files changed, 3 insertions(+), 22 deletions(-)

diff --git a/metadata_backend/conf/conf.py b/metadata_backend/conf/conf.py
index 816985a79..ebe4e0b35 100644
--- a/metadata_backend/conf/conf.py
+++ b/metadata_backend/conf/conf.py
@@ -94,32 +94,13 @@ def set_conf() -> Tuple[str, str]:
 connectTimeout = 15000
 
 
-async def create_db_client() -> AsyncIOMotorClient:
+def create_db_client() -> AsyncIOMotorClient:
     """Initialize database client for AioHTTP App.
 
     :returns: Coroutine-based Motor client for Mongo operations
     """
     LOG.debug("initialised DB client")
-    client = AsyncIOMotorClient(url, connectTimeoutMS=connectTimeout, serverSelectionTimeoutMS=serverTimeout)
-    await create_indexes(client, mongo_database)
-    return client
-
-
-async def create_indexes(client: AsyncIOMotorClient, mongo_database: str) -> None:
-    """Create Indexes for collections."""
-    db = client[mongo_database]
-    collections = await db.list_collection_names()
-
-    if "folder" not in collections:
-        await db.create_collection("folder")
-
-    db.folder.create_index([("dateCreated", -1)])
-    db.folder.create_index([("folderId", 1)], unique=True)
-
-    if "user" not in collections:
-        await db.create_collection("user")
-
-    db.user.create_index([("userId", 1)], unique=True)
+    return AsyncIOMotorClient(url, connectTimeoutMS=connectTimeout, serverSelectionTimeoutMS=serverTimeout)
 
 
 # 2) Load schema types and descriptions from json
diff --git a/metadata_backend/server.py b/metadata_backend/server.py
index 8eb982c4a..49de9e973 100644
--- a/metadata_backend/server.py
+++ b/metadata_backend/server.py
@@ -119,7 +119,7 @@ async def init() -> web.Application:
         ]
         server.router.add_routes(frontend_routes)
         LOG.info("Frontend routes loaded")
-    server["db_client"] = await create_db_client()
+    server["db_client"] = create_db_client()
     server.on_shutdown.append(kill_sess_on_shutdown)
     LOG.info("Database client loaded")
     return server

From 83ef672cb8f26acaac1650dacf92790dbc40b0d8 Mon Sep 17 00:00:00 2001
From: Stefan Negru <stefan@blankdots.com>
Date: Thu, 12 Aug 2021 22:17:39 +0300
Subject: [PATCH 05/25] add bandit to check for known issues

---
 .github/workflows/style.yml |  2 ++
 tox.ini                     | 11 +++++++++--
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/style.yml b/.github/workflows/style.yml
index ef83af0ff..9a0287bcf 100644
--- a/.github/workflows/style.yml
+++ b/.github/workflows/style.yml
@@ -28,3 +28,5 @@ jobs:
       run: tox -e flake8
     - name: Type hints check
       run: tox -e mypy
+    - name: Run bandit static code analysis
+      run: tox -e bandit
diff --git a/tox.ini b/tox.ini
index d4b651818..b724108db 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,5 +1,5 @@
 [tox]
-envlist = py38, flake8, mypy, docs, black
+envlist = py38, flake8, mypy, docs, black, bandit
 skipsdist = True
 
 [flake8]
@@ -41,6 +41,13 @@ deps =
     black
 commands = black . -l 120 --check
 
+[testenv:bandit]
+skip_install = true
+; plain search for known vulnerable code
+deps =
+    bandit
+commands = bandit -r metadata_backend/
+
 [testenv]
 passenv = COVERALLS_REPO_TOKEN
 deps =
@@ -52,4 +59,4 @@ commands = py.test -x --cov=metadata_backend tests/
 
 [gh-actions]
 python =
-    3.8: flake8, py38, docs, mypy, black
+    3.8: flake8, py38, docs, mypy, black, bandit

From 16c9739056d657cf4fc1ac609f3562626c2a324d Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 16 Aug 2021 09:05:12 +0000
Subject: [PATCH 06/25] Bump uvloop from 0.15.3 to 0.16.0

Bumps [uvloop](https://github.com/MagicStack/uvloop) from 0.15.3 to 0.16.0.
- [Release notes](https://github.com/MagicStack/uvloop/releases)
- [Commits](https://github.com/MagicStack/uvloop/compare/v0.15.3...v0.16.0)

---
updated-dependencies:
- dependency-name: uvloop
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 64264dd95..f2a5e3af5 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -4,6 +4,6 @@ gunicorn==20.1.0
 jsonschema==3.2.0
 motor==2.5.0
 python-dateutil==2.8.2
-uvloop==0.15.3
+uvloop==0.16.0
 xmlschema==1.7.0
 Authlib==0.15.4

From 0f4536223ca06a61bcb4d753689caa7f3570421c Mon Sep 17 00:00:00 2001
From: Stefan Negru <stefan@blankdots.com>
Date: Fri, 20 Aug 2021 16:20:13 +0300
Subject: [PATCH 07/25] bump package version

---
 docs/conf.py                 | 2 +-
 metadata_backend/__init__.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/conf.py b/docs/conf.py
index e21467d3f..af2536b80 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -11,7 +11,7 @@
 author = "CSC Developers"
 
 # The full version, including alpha/beta/rc tags
-release = "0.10.0"
+release = "0.11.0"
 
 
 # -- General configuration ---------------------------------------------------
diff --git a/metadata_backend/__init__.py b/metadata_backend/__init__.py
index 427541984..b5a0f65d8 100644
--- a/metadata_backend/__init__.py
+++ b/metadata_backend/__init__.py
@@ -1,5 +1,5 @@
 """Backend for submitting and validating XML Files containing ENA metadata."""
 
 __title__ = "metadata_backend"
-__version__ = VERSION = "0.10.0"
+__version__ = VERSION = "0.11.0"
 __author__ = "CSC Developers"

From 822ad0ed5ae9f9f30254ee2ad6593898a1e4a56f Mon Sep 17 00:00:00 2001
From: Stefan Negru <stefan@blankdots.com>
Date: Fri, 20 Aug 2021 16:20:33 +0300
Subject: [PATCH 08/25] bump npm version docker

---
 Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index a3a5f1a43..b317c8c81 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -9,7 +9,7 @@ ARG BRANCH=master
 RUN git clone -b ${BRANCH} https://github.com/CSCfi/metadata-submitter-frontend.git
 
 WORKDIR /metadata-submitter-frontend
-RUN npm install -g npm@7.20.0 \
+RUN npm install -g npm@7.21.0 \
     && npx --quiet pinst --disable \
     && npm install --production \
     && npm run build --production

From bcf70c359250411ea233e978ba9768c14577d66e Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 23 Aug 2021 09:03:26 +0000
Subject: [PATCH 09/25] Bump tox from 3.24.1 to 3.24.3

Bumps [tox](https://github.com/tox-dev/tox) from 3.24.1 to 3.24.3.
- [Release notes](https://github.com/tox-dev/tox/releases)
- [Changelog](https://github.com/tox-dev/tox/blob/master/docs/changelog.rst)
- [Commits](https://github.com/tox-dev/tox/compare/3.24.1...3.24.3)

---
updated-dependencies:
- dependency-name: tox
  dependency-type: direct:development
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 002b133db..b336b3302 100644
--- a/setup.py
+++ b/setup.py
@@ -29,7 +29,7 @@
     packages=find_packages(exclude=["tests"]),
     install_requires=requirements,
     extras_require={
-        "test": ["coverage==5.5", "coveralls==3.2.0", "pytest==6.2.4", "pytest-cov==2.12.1", "tox==3.24.1"],
+        "test": ["coverage==5.5", "coveralls==3.2.0", "pytest==6.2.4", "pytest-cov==2.12.1", "tox==3.24.3"],
         "docs": ["sphinx >= 1.4", "sphinx_rtd_theme==0.5.2"],
     },
     package_data={

From d8bb851b8fec6d736e18ebbf4e0ccacaf39c37a0 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 23 Aug 2021 09:03:29 +0000
Subject: [PATCH 10/25] Bump motor from 2.5.0 to 2.5.1

Bumps [motor](https://github.com/mongodb/motor) from 2.5.0 to 2.5.1.
- [Release notes](https://github.com/mongodb/motor/releases)
- [Changelog](https://github.com/mongodb/motor/blob/master/doc/changelog.rst)
- [Commits](https://github.com/mongodb/motor/compare/2.5.0...2.5.1)

---
updated-dependencies:
- dependency-name: motor
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index f2a5e3af5..29b627adc 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -2,7 +2,7 @@ aiohttp==3.7.4.post0
 cryptography==3.4.7
 gunicorn==20.1.0
 jsonschema==3.2.0
-motor==2.5.0
+motor==2.5.1
 python-dateutil==2.8.2
 uvloop==0.16.0
 xmlschema==1.7.0

From dfaf070d9dd314533d26d3241f9abb3251dbd2ef Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Joonatan=20M=C3=A4kinen?= <joonatan.makinen@csc.fi>
Date: Mon, 9 Aug 2021 09:48:50 +0300
Subject: [PATCH 11/25] add required DOI properties to the folder schema

---
 metadata_backend/helpers/schemas/folders.json | 151 ++++++++++++++++++
 1 file changed, 151 insertions(+)

diff --git a/metadata_backend/helpers/schemas/folders.json b/metadata_backend/helpers/schemas/folders.json
index 509e03bde..dd5f25383 100644
--- a/metadata_backend/helpers/schemas/folders.json
+++ b/metadata_backend/helpers/schemas/folders.json
@@ -26,6 +26,157 @@
       "type": "boolean",
       "title": "Published Folder"
     },
+    "doi": {
+      "type": "object",
+      "title": "The DOI schema",
+      "required": [
+        "identifier",
+        "url",
+        "creators",
+        "titles",
+        "publisher",
+        "subjects",
+        "publicationYear",
+        "resourceType",
+      ],
+      "properties": {
+        "identifier": {
+          "type": "object",
+          "title": "DOI id generated according to DOI recommendations",
+          "required": [
+            "identifierType"
+          ],
+          "properties": {
+            "identifierType": {
+              "type": "string",
+              "title": "Type of identifier (= DOI)"
+            },
+            "doi": {
+              "type": "string",
+              "title": "Character string of DOI handle"
+            },
+            "id": {
+              "type": "string",
+              "title": "Display url for DOI id"
+            }
+          }
+        },
+        "url": {
+          "type": "string",
+          "title": "URL of the digital location of the object"
+        },
+        "creators": {
+          "type": "array",
+          "title": "List of creators",
+          "items": {
+            "type": "object",
+            "title": "Creator objects",
+            "properties": {
+              "name": {
+                "type": "string",
+                "title": "Full name"
+              },
+              "nameType": {
+                "type": "string",
+                "title": "Type of name"
+              },
+              "givenName": {
+                "type": "string",
+                "title": "First name"
+              },
+              "familyName": {
+                "type": "string",
+                "title": "Last name"
+              },
+              "nameIdentifiers": {
+                "type": "array",
+                "title": "List of name identifiers",
+                "items": {
+                  "type": "object",
+                  "title": "Name identifier object"
+                }
+              },
+              "affiliation": {
+                "type": "array",
+                "title": "List of affiliations",
+                "items": {
+                  "type": "object",
+                  "title": "Name affiliation object"
+                }
+              },
+            }
+          },
+          "uniqueItems": true
+        },
+        "titles": {
+          "type": "array",
+          "title": "List of titles (in different languages)",
+          "items": {
+            "type": "object",
+            "title": "Title objects",
+            "properties": {
+              "lang": {
+                "type": "string",
+                "title": "Language of the title"
+              },
+              "title": {
+                "type": "string",
+                "title": "Full title"
+              },
+              "titleType": {
+                "type": "string",
+                "title": "Type of title"
+              }
+            }
+          },
+          "uniqueItems": true
+        },
+        "publisher": {
+          "type": "string",
+          "title": "Full name of publisher from Research Organization Registry"
+        },
+        "subjects": {
+          "type": "array",
+          "title": "List of subjects specified by FOS",
+          "items": {
+            "type": "object",
+            "title": "Subject objects",
+            "properties": {
+              "subject": {
+                "type": "string",
+                "title": "FOS defined subject name"
+              },
+              "subjectScheme": {
+                "type": "string",
+                "title": "Subject scheme name"
+              }
+            }
+          },
+          "uniqueItems": true
+        },
+        "publicationYear": {
+          "type": "integer",
+          "title": "Year of publication"
+        },
+        "resourceType": {
+          "type": "object",
+          "title": "Type info of the resource",
+          "required": [
+            "resourceTypeGeneral"
+          ],
+          "properties": {
+            "resourceTypeGeneral": {
+              "type": "string",
+              "title": "Mandatory general type name"
+            },
+            "type": {
+              "type": "string",
+              "title": "Name of resource type"
+            }
+          }
+        }
+      }
+    },
     "metadataObjects": {
       "type": "array",
       "title": "The metadataObjects schema",

From 4af2b5c35054cb8f13a495ae96875b7d7862029e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Joonatan=20M=C3=A4kinen?= <joonatan.makinen@csc.fi>
Date: Mon, 9 Aug 2021 09:49:35 +0300
Subject: [PATCH 12/25] add required DOI properties to OpenApi specs

---
 docs/specification.yml | 108 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 108 insertions(+)

diff --git a/docs/specification.yml b/docs/specification.yml
index 07a68a068..c28617304 100644
--- a/docs/specification.yml
+++ b/docs/specification.yml
@@ -1290,6 +1290,7 @@ components:
         - published
         - metadataObjects
         - drafts
+        - doi
       additionalProperties: false
       properties:
         folderId:
@@ -1304,6 +1305,113 @@ components:
         published:
           type: boolean
           description: If folder is published or not
+        doi:
+          type: object
+          required:
+            - identifier
+            - url
+            - creators
+            - titles
+            - publisher
+            - subjects
+            - publicationYear
+            - resourceType
+          properties:
+            identifier:
+              type: object
+              description: DOI id generated according to DOI recommendations
+              required:
+                - identifierType
+              properties:
+                identifierType:
+                  type: string
+                  example: "DOI"
+                doi:
+                  type: string
+                  description: Character string of DOI handle
+                  example: "prefix/suffix"
+                id:
+                  type: string
+                  description: Display url for DOI id
+                  example: "https://doi.org/prefix/suffix"
+            url:
+              type: string
+              description: URL of the digital location of the object
+            creators:
+              type: array
+              items:
+                type: object
+                properties:
+                  name:
+                    type: string
+                    description: Full name
+                    example: "Last name, First name"
+                  nameType:
+                    type: string
+                    description: Type of name
+                    example: "Personal"
+                  givenName:
+                    type: string
+                    description: Official first name
+                    example: "First name"
+                  familyName:
+                    type: string
+                    description: Official last name
+                    example: "Last name"
+                  nameIdentifiers:
+                    type: array
+                    items:
+                      type: object
+                  affiliation:
+                    type: array
+                    items:
+                      type: object
+            titles:
+              type: array
+              items:
+                type: object
+                properties:
+                  lang:
+                    type: string
+                    description: Language of the title
+                    example: "en"
+                  title:
+                    type: string
+                    description: Full title
+                  titleType:
+                    type: string
+            publisher:
+              type: string
+              description: Full name of publisher from Research Organization Registry
+            subjects:
+              type: array
+              items:
+                type: object
+                properties:
+                  subject:
+                    type: string
+                    description: FOS defined subject name
+                    example: "FOS: field"
+                  subjectScheme:
+                    type: string
+                    description: Subject scheme name
+                    example: "Fields of Science and Technology (FOS)"
+            publicationYear:
+              type: integer
+              description: Year of publication
+              example: 2021
+            resourceType:
+              type: object
+              required:
+                - resourceTypeGeneral
+              properties:
+                resourceTypeGeneral:
+                  type: string
+                  description: Mandatory general type name
+                  example: "Dataset"
+                type:
+                  type: string
+                  description: Name of resource type
         metadataObjects:
           type: array
           items:

From 0cb97b96f8c78287d8ce31d438120e5e1c363e2a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Joonatan=20M=C3=A4kinen?= <joonatan.makinen@csc.fi>
Date: Tue, 10 Aug 2021 10:51:20 +0300
Subject: [PATCH 13/25] fix schema formatting

---
 metadata_backend/helpers/schemas/folders.json | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/metadata_backend/helpers/schemas/folders.json b/metadata_backend/helpers/schemas/folders.json
index dd5f25383..6b2a53d1a 100644
--- a/metadata_backend/helpers/schemas/folders.json
+++ b/metadata_backend/helpers/schemas/folders.json
@@ -37,7 +37,7 @@
         "publisher",
         "subjects",
         "publicationYear",
-        "resourceType",
+        "resourceType"
       ],
       "properties": {
         "identifier": {
@@ -103,7 +103,7 @@
                   "type": "object",
                   "title": "Name affiliation object"
                 }
-              },
+              }
             }
           },
           "uniqueItems": true

From 6cb11e8eccd39cf0fca01f57b4011028c8225178 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Joonatan=20M=C3=A4kinen?= <joonatan.makinen@csc.fi>
Date: Tue, 10 Aug 2021 12:40:14 +0300
Subject: [PATCH 14/25] add other recommended DOI properties to folder schema

---
 metadata_backend/helpers/schemas/folders.json | 232 +++++++++++++++++-
 1 file changed, 227 insertions(+), 5 deletions(-)

diff --git a/metadata_backend/helpers/schemas/folders.json b/metadata_backend/helpers/schemas/folders.json
index 6b2a53d1a..2c15bd473 100644
--- a/metadata_backend/helpers/schemas/folders.json
+++ b/metadata_backend/helpers/schemas/folders.json
@@ -74,7 +74,7 @@
             "properties": {
               "name": {
                 "type": "string",
-                "title": "Full name"
+                "title": "Full name of creator"
               },
               "nameType": {
                 "type": "string",
@@ -93,7 +93,21 @@
                 "title": "List of name identifiers",
                 "items": {
                   "type": "object",
-                  "title": "Name identifier object"
+                  "title": "Name identifier object",
+                  "properties": {
+                    "schemeUri": {
+                      "type": "string",
+                      "title": "URI of the name identifier scheme"
+                    },
+                    "nameIdentifier": {
+                      "type": "string",
+                      "title": "Location of name identifier"
+                    },
+                    "nameIdentifierScheme": {
+                      "type": "string",
+                      "title": "Name of name identifier scheme"
+                    }
+                  }
                 }
               },
               "affiliation": {
@@ -101,7 +115,25 @@
                 "title": "List of affiliations",
                 "items": {
                   "type": "object",
-                  "title": "Name affiliation object"
+                  "title": "Name affiliation object",
+                  "properties": {
+                    "name": {
+                      "type": "string",
+                      "title": "Name of the place of affiliation"
+                    },
+                    "schemeUri": {
+                      "type": "string",
+                      "title": "URI of the affiliation scheme"
+                    },
+                    "affiliationIdentifier": {
+                      "type": "string",
+                      "title": "Location of affiliation identifier"
+                    },
+                    "affiliationIdentifierScheme": {
+                      "type": "string",
+                      "title": "Name of affiliation identifier scheme"
+                    }
+                  }
                 }
               }
             }
@@ -110,7 +142,7 @@
         },
         "titles": {
           "type": "array",
-          "title": "List of titles (in different languages)",
+          "title": "List of titles",
           "items": {
             "type": "object",
             "title": "Title objects",
@@ -144,7 +176,7 @@
             "properties": {
               "subject": {
                 "type": "string",
-                "title": "FOS defined subject name"
+                "title": "FOS identifier"
               },
               "subjectScheme": {
                 "type": "string",
@@ -174,6 +206,196 @@
               "title": "Name of resource type"
             }
           }
+        },
+        "contributors": {
+          "type": "array",
+          "title": "List of contributors",
+          "items": {
+            "type": "object",
+            "title": "Contributor object",
+            "properties": {
+              "name": {
+                "type": "string",
+                "title": "Full name of contributor"
+              },
+              "nameType": {
+                "type": "string",
+                "title": "Type of name"
+              },
+              "givenName": {
+                "type": "string",
+                "title": "First name"
+              },
+              "familyName": {
+                "type": "string",
+                "title": "Last name"
+              },
+              "contributorType": {
+                "type": "string",
+                "title": "Type of contributor"
+              },
+              "nameIdentifiers": {
+                "type": "array",
+                "title": "List of name identifiers",
+                "items": {
+                  "type": "object",
+                  "title": "Name identifier object",
+                  "properties": {
+                    "schemeUri": {
+                      "type": "string",
+                      "title": "URI of the name identifier scheme"
+                    },
+                    "nameIdentifier": {
+                      "type": "string",
+                      "title": "Location of name identifier"
+                    },
+                    "nameIdentifierScheme": {
+                      "type": "string",
+                      "title": "Name of name identifier scheme"
+                    }
+                  }
+                }
+              },
+              "affiliation": {
+                "type": "array",
+                "title": "List of affiliations",
+                "items": {
+                  "type": "object",
+                  "title": "Name affiliation object",
+                  "properties": {
+                    "name": {
+                      "type": "string",
+                      "title": "Name of the place of affiliation"
+                    },
+                    "schemeUri": {
+                      "type": "string",
+                      "title": "URI of the affiliation scheme"
+                    },
+                    "affiliationIdentifier": {
+                      "type": "string",
+                      "title": "Location of affiliation identifier"
+                    },
+                    "affiliationIdentifierScheme": {
+                      "type": "string",
+                      "title": "Name of affiliation identifier scheme"
+                    }
+                  }
+                }
+              }
+            }
+          }
+        },
+        "dates": {
+          "type": "array",
+          "title": "List of relevant dates to publication",
+          "items": {
+            "type": "object",
+            "title": "Date object",
+            "properties": {
+              "date": {
+                "type": "string",
+                "title": "The date value"
+              },
+              "dateType": {
+                "type": "string",
+                "title": "Relevance of the date"
+              },
+              "dateInformation": {
+                "type": "string",
+                "title": "Specific event of the date"
+              }
+            }
+          }
+        },
+        "descriptions": {
+          "type": "array",
+          "title": "List of descriptions",
+          "items": {
+            "type": "object",
+            "title": "Description object",
+            "properties": {
+              "lang": {
+                "type": "string",
+                "title": "Language of the description"
+              },
+              "description": {
+                "type": "string",
+                "title": "Full description"
+              },
+              "descriptionType": {
+                "type": "string",
+                "title": "Type of description"
+              }
+            }
+          }
+        },
+        "geoLocations": {
+          "type": "array",
+          "title": "List of GeoLocations",
+          "items": {
+            "type": "object",
+            "title": "GeoLocation object",
+            "properties": {
+              "geoLocationPlace": {
+                "type": "string",
+                "title": "Name of GeoLocation"
+              },
+              "geoLocationPoint": {
+                "type": "object",
+                "title": "GeoLocation point object",
+                "properties": {
+                  "pointLongitude": {
+                    "type": "string",
+                    "title": "Longitude coordinate"
+                  },
+                  "pointLatitude": {
+                    "type": "string",
+                    "title": "Latitude coordinate"
+                  }
+                }
+              },
+              "geoLocationBox": {
+                "type": "object",
+                "title": "GeoLocation box object",
+                "properties": {
+                  "westBoundLongitude": {
+                    "type": "string",
+                    "title": "Longitude coordinate of west bound"
+                  },
+                  "eastBoundLongitude": {
+                    "type": "string",
+                    "title": "Longitude coordinate of east bound"
+                  },
+                  "southBoundLatitude": {
+                    "type": "string",
+                    "title": "Latitude coordinate of south bound"
+                  },
+                  "northBoundLatitude": {
+                    "type": "string",
+                    "title": "Latitude coordinate of north bound"
+                  }
+                }
+              },
+              "geoLocationPolygon": {
+                "type": "array",
+                "title": "List of polygon points",
+                "items": {
+                  "type": "object",
+                  "title": "Polygon point object",
+                  "properties": {
+                    "pointLongitude": {
+                      "type": "string",
+                      "title": "Longitude coordinate"
+                    },
+                    "pointLatitude": {
+                      "type": "string",
+                      "title": "Latitude coordinate"
+                    }
+                  }
+                }
+              }
+            }
+          }
         }
       }
     },

From 51bcbd372fc425858cee5896eb4c4a8a24a48fe1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Joonatan=20M=C3=A4kinen?= <joonatan.makinen@csc.fi>
Date: Wed, 11 Aug 2021 13:28:38 +0300
Subject: [PATCH 15/25] add remaining optional DOI properties to the schema

---
 metadata_backend/helpers/schemas/folders.json | 166 +++++++++++++++++-
 1 file changed, 160 insertions(+), 6 deletions(-)

diff --git a/metadata_backend/helpers/schemas/folders.json b/metadata_backend/helpers/schemas/folders.json
index 2c15bd473..5789f29bd 100644
--- a/metadata_backend/helpers/schemas/folders.json
+++ b/metadata_backend/helpers/schemas/folders.json
@@ -97,11 +97,11 @@
                   "properties": {
                     "schemeUri": {
                       "type": "string",
-                      "title": "URI of the name identifier scheme"
+                      "title": "URI (location) of the name identifier scheme"
                     },
                     "nameIdentifier": {
                       "type": "string",
-                      "title": "Location of name identifier"
+                      "title": "URI (location) of name identifier"
                     },
                     "nameIdentifierScheme": {
                       "type": "string",
@@ -123,7 +123,7 @@
                     },
                     "schemeUri": {
                       "type": "string",
-                      "title": "URI of the affiliation scheme"
+                      "title": "URI (location) of the affiliation scheme"
                     },
                     "affiliationIdentifier": {
                       "type": "string",
@@ -243,7 +243,7 @@
                   "properties": {
                     "schemeUri": {
                       "type": "string",
-                      "title": "URI of the name identifier scheme"
+                      "title": "URI (location) of the name identifier scheme"
                     },
                     "nameIdentifier": {
                       "type": "string",
@@ -269,7 +269,7 @@
                     },
                     "schemeUri": {
                       "type": "string",
-                      "title": "URI of the affiliation scheme"
+                      "title": "URI (location) of the affiliation scheme"
                     },
                     "affiliationIdentifier": {
                       "type": "string",
@@ -396,8 +396,162 @@
               }
             }
           }
+        },
+        "language": {
+          "type": "string",
+          "title": "Language code"
+        },
+        "alternateIdentifiers": {
+          "type": "array",
+          "title": "List of alternate identifiers",
+          "items": {
+            "type": "object",
+            "title": "Alternate identifier object",
+            "properties": {
+              "alternateIdentifier": {
+                "type": "string",
+                "title": "Alternate identifier info"
+              },
+              "alternateIdentifierType": {
+                "type": "string",
+                "title": "Type of alternate identifier"
+              }
+            }
+          }
+        },
+        "relatedIdentifiers": {
+          "type": "array",
+          "title": "List of related identifiers",
+          "items": {
+            "type": "object",
+            "title": "Related identifier object",
+            "properties": {
+              "relatedIdentifier": {
+                "type": "string",
+                "title": "Related identifier info"
+              },
+              "relatedIdentifierType": {
+                "type": "string",
+                "title": "Type of related identifier"
+              },
+              "relationType": {
+                "type": "string",
+                "title": "Specification of the relation"
+              },
+              "relatedMetadataScheme": {
+                "type": "string",
+                "title": "Scheme of related metadata"
+              },
+              "schemeUri": {
+                "type": "string",
+                "title": "URI (location) of the related metadata scheme"
+              },
+              "schemeType": {
+                "type": "string",
+                "title": "Type of the related metadata scheme"
+              },
+              "resourceTypeGeneral": {
+                "type": "string",
+                "title": "Optional general type name"
+              }
+            }
+          }
+        },
+        "sizes": {
+          "type": "array",
+          "title": "List of sizes",
+          "items": {
+            "type": "string",
+            "title": "Size or duration info about the resource"
+          }
+        },
+        "formats": {
+          "type": "array",
+          "title": "List of formats",
+          "items": {
+            "type": "string",
+            "title": "Technical format of the resource"
+          }
+        },
+        "version": {
+          "type": "string",
+          "title": "Version number of the resource"
+        },
+        "rightsList": {
+          "type": "array",
+          "title": "List of any rights information",
+          "items": {
+            "type": "object",
+            "title": "Rights object",
+            "properties": {
+              "lang": {
+                "type": "string",
+                "title": "Language code of the referenced rights"
+              },
+              "rightsUri": {
+                "type": "string",
+                "title": "URI (location) of the referenced rights"
+              },
+              "rightsIdentifier": {
+                "type": "string",
+                "title": "Title of the rights identifier"
+              },
+              "rightsIdentifierScheme": {
+                "type": "string",
+                "title": "Title of rights identifier scheme"
+              },
+              "schemeUri": {
+                "type": "string",
+                "title": "URI (location) of the scheme for rights"
+              }
+            }
+          }
+        },
+        "fundingReferences": {
+          "type": "array",
+          "title": "List of funding references",
+          "itmes": {
+            "type": "object",
+            "title": "Funding reference object",
+            "required": [
+              "funderName",
+              "funderIdentifier",
+              "funderIdentifierType"
+            ],
+            "properties": {
+              "funderName": {
+                "type": "string",
+                "title": "Name of the funding provider"
+              },
+              "funderIdentifier": {
+                "type": "string",
+                "title": "Unique identifier for funding entity"
+              },
+              "funderIdentifierType": {
+                "type": "string",
+                "title": "Type of identifier for funding entity"
+              },
+              "schemeUri": {
+                "type": "string",
+                "title": "URI (location) of scheme for funder identifier"
+              },
+              "awardNumber": {
+                "type": "string",
+                "title": "The code assigned by the funder to a sponsored award"
+              },
+              "awardTitle": {
+                "type": "string",
+                "title": "The human readable title of the award"
+              },
+              "awardUri": {
+                "type": "string",
+                "title": "URI (location) of the award"
+              }
+            }
+          }
         }
-      }
+      },
+      "additionalProperties": false
     },
     "metadataObjects": {
       "type": "array",

From 37fa0f4413a58dd74571845be93900a79a4f352c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Joonatan=20M=C3=A4kinen?= <joonatan.makinen@csc.fi>
Date: Wed, 11 Aug 2021 21:16:58 +0300
Subject: [PATCH 16/25] add accuracy to DOI schema propreties

---
 metadata_backend/helpers/schemas/folders.json | 150 +++++++++++-------
 1 file changed, 95 insertions(+), 55 deletions(-)

diff --git a/metadata_backend/helpers/schemas/folders.json b/metadata_backend/helpers/schemas/folders.json
index 5789f29bd..84553b4f5 100644
--- a/metadata_backend/helpers/schemas/folders.json
+++ b/metadata_backend/helpers/schemas/folders.json
@@ -42,9 +42,10 @@
       "properties": {
         "identifier": {
           "type": "object",
-          "title": "DOI id generated according to DOI recommendations",
+          "title": "identifier object",
           "required": [
-            "identifierType"
+            "identifierType",
+            "doi"
           ],
           "properties": {
             "identifierType": {
@@ -53,11 +54,7 @@
             },
             "doi": {
               "type": "string",
-              "title": "Character string of DOI handle"
-            },
-            "id": {
-              "type": "string",
-              "title": "Display url for DOI id"
+              "title": "A persistent identifier for a resource"
             }
           }
         },
@@ -70,11 +67,11 @@
           "title": "List of creators",
           "items": {
             "type": "object",
-            "title": "Creator objects",
+            "title": "Main researchers involved with data or the authors of the publication",
             "properties": {
               "name": {
                 "type": "string",
-                "title": "Full name of creator"
+                "title": "Full name of creator (format: Family, Given)"
               },
               "nameType": {
                 "type": "string",
@@ -108,7 +105,8 @@
                       "title": "Name of name identifier scheme"
                     }
                   }
-                }
+                },
+                "uniqueItems": true
               },
               "affiliation": {
                 "type": "array",
@@ -134,9 +132,11 @@
                       "title": "Name of affiliation identifier scheme"
                     }
                   }
-                }
+                },
+                "uniqueItems": true
               }
-            }
+            },
+            "additionalProperties": false
           },
           "uniqueItems": true
         },
@@ -149,17 +149,18 @@
             "properties": {
               "lang": {
                 "type": "string",
-                "title": "Language of the title"
+                "title": "Language code of the title"
               },
               "title": {
                 "type": "string",
-                "title": "Full title"
+                "title": "A name or title by which a resource is known"
               },
               "titleType": {
                 "type": "string",
                 "title": "Type of title"
               }
-            }
+            },
+            "additionalProperties": false
           },
           "uniqueItems": true
         },
@@ -169,10 +170,13 @@
         },
         "subjects": {
           "type": "array",
-          "title": "List of subjects specified by FOS",
+          "title": "List of subject identifiers specified by FOS",
           "items": {
             "type": "object",
             "title": "Subject objects",
+            "required": [
+              "subject"
+            ],
             "properties": {
               "subject": {
                 "type": "string",
@@ -182,41 +186,47 @@
                 "type": "string",
                 "title": "Subject scheme name"
               }
-            }
+            },
+            "additionalProperties": true
           },
           "uniqueItems": true
         },
         "publicationYear": {
           "type": "integer",
-          "title": "Year of publication"
+          "title": "Year when the data is made publicly available"
         },
         "resourceType": {
           "type": "object",
           "title": "Type info of the resource",
           "required": [
+            "type",
             "resourceTypeGeneral"
           ],
           "properties": {
-            "resourceTypeGeneral": {
-              "type": "string",
-              "title": "Mandatory general type name"
-            },
             "type": {
               "type": "string",
               "title": "Name of resource type"
+            },
+            "resourceTypeGeneral": {
+              "type": "string",
+              "title": "Mandatory general type name"
             }
-          }
+          },
+          "additionalProperties": false
         },
         "contributors": {
           "type": "array",
           "title": "List of contributors",
           "items": {
             "type": "object",
-            "title": "Contributor object",
+            "title": "The institution or person responsible for contributing to the developement of the dataset",
+            "required": [
+              "contributorType"
+            ],
             "properties": {
               "name": {
                 "type": "string",
-                "title": "Full name of contributor"
+                "title": "Full name of contributor (format: Family, Given)"
               },
               "nameType": {
                 "type": "string",
@@ -282,8 +292,10 @@
                   }
                 }
               }
-            }
-          }
+            },
+            "additionalProperties": false
+          },
+          "uniqueItems": true
         },
         "dates": {
           "type": "array",
@@ -291,10 +303,14 @@
           "items": {
             "type": "object",
             "title": "Date object",
+            "required": [
+              "date",
+              "dateType"
+            ],
             "properties": {
               "date": {
                 "type": "string",
-                "title": "The date value"
+                "title": "A standard format for a date value"
               },
               "dateType": {
                 "type": "string",
@@ -304,8 +320,10 @@
                 "type": "string",
                 "title": "Specific event of the date"
               }
-            }
-          }
+            },
+            "additionalProperties": false
+          },
+          "uniqueItems": true
         },
         "descriptions": {
           "type": "array",
@@ -316,18 +334,20 @@
             "properties": {
               "lang": {
                 "type": "string",
-                "title": "Language of the description"
+                "title": "Language code of the description"
               },
               "description": {
                 "type": "string",
-                "title": "Full description"
+                "title": "Additional information that does not fit in any of the other categories"
               },
               "descriptionType": {
                 "type": "string",
                 "title": "Type of description"
               }
-            }
-          }
+            },
+            "additionalProperties": false
+          },
+          "uniqueItems": true
         },
         "geoLocations": {
           "type": "array",
@@ -338,11 +358,11 @@
             "properties": {
               "geoLocationPlace": {
                 "type": "string",
-                "title": "Name of GeoLocation"
+                "title": "Spatial region or named place where the data was gathered"
               },
               "geoLocationPoint": {
                 "type": "object",
-                "title": "GeoLocation point object",
+                "title": "A point containing a single latitude-longitude pair",
                 "properties": {
                   "pointLongitude": {
                     "type": "string",
@@ -352,11 +372,12 @@
                     "type": "string",
                     "title": "Latitude coordinate"
                   }
-                }
+                },
+                "additionalProperties": false
               },
               "geoLocationBox": {
                 "type": "object",
-                "title": "GeoLocation box object",
+                "title": "A box determined by two longitude and two latitude borders",
                 "properties": {
                   "westBoundLongitude": {
                     "type": "string",
@@ -378,7 +399,7 @@
               },
               "geoLocationPolygon": {
                 "type": "array",
-                "title": "List of polygon points",
+                "title": "A drawn polygon area, defined by a set of polygon points",
                 "items": {
                   "type": "object",
                   "title": "Polygon point object",
@@ -394,19 +415,25 @@
                   }
                 }
               }
-            }
-          }
+            },
+            "additionalProperties": false
+          },
+          "uniqueItems": true
         },
         "language": {
           "type": "string",
-          "title": "Language code"
+          "title": "Code of the primary language of the resource"
         },
         "alternateIdentifiers": {
           "type": "array",
           "title": "List of alternate identifiers",
           "items": {
             "type": "object",
-            "title": "Alternate identifier object",
+            "title": "An identifier or identifiers other than the primary Identifier of the resource",
+            "required": [
+              "alternateIdentifier",
+              "alternateIdentifierType"
+            ],
             "properties": {
               "alternateIdentifier": {
                 "type": "string",
@@ -416,15 +443,22 @@
                 "type": "string",
                 "title": "Type of alternate identifier"
               }
-            }
-          }
+            },
+            "additionalProperties": false
+          },
+          "uniqueItems": true
         },
         "relatedIdentifiers": {
           "type": "array",
           "title": "List of related identifiers",
           "items": {
             "type": "object",
-            "title": "Related identifier object",
+            "title": "Identifier of related resources",
+            "required": [
+              "relatedIdentifier",
+              "relatedIdentifierType",
+              "relationType"
+            ],
             "properties": {
               "relatedIdentifier": {
                 "type": "string",
@@ -454,15 +488,17 @@
                 "type": "string",
                 "title": "Optional general type name"
               }
-            }
-          }
+            },
+            "additionalProperties": false
+          },
+          "uniqueItems": true
         },
         "sizes": {
           "type": "array",
           "title": "List of sizes",
           "items": {
             "type": "string",
-            "title": "Size or duration info about the resource"
+            "title": "Unstructured size information about the resource"
           }
         },
         "formats": {
@@ -482,7 +518,7 @@
           "title": "List of any rights information",
           "items": {
             "type": "object",
-            "title": "Rights object",
+            "title": "Any rights information about the resource",
             "properties": {
               "lang": {
                 "type": "string",
@@ -504,15 +540,17 @@
                 "type": "string",
                 "title": "URI (location) of the scheme for rights"
               }
-            }
-          }
+            },
+            "additionalProperties": false
+          },
+          "uniqueItems": true
         },
         "fundingReferences": {
           "type": "array",
           "title": "List of funding references",
           "itmes": {
             "type": "object",
-            "title": "Funding reference object",
+            "title": "Information about financial support for the resource",
             "required": [
               "funderName",
               "funderIdentifier",
@@ -547,8 +585,10 @@
                 "type": "string",
                 "title": "URI (location) of the award"
               }
-            }
-          }
+            },
+            "additionalProperties": false
+          },
+          "uniqueItems": true
         }
       },
       "additionalProperties": false

From fee36cdec403cbca235591ed817c52b47f34f0ff Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Joonatan=20M=C3=A4kinen?= <joonatan.makinen@csc.fi>
Date: Wed, 11 Aug 2021 23:43:14 +0300
Subject: [PATCH 17/25] add integration test to test new change

---
 metadata_backend/api/handlers.py |  2 +-
 tests/integration/run_tests.py   | 43 ++++++++++++++++++++++++++++++++
 2 files changed, 44 insertions(+), 1 deletion(-)

diff --git a/metadata_backend/api/handlers.py b/metadata_backend/api/handlers.py
index fb109c7a5..bf2818ee6 100644
--- a/metadata_backend/api/handlers.py
+++ b/metadata_backend/api/handlers.py
@@ -501,7 +501,7 @@ def _check_patch_folder(self, patch_ops: Any) -> None:
                     raise web.HTTPBadRequest(reason=reason)
                 pass
             else:
-                if all(i not in op["path"] for i in _required_paths + _arrays):
+                if all(i not in op["path"] for i in _required_paths + _arrays + ["/doi"]):
                     reason = f"Request contains '{op['path']}' key that cannot be updated to folders."
                     LOG.error(reason)
                     raise web.HTTPBadRequest(reason=reason)
diff --git a/tests/integration/run_tests.py b/tests/integration/run_tests.py
index 3b4b2c00a..936d07507 100644
--- a/tests/integration/run_tests.py
+++ b/tests/integration/run_tests.py
@@ -745,6 +745,48 @@ async def test_crud_folders_works_no_publish(sess):
         assert expected_true, "folder still exists at user"
 
 
+async def test_adding_doi_info_to_folder_works(sess):
+    """Test that proper DOI info can be added to folder and bad DOI info cannot be.
+
+    :param sess: HTTP session in which request call is made
+    """
+    # Create new folder and check its creation succeeded
+    folder_data = {"name": "DOI Folder", "description": "Mock Base folder for adding DOI info"}
+    folder_id = await post_folder(sess, folder_data)
+    async with sess.get(f"{folders_url}/{folder_id}") as resp:
+        LOG.debug(f"Checking that folder {folder_id} was created")
+        assert resp.status == 200, "HTTP Status code error"
+
+    # Get correctly formatted DOI info and patch it into the new folder successfully
+    doi_data = await create_request_json_data("doi", "test_doi.json")
+    patch_add_doi = [
+        {"op": "add", "path": "/doi", "value": json.loads(doi_data)}
+    ]
+    folder_id = await patch_folder(sess, folder_id, patch_add_doi)
+    async with sess.get(f"{folders_url}/{folder_id}") as resp:
+        LOG.debug(f"Checking that folder {folder_id} was patched")
+        res = await resp.json()
+        assert res["folderId"] == folder_id, "expected folder id does not match"
+        assert res["name"] == folder_data["name"], "expected folder name does not match"
+        assert res["description"] == folder_data["description"], "folder description content mismatch"
+        assert res["published"] is False, "folder is published, expected False"
+        assert res["doi"] == json.loads(doi_data), "folder doi does not match"
+
+    # Test that an incomplete DOI object fails to patch into the folder
+    patch_add_bad_doi = [
+        {"op": "add", "path": "/doi", "value": {"identifier": {}}}
+    ]
+    async with sess.patch(f"{folders_url}/{folder_id}", data=json.dumps(patch_add_bad_doi)) as resp:
+        LOG.debug(f"Tried updating folder {folder_id}")
+        # assert resp.status == 400, "HTTP Status code error"
+
+    # Delete folder
+    await delete_folder(sess, folder_id)
+    async with sess.get(f"{folders_url}/{folder_id}") as resp:
+        LOG.debug(f"Checking that folder {folder_id} was deleted")
+        assert resp.status == 404, "HTTP Status code error"
+
+
 async def test_getting_paginated_folders(sess):
     """Check that /folders returns folders with correct paginations.
 
@@ -1176,6 +1218,7 @@ async def main():
         LOG.debug("=== Testing basic CRUD folder operations ===")
         await test_crud_folders_works(sess)
         await test_crud_folders_works_no_publish(sess)
+        await test_adding_doi_info_to_folder_works(sess)
 
         # Test getting a list of folders and draft templates owned by the user
         LOG.debug("=== Testing getting folders, draft folders and draft templates with pagination ===")

From 54df2be6a45448fc7d1620717c57abebf97ddb9f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Joonatan=20M=C3=A4kinen?= <joonatan.makinen@csc.fi>
Date: Thu, 12 Aug 2021 11:27:24 +0300
Subject: [PATCH 18/25] add test doi object

---
 metadata_backend/helpers/schemas/folders.json | 22 +++---
 tests/test_files/doi/test_doi.json            | 67 +++++++++++++++++++
 2 files changed, 78 insertions(+), 11 deletions(-)
 create mode 100644 tests/test_files/doi/test_doi.json

diff --git a/metadata_backend/helpers/schemas/folders.json b/metadata_backend/helpers/schemas/folders.json
index 84553b4f5..667f6587a 100644
--- a/metadata_backend/helpers/schemas/folders.json
+++ b/metadata_backend/helpers/schemas/folders.json
@@ -93,15 +93,15 @@
                   "title": "Name identifier object",
                   "properties": {
                     "schemeUri": {
-                      "type": "string",
+                      "type": ["string", "null"],
                       "title": "URI (location) of the name identifier scheme"
                     },
                     "nameIdentifier": {
-                      "type": "string",
+                      "type": ["string", "null"],
                       "title": "URI (location) of name identifier"
                     },
                     "nameIdentifierScheme": {
-                      "type": "string",
+                      "type": ["string", "null"],
                       "title": "Name of name identifier scheme"
                     }
                   }
@@ -156,7 +156,7 @@
                 "title": "A name or title by which a resource is known"
               },
               "titleType": {
-                "type": "string",
+                "type": ["string", "null"],
                 "title": "Type of title"
               }
             },
@@ -252,15 +252,15 @@
                   "title": "Name identifier object",
                   "properties": {
                     "schemeUri": {
-                      "type": "string",
+                      "type": ["string", "null"],
                       "title": "URI (location) of the name identifier scheme"
                     },
                     "nameIdentifier": {
-                      "type": "string",
+                      "type": ["string", "null"],
                       "title": "Location of name identifier"
                     },
                     "nameIdentifierScheme": {
-                      "type": "string",
+                      "type": ["string", "null"],
                       "title": "Name of name identifier scheme"
                     }
                   }
@@ -570,19 +570,19 @@
                 "title": "Type of identifier for funding entity"
               },
               "schemeUri": {
-                "type": "string",
+                "type": ["string", "null"],
                 "title": "URI (location) of scheme for funder identifier"
               },
               "awardNumber": {
-                "type": "string",
+                "type": ["string", "null"],
                 "title": "The code assigned by the funder to a sponsored award"
               },
               "awardTitle": {
-                "type": "string",
+                "type": ["string", "null"],
                 "title": "The human readable title of the award"
               },
               "awardUri": {
-                "type": "string",
+                "type": ["string", "null"],
                 "title": "URI (location) of the award"
               }
             },
diff --git a/tests/test_files/doi/test_doi.json b/tests/test_files/doi/test_doi.json
new file mode 100644
index 000000000..755822e92
--- /dev/null
+++ b/tests/test_files/doi/test_doi.json
@@ -0,0 +1,67 @@
+{
+  "identifier": {
+    "doi": "10.xxxx/test",
+    "identifierType": "DOI"
+  },
+  "url": "https://some.test.url",
+  "resourceType": {
+    "type": "XML",
+    "resourceTypeGeneral": "Dataset"
+  },
+  "creators": [
+    {
+      "name": "Creator, Test",
+      "nameType": "Personal",
+      "givenName": "Test",
+      "familyName": "Creator",
+      "affiliation": [
+        {
+          "name": "affiliation place",
+          "schemeUri": "https://ror.org",
+          "affiliationIdentifier": "https://ror.org/test1",
+          "affiliationIdentifierScheme": "ROR"
+        }
+      ],
+      "nameIdentifiers": []
+    }
+  ],
+  "titles": [
+    {
+      "lang": "en",
+      "title": "Test resource",
+      "titleType": null
+    }
+  ],
+  "publisher": "Test publisher",
+  "subjects": [
+    {
+      "subject": "FOS: Computer and information sciences",
+      "subjectScheme": "Fields of Science and Technology (FOS)"
+    }
+  ],
+  "publicationYear": 2021,
+  "contributors": [
+    {
+      "name": "Contributor, Test",
+      "nameType": "Personal",
+      "givenName": "Test",
+      "familyName": "contributor",
+      "affiliation": [
+        {
+          "name": "affiliation place",
+          "schemeUri": "https://ror.org",
+          "affiliationIdentifier": "https://ror.org/test2",
+          "affiliationIdentifierScheme": "ROR"
+        }
+      ],
+      "contributorType": "Researcher",
+      "nameIdentifiers": [
+        {
+          "schemeUri": null,
+          "nameIdentifier": null,
+          "nameIdentifierScheme": null
+        }
+      ]
+    }
+  ]
+}

From 754e1c3e18687d7eea2bbfcf7577cacc153acbfa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Joonatan=20M=C3=A4kinen?= <joonatan.makinen@csc.fi>
Date: Thu, 12 Aug 2021 11:29:05 +0300
Subject: [PATCH 19/25] make the integration test work

---
 metadata_backend/api/handlers.py | 13 ++++++++++---
 tests/integration/run_tests.py   | 24 +++++++++++++++---------
 2 files changed, 25 insertions(+), 12 deletions(-)

diff --git a/metadata_backend/api/handlers.py b/metadata_backend/api/handlers.py
index bf2818ee6..c1b2a0943 100644
--- a/metadata_backend/api/handlers.py
+++ b/metadata_backend/api/handlers.py
@@ -643,12 +643,19 @@ async def patch_folder(self, req: Request) -> Response:
         patch_ops = await self._get_data(req)
         self._check_patch_folder(patch_ops)
 
+        # Validate against folders schema if DOI is being added
+        for op in patch_ops:
+            if op["path"] == "/doi":
+                curr_folder = await operator.read_folder(folder_id)
+                curr_folder["doi"] = op["value"]
+                JSONValidator(curr_folder, "folders").validate
+
         await self._handle_check_ownedby_user(req, "folders", folder_id)
 
-        folder = await operator.update_folder(folder_id, patch_ops if isinstance(patch_ops, list) else [patch_ops])
+        upd_folder = await operator.update_folder(folder_id, patch_ops if isinstance(patch_ops, list) else [patch_ops])
 
-        body = json.dumps({"folderId": folder})
-        LOG.info(f"PATCH folder with ID {folder} was successful.")
+        body = json.dumps({"folderId": upd_folder})
+        LOG.info(f"PATCH folder with ID {upd_folder} was successful.")
         return web.Response(body=body, status=200, content_type="application/json")
 
     async def publish_folder(self, req: Request) -> Response:
diff --git a/tests/integration/run_tests.py b/tests/integration/run_tests.py
index 936d07507..79d066918 100644
--- a/tests/integration/run_tests.py
+++ b/tests/integration/run_tests.py
@@ -758,11 +758,11 @@ async def test_adding_doi_info_to_folder_works(sess):
         assert resp.status == 200, "HTTP Status code error"
 
     # Get correctly formatted DOI info and patch it into the new folder successfully
-    doi_data = await create_request_json_data("doi", "test_doi.json")
-    patch_add_doi = [
-        {"op": "add", "path": "/doi", "value": json.loads(doi_data)}
-    ]
+    doi_data_raw = await create_request_json_data("doi", "test_doi.json")
+    doi_data = json.loads(doi_data_raw)
+    patch_add_doi = [{"op": "add", "path": "/doi", "value": doi_data}]
     folder_id = await patch_folder(sess, folder_id, patch_add_doi)
+
     async with sess.get(f"{folders_url}/{folder_id}") as resp:
         LOG.debug(f"Checking that folder {folder_id} was patched")
         res = await resp.json()
@@ -770,15 +770,21 @@ async def test_adding_doi_info_to_folder_works(sess):
         assert res["name"] == folder_data["name"], "expected folder name does not match"
         assert res["description"] == folder_data["description"], "folder description content mismatch"
         assert res["published"] is False, "folder is published, expected False"
-        assert res["doi"] == json.loads(doi_data), "folder doi does not match"
+        assert res["doi"] == doi_data, "folder doi does not match"
 
     # Test that an incomplete DOI object fails to patch into the folder
-    patch_add_bad_doi = [
-        {"op": "add", "path": "/doi", "value": {"identifier": {}}}
-    ]
+    patch_add_bad_doi = [{"op": "add", "path": "/doi", "value": {"identifier": {}}}]
     async with sess.patch(f"{folders_url}/{folder_id}", data=json.dumps(patch_add_bad_doi)) as resp:
         LOG.debug(f"Tried updating folder {folder_id}")
-        # assert resp.status == 400, "HTTP Status code error"
+        assert resp.status == 400, "HTTP Status code error"
+        res = await resp.json()
+        assert res["detail"] == "Provided input does not seem correct for field: 'doi'", "expected error does not match"
+
+    # Check the existing DOI info is not altered
+    async with sess.get(f"{folders_url}/{folder_id}") as resp:
+        LOG.debug(f"Checking that folder {folder_id} was not patched with bad DOI")
+        res = await resp.json()
+        assert res["doi"] == doi_data, "folder doi does not match"
 
     # Delete folder
     await delete_folder(sess, folder_id)

From d9108ae5bf4ddd18e27dd355454465400df13992 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Joonatan=20M=C3=A4kinen?= <joonatan.makinen@csc.fi>
Date: Mon, 23 Aug 2021 08:59:04 +0300
Subject: [PATCH 20/25] split doi schema into doiInfo and extraInfo

---
 metadata_backend/helpers/schemas/folders.json | 177 ++++++------------
 1 file changed, 59 insertions(+), 118 deletions(-)

diff --git a/metadata_backend/helpers/schemas/folders.json b/metadata_backend/helpers/schemas/folders.json
index 667f6587a..a5e81483d 100644
--- a/metadata_backend/helpers/schemas/folders.json
+++ b/metadata_backend/helpers/schemas/folders.json
@@ -26,42 +26,14 @@
       "type": "boolean",
       "title": "Published Folder"
     },
-    "doi": {
+    "doiInfo": {
       "type": "object",
-      "title": "The DOI schema",
+      "title": "The DOI info schema",
       "required": [
-        "identifier",
-        "url",
         "creators",
-        "titles",
-        "publisher",
-        "subjects",
-        "publicationYear",
-        "resourceType"
+        "subjects"
       ],
       "properties": {
-        "identifier": {
-          "type": "object",
-          "title": "identifier object",
-          "required": [
-            "identifierType",
-            "doi"
-          ],
-          "properties": {
-            "identifierType": {
-              "type": "string",
-              "title": "Type of identifier (= DOI)"
-            },
-            "doi": {
-              "type": "string",
-              "title": "A persistent identifier for a resource"
-            }
-          }
-        },
-        "url": {
-          "type": "string",
-          "title": "URL of the digital location of the object"
-        },
         "creators": {
           "type": "array",
           "title": "List of creators",
@@ -140,34 +112,6 @@
           },
           "uniqueItems": true
         },
-        "titles": {
-          "type": "array",
-          "title": "List of titles",
-          "items": {
-            "type": "object",
-            "title": "Title objects",
-            "properties": {
-              "lang": {
-                "type": "string",
-                "title": "Language code of the title"
-              },
-              "title": {
-                "type": "string",
-                "title": "A name or title by which a resource is known"
-              },
-              "titleType": {
-                "type": ["string", "null"],
-                "title": "Type of title"
-              }
-            },
-            "additionalProperties": false
-          },
-          "uniqueItems": true
-        },
-        "publisher": {
-          "type": "string",
-          "title": "Full name of publisher from Research Organization Registry"
-        },
         "subjects": {
           "type": "array",
           "title": "List of subject identifiers specified by FOS",
@@ -191,29 +135,6 @@
           },
           "uniqueItems": true
         },
-        "publicationYear": {
-          "type": "integer",
-          "title": "Year when the data is made publicly available"
-        },
-        "resourceType": {
-          "type": "object",
-          "title": "Type info of the resource",
-          "required": [
-            "type",
-            "resourceTypeGeneral"
-          ],
-          "properties": {
-            "type": {
-              "type": "string",
-              "title": "Name of resource type"
-            },
-            "resourceTypeGeneral": {
-              "type": "string",
-              "title": "Mandatory general type name"
-            }
-          },
-          "additionalProperties": false
-        },
         "contributors": {
           "type": "array",
           "title": "List of contributors",
@@ -509,42 +430,6 @@
             "title": "Technical format of the resource"
           }
         },
-        "version": {
-          "type": "string",
-          "title": "Version number of the resource"
-        },
-        "rightsList": {
-          "type": "array",
-          "title": "List of any rights information",
-          "items": {
-            "type": "object",
-            "title": "Any rights information about the resource",
-            "properties": {
-              "lang": {
-                "type": "string",
-                "title": "Language code of the referenced rights"
-              },
-              "rightsUri": {
-                "type": "string",
-                "title": "URI (location) of the referenced rights"
-              },
-              "rightsIdentifier": {
-                "type": "string",
-                "title": "Title of the rights identifier"
-              },
-              "rightsIdentifierScheme": {
-                "type": "string",
-                "title": "Title of rights identifier scheme"
-              },
-              "schemeUri": {
-                "type": "string",
-                "title": "URI (location) of the scheme for rights"
-              }
-            },
-            "additionalProperties": false
-          },
-          "uniqueItems": true
-        },
         "fundingReferences": {
           "type": "array",
           "title": "List of funding references",
@@ -593,6 +478,62 @@
       },
       "additionalProperties": false
     },
+    "extraInfo": {
+      "type": "object",
+      "title": "The extra DOI info schema",
+      "properties": {
+        "identifier": {
+          "type": "object",
+          "title": "identifier object",
+          "required": [
+            "identifierType",
+            "doi"
+          ],
+          "properties": {
+            "identifierType": {
+              "type": "string",
+              "title": "Type of identifier (= DOI)"
+            },
+            "doi": {
+              "type": "string",
+              "title": "A persistent identifier for a resource"
+            }
+          }
+        },
+        "publisher": {
+          "type": "string",
+          "title": "Full name of publisher from Research Organization Registry"
+        },
+        "resourceType": {
+          "type": "object",
+          "title": "Type info of the resource",
+          "required": [
+            "type",
+            "resourceTypeGeneral"
+          ],
+          "properties": {
+            "type": {
+              "type": "string",
+              "title": "Name of resource type"
+            },
+            "resourceTypeGeneral": {
+              "type": "string",
+              "title": "Mandatory general type name"
+            }
+          },
+          "additionalProperties": false
+        },
+        "url": {
+          "type": "string",
+          "title": "URL of the digital location of the object"
+        },
+        "version": {
+          "type": "string",
+          "title": "Version number of the resource"
+        }
+      },
+      "additionalProperties": false
+    },
     "metadataObjects": {
       "type": "array",
       "title": "The metadataObjects schema",

From 699741e81b286a19564f4ad5c401d8dfc3637189 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Joonatan=20M=C3=A4kinen?= <joonatan.makinen@csc.fi>
Date: Mon, 23 Aug 2021 08:59:41 +0300
Subject: [PATCH 21/25] alter handler and tests according to changed schema

---
 metadata_backend/api/handlers.py   |  6 +++---
 tests/integration/run_tests.py     | 18 +++++++++++++-----
 tests/test_files/doi/test_doi.json | 18 ------------------
 3 files changed, 16 insertions(+), 26 deletions(-)

diff --git a/metadata_backend/api/handlers.py b/metadata_backend/api/handlers.py
index c1b2a0943..38a6b66bf 100644
--- a/metadata_backend/api/handlers.py
+++ b/metadata_backend/api/handlers.py
@@ -501,7 +501,7 @@ def _check_patch_folder(self, patch_ops: Any) -> None:
                     raise web.HTTPBadRequest(reason=reason)
                 pass
             else:
-                if all(i not in op["path"] for i in _required_paths + _arrays + ["/doi"]):
+                if all(i not in op["path"] for i in _required_paths + _arrays + ["/doiInfo"]):
                     reason = f"Request contains '{op['path']}' key that cannot be updated to folders."
                     LOG.error(reason)
                     raise web.HTTPBadRequest(reason=reason)
@@ -645,9 +645,9 @@ async def patch_folder(self, req: Request) -> Response:
 
         # Validate against folders schema if DOI is being added
         for op in patch_ops:
-            if op["path"] == "/doi":
+            if op["path"] == "/doiInfo":
                 curr_folder = await operator.read_folder(folder_id)
-                curr_folder["doi"] = op["value"]
+                curr_folder["doiInfo"] = op["value"]
                 JSONValidator(curr_folder, "folders").validate
 
         await self._handle_check_ownedby_user(req, "folders", folder_id)
diff --git a/tests/integration/run_tests.py b/tests/integration/run_tests.py
index 79d066918..06fb5a483 100644
--- a/tests/integration/run_tests.py
+++ b/tests/integration/run_tests.py
@@ -760,7 +760,7 @@ async def test_adding_doi_info_to_folder_works(sess):
     # Get correctly formatted DOI info and patch it into the new folder successfully
     doi_data_raw = await create_request_json_data("doi", "test_doi.json")
     doi_data = json.loads(doi_data_raw)
-    patch_add_doi = [{"op": "add", "path": "/doi", "value": doi_data}]
+    patch_add_doi = [{"op": "add", "path": "/doiInfo", "value": doi_data}]
     folder_id = await patch_folder(sess, folder_id, patch_add_doi)
 
     async with sess.get(f"{folders_url}/{folder_id}") as resp:
@@ -770,21 +770,29 @@ async def test_adding_doi_info_to_folder_works(sess):
         assert res["name"] == folder_data["name"], "expected folder name does not match"
         assert res["description"] == folder_data["description"], "folder description content mismatch"
         assert res["published"] is False, "folder is published, expected False"
-        assert res["doi"] == doi_data, "folder doi does not match"
+        assert res["doiInfo"] == doi_data, "folder doi does not match"
 
     # Test that an incomplete DOI object fails to patch into the folder
-    patch_add_bad_doi = [{"op": "add", "path": "/doi", "value": {"identifier": {}}}]
+    patch_add_bad_doi = [{"op": "replace", "path": "/doiInfo", "value": {"identifier": {}}}]
     async with sess.patch(f"{folders_url}/{folder_id}", data=json.dumps(patch_add_bad_doi)) as resp:
         LOG.debug(f"Tried updating folder {folder_id}")
         assert resp.status == 400, "HTTP Status code error"
         res = await resp.json()
-        assert res["detail"] == "Provided input does not seem correct for field: 'doi'", "expected error does not match"
+        assert res["detail"] == "Provided input does not seem correct for field: 'doiInfo'", "expected error mismatch"
 
     # Check the existing DOI info is not altered
     async with sess.get(f"{folders_url}/{folder_id}") as resp:
         LOG.debug(f"Checking that folder {folder_id} was not patched with bad DOI")
         res = await resp.json()
-        assert res["doi"] == doi_data, "folder doi does not match"
+        assert res["doiInfo"] == doi_data, "folder doi does not match"
+
+    # Test that extraInfo cannot be altered
+    patch_add_bad_doi = [{"op": "add", "path": "/extraInfo", "value": {"publisher": "something"}}]
+    async with sess.patch(f"{folders_url}/{folder_id}", data=json.dumps(patch_add_bad_doi)) as resp:
+        LOG.debug(f"Tried updating folder {folder_id}")
+        assert resp.status == 400, "HTTP Status code error"
+        res = await resp.json()
+        assert res["detail"] == "Request contains '/extraInfo' key that cannot be updated to folders.", "error mismatch"
 
     # Delete folder
     await delete_folder(sess, folder_id)
diff --git a/tests/test_files/doi/test_doi.json b/tests/test_files/doi/test_doi.json
index 755822e92..3f3257dec 100644
--- a/tests/test_files/doi/test_doi.json
+++ b/tests/test_files/doi/test_doi.json
@@ -1,13 +1,4 @@
 {
-  "identifier": {
-    "doi": "10.xxxx/test",
-    "identifierType": "DOI"
-  },
-  "url": "https://some.test.url",
-  "resourceType": {
-    "type": "XML",
-    "resourceTypeGeneral": "Dataset"
-  },
   "creators": [
     {
       "name": "Creator, Test",
@@ -25,21 +16,12 @@
       "nameIdentifiers": []
     }
   ],
-  "titles": [
-    {
-      "lang": "en",
-      "title": "Test resource",
-      "titleType": null
-    }
-  ],
-  "publisher": "Test publisher",
   "subjects": [
     {
       "subject": "FOS: Computer and information sciences",
       "subjectScheme": "Fields of Science and Technology (FOS)"
     }
   ],
-  "publicationYear": 2021,
   "contributors": [
     {
       "name": "Contributor, Test",

From df724be7bcb5fd09cc79f8ae20ce56bb36ea044e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Joonatan=20M=C3=A4kinen?= <joonatan.makinen@csc.fi>
Date: Mon, 23 Aug 2021 14:49:13 +0300
Subject: [PATCH 22/25] add extraInfo in openapi specs

---
 docs/specification.yml | 82 ++++++++++++++++++------------------------
 1 file changed, 34 insertions(+), 48 deletions(-)

diff --git a/docs/specification.yml b/docs/specification.yml
index c28617304..f054db993 100644
--- a/docs/specification.yml
+++ b/docs/specification.yml
@@ -1290,7 +1290,8 @@ components:
         - published
         - metadataObjects
         - drafts
-        - doi
+        - doiInfo
+        - extraInfo
       additionalProperties: false
       properties:
         folderId:
@@ -1305,38 +1306,13 @@ components:
         published:
           type: boolean
           description: If folder is published or not
-        doi:
+        doiInfo:
           type: object
           required:
-            - identifier
-            - url
             - creators
             - titles
-            - publisher
             - subjects
-            - publicationYear
-            - resourceType
           properties:
-            identifier:
-              type: object
-              description: DOI id generated according to DOI recommendations
-              required:
-                - identifierType
-              properties:
-                identifierType:
-                  type: string
-                  example: "DOI"
-                doi:
-                  type: string
-                  description: Character string of DOI handle
-                  example: "prefix/suffix"
-                id:
-                  type: string
-                  description: Display url for DOI id
-                  example: "https://doi.org/prefix/suffix"
-            url:
-              type: string
-              description: URL of the digital location of the object
             creators:
               type: array
               items:
@@ -1366,23 +1342,6 @@ components:
                     type: array
                     items:
                       type: object
-            titles:
-              type: array
-              items:
-                type: object
-                properties:
-                  lang:
-                    type: string
-                    description: Language of the title
-                    example: "en"
-                  title:
-                    type: string
-                    description: Full title
-                  titleType:
-                    type: string
-            publisher:
-              type: string
-              description: Full name of publisher from Research Organization Registry
             subjects:
               type: array
               items:
@@ -1396,10 +1355,37 @@ components:
                     type: string
                     description: Subject scheme name
                     example: "Fields of Science and Technology (FOS)"
-            publicationYear:
-              type: integer
-              description: Year of publication
-              example: 2021
+        extraInfo:
+          type: object
+          required:
+            - identifier
+            - url
+            - resourceType
+            - publisher
+          properties:
+            identifier:
+              type: object
+              description: DOI id generated according to DOI recommendations
+              required:
+                - identifierType
+              properties:
+                identifierType:
+                  type: string
+                  example: "DOI"
+                doi:
+                  type: string
+                  description: Character string of DOI handle
+                  example: "prefix/suffix"
+                id:
+                  type: string
+                  description: Display url for DOI id
+                  example: "https://doi.org/prefix/suffix"
+            url:
+              type: string
+              description: URL of the digital location of the object
+            publisher:
+              type: string
+              description: Full name of publisher from Research Organization Registry
             resourceType:
               type: object
               required:

From f90fd982c1f668888147d8b51e661ac0d344e8d1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Joonatan=20M=C3=A4kinen?= <joonatan.makinen@csc.fi>
Date: Mon, 23 Aug 2021 20:07:56 +0300
Subject: [PATCH 23/25] do folder patch check differently

---
 metadata_backend/api/handlers.py | 6 +++---
 tests/integration/run_tests.py   | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/metadata_backend/api/handlers.py b/metadata_backend/api/handlers.py
index 38a6b66bf..a0ce381f0 100644
--- a/metadata_backend/api/handlers.py
+++ b/metadata_backend/api/handlers.py
@@ -489,7 +489,7 @@ def _check_patch_folder(self, patch_ops: Any) -> None:
         """
         _required_paths = ["/name", "/description"]
         _required_values = ["schema", "accessionId"]
-        _arrays = ["/metadataObjects/-", "/drafts/-"]
+        _arrays = ["/metadataObjects/-", "/drafts/-", "/doiInfo"]
         _tags = re.compile("^/(metadataObjects|drafts)/[0-9]*/(tags)$")
 
         for op in patch_ops:
@@ -501,7 +501,7 @@ def _check_patch_folder(self, patch_ops: Any) -> None:
                     raise web.HTTPBadRequest(reason=reason)
                 pass
             else:
-                if all(i not in op["path"] for i in _required_paths + _arrays + ["/doiInfo"]):
+                if all(i not in op["path"] for i in _required_paths + _arrays):
                     reason = f"Request contains '{op['path']}' key that cannot be updated to folders."
                     LOG.error(reason)
                     raise web.HTTPBadRequest(reason=reason)
@@ -513,7 +513,7 @@ def _check_patch_folder(self, patch_ops: Any) -> None:
                     reason = f"{op['op']} on {op['path']}; replacing all objects is not allowed."
                     LOG.error(reason)
                     raise web.HTTPUnauthorized(reason=reason)
-                if op["path"] in _arrays:
+                if op["path"] in _arrays and op["path"] != "/doiInfo":
                     _ops = op["value"] if isinstance(op["value"], list) else [op["value"]]
                     for item in _ops:
                         if not all(key in item.keys() for key in _required_values):
diff --git a/tests/integration/run_tests.py b/tests/integration/run_tests.py
index 06fb5a483..f1b56c7aa 100644
--- a/tests/integration/run_tests.py
+++ b/tests/integration/run_tests.py
@@ -773,7 +773,7 @@ async def test_adding_doi_info_to_folder_works(sess):
         assert res["doiInfo"] == doi_data, "folder doi does not match"
 
     # Test that an incomplete DOI object fails to patch into the folder
-    patch_add_bad_doi = [{"op": "replace", "path": "/doiInfo", "value": {"identifier": {}}}]
+    patch_add_bad_doi = [{"op": "add", "path": "/doiInfo", "value": {"identifier": {}}}]
     async with sess.patch(f"{folders_url}/{folder_id}", data=json.dumps(patch_add_bad_doi)) as resp:
         LOG.debug(f"Tried updating folder {folder_id}")
         assert resp.status == 400, "HTTP Status code error"

From a45f9c73635a0aef31fa5ab1f9029c16ac00fd52 Mon Sep 17 00:00:00 2001
From: Stefan Negru <stefan@blankdots.com>
Date: Mon, 30 Aug 2021 08:48:11 +0300
Subject: [PATCH 24/25] fix type for date created to integer not int

---
 metadata_backend/helpers/schemas/folders.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/metadata_backend/helpers/schemas/folders.json b/metadata_backend/helpers/schemas/folders.json
index a5e81483d..b1b4a9243 100644
--- a/metadata_backend/helpers/schemas/folders.json
+++ b/metadata_backend/helpers/schemas/folders.json
@@ -19,7 +19,7 @@
       "title": "Folder Description"
     },
     "dateCreated": {
-      "type": "int",
+      "type": "integer",
       "title": "Unix time stamp of creation, used for indexing"
     },
     "published": {

From 2890a94f53a7c8921e6fb408617be8bb61ea9383 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 30 Aug 2021 09:06:03 +0000
Subject: [PATCH 25/25] Bump cryptography from 3.4.7 to 3.4.8

Bumps [cryptography](https://github.com/pyca/cryptography) from 3.4.7 to 3.4.8.
- [Release notes](https://github.com/pyca/cryptography/releases)
- [Changelog](https://github.com/pyca/cryptography/blob/main/CHANGELOG.rst)
- [Commits](https://github.com/pyca/cryptography/compare/3.4.7...3.4.8)

---
updated-dependencies:
- dependency-name: cryptography
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 29b627adc..191c5e38a 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,5 +1,5 @@
 aiohttp==3.7.4.post0
-cryptography==3.4.7
+cryptography==3.4.8
 gunicorn==20.1.0
 jsonschema==3.2.0
 motor==2.5.1