Merge pull request #197 from CSCfi/feature/allow_put_xml_objects

allow put XML objects

metadata_backend/api/handlers.py

@@ -402,7 +402,9 @@ async def put_object(self, req: Request) -> Response:
         else:
             content = await self._get_data(req)
             if not req.path.startswith("/drafts"):
-                JSONValidator(content, schema_type).validate
+                reason = "Replacing objects only allowed for XML."
+                LOG.error(reason)
+                raise web.HTTPUnsupportedMediaType(reason=reason)
             operator = Operator(db_client)
 
         await operator.check_exists(collection, accession_id)

metadata_backend/server.py

@@ -65,6 +65,7 @@ async def init() -> web.Application:
         web.delete("/objects/{schema}/{accessionId}", rest_handler.delete_object),
         web.get("/objects/{schema}", rest_handler.query_objects),
         web.post("/objects/{schema}", rest_handler.post_object),
+        web.put("/objects/{schema}/{accessionId}", rest_handler.put_object),
         web.get("/drafts/{schema}/{accessionId}", rest_handler.get_object),
         web.put("/drafts/{schema}/{accessionId}", rest_handler.put_object),
         web.patch("/drafts/{schema}/{accessionId}", rest_handler.patch_object),

tests/integration/run_tests.py

@@ -17,9 +17,7 @@
 from aiohttp import FormData
 
 # === Global vars ===
-FORMAT = (
-    "[%(asctime)s][%(name)s][%(process)d %(processName)s]" "[%(levelname)-8s](L:%(lineno)s) %(funcName)s: %(message)s"
-)
+FORMAT = "[%(asctime)s][%(name)s][%(process)d %(processName)s][%(levelname)-8s](L:%(lineno)s) %(funcName)s: %(message)s"
 logging.basicConfig(format=FORMAT, datefmt="%Y-%m-%d %H:%M:%S")
 LOG = logging.getLogger(__name__)
 LOG.setLevel(logging.DEBUG)
@@ -81,26 +79,26 @@ async def create_request_data(schema, filename):
     :param schema: name of the schema (folder) used for testing
     :param filename: name of the file used for testing.
     """
-    data = FormData()
+    request_data = FormData()
     path_to_file = testfiles_root / schema / filename
     path = path_to_file.as_posix()
     async with aiofiles.open(path, mode="r") as f:
-        data.add_field(schema.upper(), await f.read(), filename=filename, content_type="text/xml")
-    return data
+        request_data.add_field(schema.upper(), await f.read(), filename=filename, content_type="text/xml")
+    return request_data
 
 
 async def create_multi_file_request_data(filepairs):
     """Create request data with multiple files.
 
     :param filepairs: tuple containing pairs of schemas and filenames used for testing
     """
-    data = FormData()
+    request_data = FormData()
     for schema, filename in filepairs:
         path_to_file = testfiles_root / schema / filename
         path = path_to_file.as_posix()
         async with aiofiles.open(path, mode="r") as f:
-            data.add_field(schema.upper(), await f.read(), filename=filename, content_type="text/xml")
-    return data
+            request_data.add_field(schema.upper(), await f.read(), filename=filename, content_type="text/xml")
+    return request_data
 
 
 async def create_request_json_data(schema, filename):
@@ -112,8 +110,8 @@ async def create_request_json_data(schema, filename):
     path_to_file = testfiles_root / schema / filename
     path = path_to_file.as_posix()
     async with aiofiles.open(path, mode="r") as f:
-        data = await f.read()
-    return data
+        request_data = await f.read()
+    return request_data
 
 
 async def post_object(sess, schema, filename):
@@ -123,8 +121,8 @@ async def post_object(sess, schema, filename):
     :param schema: name of the schema (folder) used for testing
     :param filename: name of the file used for testing.
     """
-    data = await create_request_data(schema, filename)
-    async with sess.post(f"{objects_url}/{schema}", data=data) as resp:
+    request_data = await create_request_data(schema, filename)
+    async with sess.post(f"{objects_url}/{schema}", data=request_data) as resp:
         LOG.debug(f"Adding new object to {schema}")
         assert resp.status == 201, "HTTP Status code error"
         ans = await resp.json()
@@ -138,8 +136,8 @@ async def post_object_json(sess, schema, filename):
     :param schema: name of the schema (folder) used for testing
     :param filename: name of the file used for testing.
     """
-    data = await create_request_json_data(schema, filename)
-    async with sess.post(f"{objects_url}/{schema}", data=data) as resp:
+    request_data = await create_request_json_data(schema, filename)
+    async with sess.post(f"{objects_url}/{schema}", data=request_data) as resp:
         LOG.debug(f"Adding new draft object to {schema}")
         assert resp.status == 201, "HTTP Status code error"
         ans = await resp.json()
@@ -165,8 +163,8 @@ async def post_draft(sess, schema, filename):
     :param schema: name of the schema (folder) used for testing
     :param filename: name of the file used for testing.
     """
-    data = await create_request_data(schema, filename)
-    async with sess.post(f"{drafts_url}/{schema}", data=data) as resp:
+    request_data = await create_request_data(schema, filename)
+    async with sess.post(f"{drafts_url}/{schema}", data=request_data) as resp:
         LOG.debug(f"Adding new object to {schema}")
         assert resp.status == 201, "HTTP Status code error"
         ans = await resp.json()
@@ -180,8 +178,8 @@ async def post_draft_json(sess, schema, filename):
     :param schema: name of the schema (folder) used for testing
     :param filename: name of the file used for testing.
     """
-    data = await create_request_json_data(schema, filename)
-    async with sess.post(f"{drafts_url}/{schema}", data=data) as resp:
+    request_data = await create_request_json_data(schema, filename)
+    async with sess.post(f"{drafts_url}/{schema}", data=request_data) as resp:
         LOG.debug(f"Adding new draft object to {schema}")
         assert resp.status == 201, "HTTP Status code error"
         ans = await resp.json()
@@ -210,15 +208,46 @@ async def put_draft(sess, schema, draft_id, update_filename):
     :param draft_id: id of the draft
     :param update_filename: name of the file used to use for updating data.
     """
-    data2 = await create_request_json_data(schema, update_filename)
-    async with sess.put(f"{drafts_url}/{schema}/{draft_id}", data=data2) as resp:
+    request_data = await create_request_json_data(schema, update_filename)
+    async with sess.put(f"{drafts_url}/{schema}/{draft_id}", data=request_data) as resp:
         LOG.debug(f"Replace draft object in {schema}")
         assert resp.status == 200, "HTTP Status code error"
         ans_put = await resp.json()
         assert ans_put["accessionId"] == draft_id, "accession ID error"
         return ans_put["accessionId"]
 
 
+async def put_object_json(sess, schema, accession_id, update_filename):
+    """Put one metadata object within session, returns accessionId.
+
+    :param sess: HTTP session in which request call is made
+    :param schema: name of the schema (folder) used for testing
+    :param draft_id: id of the draft
+    :param update_filename: name of the file used to use for updating data.
+    """
+    request_data = await create_request_json_data(schema, update_filename)
+    async with sess.put(f"{objects_url}/{schema}/{accession_id}", data=request_data) as resp:
+        LOG.debug(f"Try to replace object in {schema}")
+        assert resp.status == 415, "HTTP Status code error"
+
+
+async def put_object_xml(sess, schema, accession_id, update_filename):
+    """Put one metadata object within session, returns accessionId.
+
+    :param sess: HTTP session in which request call is made
+    :param schema: name of the schema (folder) used for testing
+    :param draft_id: id of the draft
+    :param update_filename: name of the file used to use for updating data.
+    """
+    request_data = await create_request_data(schema, update_filename)
+    async with sess.put(f"{objects_url}/{schema}/{accession_id}", data=request_data) as resp:
+        LOG.debug(f"Replace object with XML data in {schema}")
+        assert resp.status == 200, "HTTP Status code error"
+        ans_put = await resp.json()
+        assert ans_put["accessionId"] == accession_id, "accession ID error"
+        return ans_put["accessionId"]
+
+
 async def patch_draft(sess, schema, draft_id, update_filename):
     """Patch one metadata object within session, return accessionId.
 
@@ -227,8 +256,8 @@ async def patch_draft(sess, schema, draft_id, update_filename):
     :param draft_id: id of the draft
     :param update_filename: name of the file used to use for updating data.
     """
-    data = await create_request_json_data(schema, update_filename)
-    async with sess.patch(f"{drafts_url}/{schema}/{draft_id}", data=data) as resp:
+    request_data = await create_request_json_data(schema, update_filename)
+    async with sess.patch(f"{drafts_url}/{schema}/{draft_id}", data=request_data) as resp:
         LOG.debug(f"Update draft object in {schema}")
         assert resp.status == 200, "HTTP Status code error"
         ans_put = await resp.json()
@@ -362,15 +391,15 @@ async def test_crud_works(sess, schema, filename, folder_id):
     async with sess.get(f"{objects_url}/{schema}/{accession_id[0]}") as resp:
         LOG.debug(f"Checking that {accession_id[0]} JSON is in {schema}")
         assert resp.status == 200, "HTTP Status code error"
-    async with sess.get(f"{objects_url}/{schema}/{accession_id[0]}" "?format=xml") as resp:
+    async with sess.get(f"{objects_url}/{schema}/{accession_id[0]}?format=xml") as resp:
         LOG.debug(f"Checking that {accession_id[0]} XML is in {schema}")
         assert resp.status == 200, "HTTP Status code error"
 
     await delete_object(sess, schema, accession_id[0])
     async with sess.get(f"{objects_url}/{schema}/{accession_id[0]}") as resp:
         LOG.debug(f"Checking that JSON object {accession_id[0]} was deleted")
         assert resp.status == 404, "HTTP Status code error"
-    async with sess.get(f"{objects_url}/{schema}/{accession_id[0]}" "?format=xml") as resp:
+    async with sess.get(f"{objects_url}/{schema}/{accession_id[0]}?format=xml") as resp:
         LOG.debug(f"Checking that XML object {accession_id[0]} was deleted")
         assert resp.status == 404, "HTTP Status code error"
 
@@ -381,6 +410,25 @@ async def test_crud_works(sess, schema, filename, folder_id):
         assert expected_true, "draft object still exists"
 
 
+async def test_put_objects(sess, folder_id):
+    """Test PUT reqs.
+
+    Tries to create new object, gets accession id and checks if correct
+    resource is returned with that id. Try to use PUT with JSON and expect failure,
+    try to use PUT with xml and expect success.
+
+    :param sess: HTTP session in which request call is made
+    :param folder_id: id of the folder used to group submission
+    """
+    accession_id = await post_object(sess, "study", "SRP000539.xml")
+    patch_object = [
+        {"op": "add", "path": "/metadataObjects/-", "value": {"accessionId": accession_id[0], "schema": "study"}}
+    ]
+    await patch_folder(sess, folder_id, patch_object)
+    await put_object_json(sess, "study", accession_id[0], "SRP000539.json")
+    await put_object_xml(sess, "study", accession_id[0], "SRP000539_put.xml")
+
+
 async def test_crud_drafts_works(sess, schema, orginal_file, update_file, folder_id):
     """Test drafts REST api POST, PUT and DELETE reqs.
 
@@ -511,7 +559,6 @@ async def test_getting_all_objects_from_schema_works(sess, folder_id):
     :param sess: HTTP session in which request call is made
     :param folder_id: id of the folder used to group submission objects
     """
-
     # Add objects
     files = await asyncio.gather(*[post_object(sess, "study", "SRP000539.xml") for _ in range(13)])
 
@@ -528,7 +575,7 @@ async def test_getting_all_objects_from_schema_works(sess, folder_id):
         assert ans["page"]["page"] == 1
         assert ans["page"]["size"] == 10
         assert ans["page"]["totalPages"] == 2
-        assert ans["page"]["totalObjects"] == 13
+        assert ans["page"]["totalObjects"] == 14
         assert len(ans["objects"]) == 10
 
     # Test with custom pagination values
@@ -538,7 +585,7 @@ async def test_getting_all_objects_from_schema_works(sess, folder_id):
         assert ans["page"]["page"] == 2
         assert ans["page"]["size"] == 3
         assert ans["page"]["totalPages"] == 5
-        assert ans["page"]["totalObjects"] == 13
+        assert ans["page"]["totalObjects"] == 14
         assert len(ans["objects"]) == 3
 
     # Test with wrong pagination values
@@ -885,6 +932,14 @@ async def main():
 
         await asyncio.gather(*[test_crud_works(sess, schema, file, basic_folder_id) for schema, file in test_xml_files])
 
+        put_object_folder = {
+            "name": "test put object",
+            "description": "put object test folder",
+        }
+        put_object_folder = await post_folder(sess, put_object_folder)
+
+        await test_put_objects(sess, put_object_folder)
+
         # Test adding and getting draft objects
         LOG.debug("=== Testing basic CRUD drafts operations ===")
         draft_folder = {

tests/test_files/study/SRP000539_put.xml

@@ -0,0 +1,34 @@
+<STUDY_SET>
+    <STUDY center_name="GEO" alias="GSE10966">
+        <IDENTIFIERS>
+            <PRIMARY_ID>SRP000539</PRIMARY_ID>
+            <EXTERNAL_ID namespace="BioProject" label="primary">PRJNA108793</EXTERNAL_ID>
+            <EXTERNAL_ID namespace="GEO">GSE10966</EXTERNAL_ID>
+        </IDENTIFIERS>
+        <DESCRIPTOR>
+            <STUDY_TITLE>Highly integrated epigenome maps in Arabidopsis - whole genome shotgun bisulfite sequencing
+            </STUDY_TITLE>
+            <STUDY_TYPE existing_study_type="Other"/>
+            <STUDY_ABSTRACT>Part of a set of highly integrated epigenome maps for Arabidopsis thaliana. Keywords:
+                Illumina high-throughput bisulfite sequencing Overall design: Whole genome shotgun bisulfite sequencing
+                of wildtype Arabidopsis plants (Columbia-0), and met1, drm1 drm2 cmt3, and ros1 dml2 dml3 null mutants
+                using the Illumina Genetic Analyzer.
+            </STUDY_ABSTRACT>
+            <CENTER_PROJECT_NAME>GSE10966</CENTER_PROJECT_NAME>
+        </DESCRIPTOR>
+        <STUDY_LINKS>
+            <STUDY_LINK>
+                <XREF_LINK>
+                    <DB>pubmed</DB>
+                    <ID>18423832</ID>
+                </XREF_LINK>
+            </STUDY_LINK>
+        </STUDY_LINKS>
+        <STUDY_ATTRIBUTES>
+            <STUDY_ATTRIBUTE>
+                <TAG>parent_bioproject</TAG>
+                <VALUE>PRJNA107265</VALUE>
+            </STUDY_ATTRIBUTE>
+        </STUDY_ATTRIBUTES>
+    </STUDY>
+</STUDY_SET>

tests/test_server.py

@@ -43,7 +43,7 @@ async def test_init(self):
     async def test_api_routes_are_set(self):
         """Test correct amount of api (no frontend) routes is set."""
         server = await self.get_application()
-        self.assertIs(len(server.router.resources()), 18)
+        self.assertIs(len(server.router.resources()), 19)
 
     @unittest_run_loop
     async def test_frontend_routes_are_set(self):