From 6df4a65a9f50661c18b7a7694175752313675824 Mon Sep 17 00:00:00 2001 From: Evgenia Lyjina Date: Tue, 28 Dec 2021 10:13:53 +0000 Subject: [PATCH 1/2] Refactors handlers and their tests Separated most of handlers to own files inside handlers folder. Refactored tests for new set up. Fixed imports in server.py. --- metadata_backend/api/handlers.py | 1368 ----------------- metadata_backend/api/handlers/__init__.py | 1 + metadata_backend/api/handlers/api_handlers.py | 307 ++++ .../api/handlers/folder_handler.py | 322 ++++ .../api/handlers/object_handler.py | 256 +++ .../api/handlers/submission_handler.py | 166 ++ .../api/handlers/templates_handler.py | 163 ++ metadata_backend/api/handlers/user_handler.py | 221 +++ metadata_backend/server.py | 23 +- tests/test_handlers.py | 544 ++++--- 10 files changed, 1770 insertions(+), 1601 deletions(-) delete mode 100644 metadata_backend/api/handlers.py create mode 100644 metadata_backend/api/handlers/__init__.py create mode 100644 metadata_backend/api/handlers/api_handlers.py create mode 100644 metadata_backend/api/handlers/folder_handler.py create mode 100644 metadata_backend/api/handlers/object_handler.py create mode 100644 metadata_backend/api/handlers/submission_handler.py create mode 100644 metadata_backend/api/handlers/templates_handler.py create mode 100644 metadata_backend/api/handlers/user_handler.py diff --git a/metadata_backend/api/handlers.py b/metadata_backend/api/handlers.py deleted file mode 100644 index d43ed51b0..000000000 --- a/metadata_backend/api/handlers.py +++ /dev/null @@ -1,1368 +0,0 @@ -"""Handle HTTP methods for server.""" -import json -import mimetypes -import re -from collections import Counter -from datetime import date, datetime -from distutils.util import strtobool -from math import ceil -from pathlib import Path -from typing import Any, AsyncGenerator, Dict, List, Tuple, Union, cast - -import ujson -from aiohttp import BodyPartReader, web -from aiohttp.web import Request, Response -from motor.motor_asyncio import AsyncIOMotorClient -from multidict import CIMultiDict, MultiDict, MultiDictProxy -from xmlschema import XMLSchemaException - -from ..conf.conf import aai_config, publisher, schema_types -from ..helpers.doi import DOIHandler -from ..helpers.logger import LOG -from ..helpers.parser import XMLToJSONParser -from ..helpers.schema_loader import JSONSchemaLoader, SchemaNotFoundException, XMLSchemaLoader -from ..helpers.validator import JSONValidator, XMLValidator -from .middlewares import decrypt_cookie, get_session -from .operators import FolderOperator, Operator, UserOperator, XMLOperator - - -class RESTAPIHandler: - """Handler for REST API methods.""" - - def _check_schema_exists(self, schema_type: str) -> None: - """Check if schema type exists. - - :param schema_type: schema type. - :raises: HTTPNotFound if schema does not exist. - """ - if schema_type not in schema_types.keys(): - reason = f"Specified schema {schema_type} was not found." - LOG.error(reason) - raise web.HTTPNotFound(reason=reason) - - def _get_page_param(self, req: Request, name: str, default: int) -> int: - """Handle page parameter value extracting. - - :param req: GET Request - :param param_name: Name of the parameter - :param default: Default value in case parameter not specified in request - :returns: Page parameter value - """ - try: - param = int(req.query.get(name, default)) - except ValueError: - reason = f"{name} parameter must be a number, now it is {req.query.get(name)}" - LOG.error(reason) - raise web.HTTPBadRequest(reason=reason) - if param < 1: - reason = f"{name} parameter must be over 0" - LOG.error(reason) - raise web.HTTPBadRequest(reason=reason) - return param - - async def _handle_check_ownedby_user(self, req: Request, collection: str, accession_id: str) -> bool: - """Check if object belongs to user. - - For this we need to check the object is in exactly 1 folder and we need to check - that folder belongs to a user. If the folder is published that means it can be - browsed by other users as well. - - :param req: HTTP request - :param collection: collection or schema of document - :param doc_id: document accession id - :raises: HTTPUnauthorized if accession id does not belong to user - :returns: bool - """ - db_client = req.app["db_client"] - current_user = get_session(req)["user_info"] - user_op = UserOperator(db_client) - _check = False - - if collection != "folders": - - folder_op = FolderOperator(db_client) - check, folder_id, published = await folder_op.check_object_in_folder(collection, accession_id) - - if published: - _check = True - elif check: - # if the draft object is found in folder we just need to check if the folder belongs to user - _check = await user_op.check_user_has_doc("folders", current_user, folder_id) - elif collection.startswith("template"): - # if collection is template but not found in a folder - # we also check if object is in templates of the user - # they will be here if they will not be deleted after publish - _check = await user_op.check_user_has_doc(collection, current_user, accession_id) - else: - _check = False - else: - _check = await user_op.check_user_has_doc(collection, current_user, accession_id) - - if not _check: - reason = f"The ID: {accession_id} does not belong to current user." - LOG.error(reason) - raise web.HTTPUnauthorized(reason=reason) - - return _check - - async def _get_collection_objects( - self, folder_op: AsyncIOMotorClient, collection: str, seq: List - ) -> AsyncGenerator: - """Get objects ids based on folder and collection. - - Considering that many objects will be returned good to have a generator. - - :param req: HTTP request - :param collection: collection or schema of document - :param seq: list of folders - :returns: AsyncGenerator - """ - for el in seq: - result = await folder_op.get_collection_objects(el, collection) - - yield result - - async def _handle_user_objects_collection(self, req: Request, collection: str) -> List: - """Retrieve list of objects accession ids belonging to user in collection. - - :param req: HTTP request - :param collection: collection or schema of document - :returns: List - """ - db_client = req.app["db_client"] - current_user = get_session(req)["user_info"] - user_op = UserOperator(db_client) - folder_op = FolderOperator(db_client) - - user = await user_op.read_user(current_user) - res = self._get_collection_objects(folder_op, collection, user["folders"]) - - dt = [] - async for r in res: - dt.extend(r) - - return dt - - async def _filter_by_user(self, req: Request, collection: str, seq: List) -> AsyncGenerator: - """For a list of objects check if these are owned by a user. - - This can be called using a partial from functools. - - :param req: HTTP request - :param collection: collection or schema of document - :param seq: list of folders - :returns: AsyncGenerator - """ - for el in seq: - if await self._handle_check_ownedby_user(req, collection, el["accessionId"]): - yield el - - async def _get_data(self, req: Request) -> Dict: - """Get the data content from a request. - - :param req: POST/PUT/PATCH request - :raises: HTTPBadRequest if request does not have proper JSON data - :returns: JSON content of the request - """ - try: - content = await req.json() - return content - except json.decoder.JSONDecodeError as e: - reason = "JSON is not correctly formatted." f" See: {e}" - LOG.error(reason) - raise web.HTTPBadRequest(reason=reason) - - async def get_schema_types(self, req: Request) -> Response: - """Get all possible metadata schema types from database. - - Basically returns which objects user can submit and query for. - :param req: GET Request - :returns: JSON list of schema types - """ - types_json = ujson.dumps([x["description"] for x in schema_types.values()], escape_forward_slashes=False) - LOG.info(f"GET schema types. Retrieved {len(schema_types)} schemas.") - return web.Response(body=types_json, status=200, content_type="application/json") - - async def get_json_schema(self, req: Request) -> Response: - """Get all JSON Schema for a specific schema type. - - Basically returns which objects user can submit and query for. - :param req: GET Request - :raises: HTTPBadRequest if request does not find the schema - :returns: JSON list of schema types - """ - schema_type = req.match_info["schema"] - self._check_schema_exists(schema_type) - - try: - schema = JSONSchemaLoader().get_schema(schema_type) - LOG.info(f"{schema_type} schema loaded.") - return web.Response( - body=ujson.dumps(schema, escape_forward_slashes=False), status=200, content_type="application/json" - ) - - except SchemaNotFoundException as error: - reason = f"{error} ({schema_type})" - LOG.error(reason) - raise web.HTTPBadRequest(reason=reason) - - async def _header_links(self, url: str, page: int, size: int, total_objects: int) -> CIMultiDict[str]: - """Create link header for pagination. - - :param url: base url for request - :param page: current page - :param size: results per page - :param total_objects: total objects to compute the total pages - :returns: JSON with query results - """ - total_pages = ceil(total_objects / size) - prev_link = f'<{url}?page={page-1}&per_page={size}>; rel="prev", ' if page > 1 else "" - next_link = f'<{url}?page={page+1}&per_page={size}>; rel="next", ' if page < total_pages else "" - last_link = f'<{url}?page={total_pages}&per_page={size}>; rel="last"' if page < total_pages else "" - comma = ", " if page > 1 and page < total_pages else "" - first_link = f'<{url}?page=1&per_page={size}>; rel="first"{comma}' if page > 1 else "" - links = f"{prev_link}{next_link}{first_link}{last_link}" - link_headers = CIMultiDict(Link=f"{links}") - LOG.debug("Link headers created") - return link_headers - - -class ObjectAPIHandler(RESTAPIHandler): - """API Handler for Objects.""" - - async def _handle_query(self, req: Request) -> Response: - """Handle query results. - - :param req: GET request with query parameters - :returns: JSON with query results - """ - collection = req.match_info["schema"] - req_format = req.query.get("format", "json").lower() - if req_format == "xml": - reason = "xml-formatted query results are not supported" - raise web.HTTPBadRequest(reason=reason) - - page = self._get_page_param(req, "page", 1) - per_page = self._get_page_param(req, "per_page", 10) - db_client = req.app["db_client"] - - filter_list = await self._handle_user_objects_collection(req, collection) - data, page_num, page_size, total_objects = await Operator(db_client).query_metadata_database( - collection, req.query, page, per_page, filter_list - ) - - result = ujson.dumps( - { - "page": { - "page": page_num, - "size": page_size, - "totalPages": ceil(total_objects / per_page), - "totalObjects": total_objects, - }, - "objects": data, - }, - escape_forward_slashes=False, - ) - url = f"{req.scheme}://{req.host}{req.path}" - link_headers = await self._header_links(url, page_num, per_page, total_objects) - LOG.debug(f"Pagination header links: {link_headers}") - LOG.info(f"Querying for objects in {collection} resulted in {total_objects} objects ") - return web.Response( - body=result, - status=200, - headers=link_headers, - content_type="application/json", - ) - - async def get_object(self, req: Request) -> Response: - """Get one metadata object by its accession id. - - Returns original XML object from backup if format query parameter is - set, otherwise JSON. - - :param req: GET request - :returns: JSON or XML response containing metadata object - """ - accession_id = req.match_info["accessionId"] - schema_type = req.match_info["schema"] - self._check_schema_exists(schema_type) - collection = f"draft-{schema_type}" if req.path.startswith("/drafts") else schema_type - - req_format = req.query.get("format", "json").lower() - db_client = req.app["db_client"] - operator = XMLOperator(db_client) if req_format == "xml" else Operator(db_client) - type_collection = f"xml-{collection}" if req_format == "xml" else collection - - await operator.check_exists(collection, accession_id) - - await self._handle_check_ownedby_user(req, collection, accession_id) - - data, content_type = await operator.read_metadata_object(type_collection, accession_id) - - data = data if req_format == "xml" else ujson.dumps(data, escape_forward_slashes=False) - LOG.info(f"GET object with accesssion ID {accession_id} from schema {collection}.") - return web.Response(body=data, status=200, content_type=content_type) - - async def post_object(self, req: Request) -> Response: - """Save metadata object to database. - - For JSON request body we validate it is consistent with the - associated JSON schema. - - :param req: POST request - :returns: JSON response containing accessionId for submitted object - """ - schema_type = req.match_info["schema"] - self._check_schema_exists(schema_type) - collection = f"draft-{schema_type}" if req.path.startswith("/drafts") else schema_type - - db_client = req.app["db_client"] - content: Union[Dict, str] - operator: Union[Operator, XMLOperator] - if req.content_type == "multipart/form-data": - files = await _extract_xml_upload(req, extract_one=True) - content, _ = files[0] - operator = XMLOperator(db_client) - else: - content = await self._get_data(req) - if not req.path.startswith("/drafts"): - JSONValidator(content, schema_type).validate - operator = Operator(db_client) - - accession_id = await operator.create_metadata_object(collection, content) - - body = ujson.dumps({"accessionId": accession_id}, escape_forward_slashes=False) - url = f"{req.scheme}://{req.host}{req.path}" - location_headers = CIMultiDict(Location=f"{url}/{accession_id}") - LOG.info(f"POST object with accesssion ID {accession_id} in schema {collection} was successful.") - return web.Response( - body=body, - status=201, - headers=location_headers, - content_type="application/json", - ) - - async def query_objects(self, req: Request) -> Response: - """Query metadata objects from database. - - :param req: GET request with query parameters (can be empty). - :returns: Query results as JSON - """ - schema_type = req.match_info["schema"] - self._check_schema_exists(schema_type) - return await self._handle_query(req) - - async def delete_object(self, req: Request) -> Response: - """Delete metadata object from database. - - :param req: DELETE request - :raises: HTTPUnauthorized if folder published - :raises: HTTPUnprocessableEntity if object does not belong to current user - :returns: HTTPNoContent response - """ - schema_type = req.match_info["schema"] - self._check_schema_exists(schema_type) - collection = f"draft-{schema_type}" if req.path.startswith("/drafts") else schema_type - - accession_id = req.match_info["accessionId"] - db_client = req.app["db_client"] - - await Operator(db_client).check_exists(collection, accession_id) - - await self._handle_check_ownedby_user(req, collection, accession_id) - - folder_op = FolderOperator(db_client) - exists, folder_id, published = await folder_op.check_object_in_folder(collection, accession_id) - if exists: - if published: - reason = "published objects cannot be deleted." - LOG.error(reason) - raise web.HTTPUnauthorized(reason=reason) - await folder_op.remove_object(folder_id, collection, accession_id) - else: - reason = "This object does not seem to belong to any user." - LOG.error(reason) - raise web.HTTPUnprocessableEntity(reason=reason) - - accession_id = await Operator(db_client).delete_metadata_object(collection, accession_id) - - LOG.info(f"DELETE object with accession ID {accession_id} in schema {collection} was successful.") - return web.Response(status=204) - - async def put_object(self, req: Request) -> Response: - """Replace metadata object in database. - - For JSON request we don't allow replacing in the DB. - - :param req: PUT request - :raises: HTTPUnsupportedMediaType if JSON replace is attempted - :returns: JSON response containing accessionId for submitted object - """ - schema_type = req.match_info["schema"] - accession_id = req.match_info["accessionId"] - self._check_schema_exists(schema_type) - collection = f"draft-{schema_type}" if req.path.startswith("/drafts") else schema_type - - db_client = req.app["db_client"] - content: Union[Dict, str] - operator: Union[Operator, XMLOperator] - if req.content_type == "multipart/form-data": - files = await _extract_xml_upload(req, extract_one=True) - content, _ = files[0] - operator = XMLOperator(db_client) - else: - content = await self._get_data(req) - if not req.path.startswith("/drafts"): - reason = "Replacing objects only allowed for XML." - LOG.error(reason) - raise web.HTTPUnsupportedMediaType(reason=reason) - operator = Operator(db_client) - - await operator.check_exists(collection, accession_id) - - await self._handle_check_ownedby_user(req, collection, accession_id) - - accession_id = await operator.replace_metadata_object(collection, accession_id, content) - - body = ujson.dumps({"accessionId": accession_id}, escape_forward_slashes=False) - LOG.info(f"PUT object with accession ID {accession_id} in schema {collection} was successful.") - return web.Response(body=body, status=200, content_type="application/json") - - async def patch_object(self, req: Request) -> Response: - """Update metadata object in database. - - We do not support patch for XML. - - :param req: PATCH request - :raises: HTTPUnauthorized if object is in published folder - :returns: JSON response containing accessionId for submitted object - """ - schema_type = req.match_info["schema"] - accession_id = req.match_info["accessionId"] - self._check_schema_exists(schema_type) - collection = f"draft-{schema_type}" if req.path.startswith("/drafts") else schema_type - - db_client = req.app["db_client"] - operator: Union[Operator, XMLOperator] - if req.content_type == "multipart/form-data": - reason = "XML patching is not possible." - raise web.HTTPUnsupportedMediaType(reason=reason) - else: - content = await self._get_data(req) - operator = Operator(db_client) - - await operator.check_exists(collection, accession_id) - - await self._handle_check_ownedby_user(req, collection, accession_id) - - folder_op = FolderOperator(db_client) - exists, _, published = await folder_op.check_object_in_folder(collection, accession_id) - if exists: - if published: - reason = "Published objects cannot be updated." - LOG.error(reason) - raise web.HTTPUnauthorized(reason=reason) - - accession_id = await operator.update_metadata_object(collection, accession_id, content) - - body = ujson.dumps({"accessionId": accession_id}, escape_forward_slashes=False) - LOG.info(f"PATCH object with accession ID {accession_id} in schema {collection} was successful.") - return web.Response(body=body, status=200, content_type="application/json") - - -class TemplatesAPIHandler(RESTAPIHandler): - """API Handler for Templates.""" - - async def get_template(self, req: Request) -> Response: - """Get one metadata template by its accession id. - - Returns JSON. - - :param req: GET request - :returns: JSON response containing template - """ - accession_id = req.match_info["accessionId"] - schema_type = req.match_info["schema"] - self._check_schema_exists(schema_type) - collection = f"template-{schema_type}" - - db_client = req.app["db_client"] - operator = Operator(db_client) - - await operator.check_exists(collection, accession_id) - - await self._handle_check_ownedby_user(req, collection, accession_id) - - data, content_type = await operator.read_metadata_object(collection, accession_id) - - data = ujson.dumps(data, escape_forward_slashes=False) - LOG.info(f"GET template with accesssion ID {accession_id} from schema {collection}.") - return web.Response(body=data, status=200, content_type=content_type) - - async def post_template(self, req: Request) -> Response: - """Save metadata template to database. - - For JSON request body we validate it is consistent with the - associated JSON schema. - - :param req: POST request - :returns: JSON response containing accessionId for submitted template - """ - schema_type = req.match_info["schema"] - self._check_schema_exists(schema_type) - collection = f"template-{schema_type}" - - db_client = req.app["db_client"] - content = await self._get_data(req) - - user_op = UserOperator(db_client) - current_user = get_session(req)["user_info"] - - operator = Operator(db_client) - - if isinstance(content, list): - tmpl_list = [] - for num, tmpl in enumerate(content): - if "template" not in tmpl: - reason = f"template key is missing from request body for element: {num}." - LOG.error(reason) - raise web.HTTPBadRequest(reason=reason) - accession_id = await operator.create_metadata_object(collection, tmpl["template"]) - data = [{"accessionId": accession_id, "schema": collection}] - if "tags" in tmpl: - data[0]["tags"] = tmpl["tags"] - await user_op.assign_objects(current_user, "templates", data) - tmpl_list.append({"accessionId": accession_id}) - - body = ujson.dumps(tmpl_list, escape_forward_slashes=False) - else: - if "template" not in content: - reason = "template key is missing from request body." - LOG.error(reason) - raise web.HTTPBadRequest(reason=reason) - accession_id = await operator.create_metadata_object(collection, content["template"]) - data = [{"accessionId": accession_id, "schema": collection}] - if "tags" in content: - data[0]["tags"] = content["tags"] - await user_op.assign_objects(current_user, "templates", data) - - body = ujson.dumps({"accessionId": accession_id}, escape_forward_slashes=False) - - url = f"{req.scheme}://{req.host}{req.path}" - location_headers = CIMultiDict(Location=f"{url}/{accession_id}") - LOG.info(f"POST template with accesssion ID {accession_id} in schema {collection} was successful.") - return web.Response( - body=body, - status=201, - headers=location_headers, - content_type="application/json", - ) - - async def patch_template(self, req: Request) -> Response: - """Update metadata template in database. - - :param req: PATCH request - :raises: HTTPUnauthorized if template is in published folder - :returns: JSON response containing accessionId for submitted template - """ - schema_type = req.match_info["schema"] - accession_id = req.match_info["accessionId"] - self._check_schema_exists(schema_type) - collection = f"template-{schema_type}" - - db_client = req.app["db_client"] - operator: Union[Operator, XMLOperator] - - content = await self._get_data(req) - operator = Operator(db_client) - - await operator.check_exists(collection, accession_id) - - await self._handle_check_ownedby_user(req, collection, accession_id) - - accession_id = await operator.update_metadata_object(collection, accession_id, content) - - body = ujson.dumps({"accessionId": accession_id}, escape_forward_slashes=False) - LOG.info(f"PATCH template with accession ID {accession_id} in schema {collection} was successful.") - return web.Response(body=body, status=200, content_type="application/json") - - async def delete_template(self, req: Request) -> Response: - """Delete metadata template from database. - - :param req: DELETE request - :raises: HTTPUnauthorized if folder published - :raises: HTTPUnprocessableEntity if template does not belong to current user - :returns: HTTPNoContent response - """ - schema_type = req.match_info["schema"] - self._check_schema_exists(schema_type) - collection = f"template-{schema_type}" - - accession_id = req.match_info["accessionId"] - db_client = req.app["db_client"] - - await Operator(db_client).check_exists(collection, accession_id) - - await self._handle_check_ownedby_user(req, collection, accession_id) - - user_op = UserOperator(db_client) - current_user = get_session(req)["user_info"] - check_user = await user_op.check_user_has_doc(collection, current_user, accession_id) - if check_user: - await user_op.remove_objects(current_user, "templates", [accession_id]) - else: - reason = "This template does not seem to belong to any user." - LOG.error(reason) - raise web.HTTPUnprocessableEntity(reason=reason) - - accession_id = await Operator(db_client).delete_metadata_object(collection, accession_id) - - LOG.info(f"DELETE template with accession ID {accession_id} in schema {collection} was successful.") - return web.Response(status=204) - - -class FolderAPIHandler(RESTAPIHandler): - """API Handler for folders.""" - - def _check_patch_folder(self, patch_ops: Any) -> None: - """Check patch operations in request are valid. - - We check that ``metadataObjects`` and ``drafts`` have ``_required_values``. - For tags we check that the ``submissionType`` takes either ``XML`` or - ``Form`` as values. - :param patch_ops: JSON patch request - :raises: HTTPBadRequest if request does not fullfil one of requirements - :raises: HTTPUnauthorized if request tries to do anything else than add or replace - :returns: None - """ - _required_paths = ["/name", "/description"] - _required_values = ["schema", "accessionId"] - _arrays = ["/metadataObjects/-", "/drafts/-", "/doiInfo"] - _tags = re.compile("^/(metadataObjects|drafts)/[0-9]*/(tags)$") - - for op in patch_ops: - if _tags.match(op["path"]): - LOG.info(f"{op['op']} on tags in folder") - if "submissionType" in op["value"].keys() and op["value"]["submissionType"] not in ["XML", "Form"]: - reason = "submissionType is restricted to either 'XML' or 'Form' values." - LOG.error(reason) - raise web.HTTPBadRequest(reason=reason) - pass - else: - if all(i not in op["path"] for i in _required_paths + _arrays): - reason = f"Request contains '{op['path']}' key that cannot be updated to folders." - LOG.error(reason) - raise web.HTTPBadRequest(reason=reason) - if op["op"] in ["remove", "copy", "test", "move"]: - reason = f"{op['op']} on {op['path']} is not allowed." - LOG.error(reason) - raise web.HTTPUnauthorized(reason=reason) - if op["op"] == "replace" and op["path"] in _arrays: - reason = f"{op['op']} on {op['path']}; replacing all objects is not allowed." - LOG.error(reason) - raise web.HTTPUnauthorized(reason=reason) - if op["path"] in _arrays and op["path"] != "/doiInfo": - _ops = op["value"] if isinstance(op["value"], list) else [op["value"]] - for item in _ops: - if not all(key in item.keys() for key in _required_values): - reason = "accessionId and schema are required fields." - LOG.error(reason) - raise web.HTTPBadRequest(reason=reason) - if ( - "tags" in item - and "submissionType" in item["tags"] - and item["tags"]["submissionType"] not in ["XML", "Form"] - ): - reason = "submissionType is restricted to either 'XML' or 'Form' values." - LOG.error(reason) - raise web.HTTPBadRequest(reason=reason) - - async def get_folders(self, req: Request) -> Response: - """Get a set of folders owned by the user with pagination values. - - :param req: GET Request - :returns: JSON list of folders available for the user - """ - page = self._get_page_param(req, "page", 1) - per_page = self._get_page_param(req, "per_page", 5) - sort = {"date": True, "score": False} - db_client = req.app["db_client"] - - user_operator = UserOperator(db_client) - current_user = get_session(req)["user_info"] - user = await user_operator.read_user(current_user) - - folder_query = {"folderId": {"$in": user["folders"]}} - # Check if only published or draft folders are requestsed - if "published" in req.query: - pub_param = req.query.get("published", "").title() - if pub_param in ["True", "False"]: - folder_query["published"] = {"$eq": bool(strtobool(pub_param))} - else: - reason = "'published' parameter must be either 'true' or 'false'" - LOG.error(reason) - raise web.HTTPBadRequest(reason=reason) - - if "name" in req.query: - name_param = req.query.get("name", "") - if name_param: - folder_query = {"$text": {"$search": name_param}} - sort["score"] = True - sort["date"] = False - - format_incoming = "%Y-%m-%d" - format_query = "%Y-%m-%d %H:%M:%S" - if "date_created_start" in req.query and "date_created_end" in req.query: - date_param_start = req.query.get("date_created_start", "") - date_param_end = req.query.get("date_created_end", "") - - if datetime.strptime(date_param_start, format_incoming) and datetime.strptime( - date_param_end, format_incoming - ): - query_start = datetime.strptime(date_param_start + " 00:00:00", format_query).timestamp() - query_end = datetime.strptime(date_param_end + " 23:59:59", format_query).timestamp() - folder_query["dateCreated"] = {"$gte": query_start, "$lte": query_end} - else: - reason = f"'date_created_start' and 'date_created_end' parameters must be formated as {format_incoming}" - LOG.error(reason) - raise web.HTTPBadRequest(reason=reason) - - if "name" in req.query and "date_created_start" in req.query: - sort["score"] = True - sort["date"] = True - - folder_operator = FolderOperator(db_client) - folders, total_folders = await folder_operator.query_folders(folder_query, page, per_page, sort) - - result = ujson.dumps( - { - "page": { - "page": page, - "size": per_page, - "totalPages": ceil(total_folders / per_page), - "totalFolders": total_folders, - }, - "folders": folders, - }, - escape_forward_slashes=False, - ) - - url = f"{req.scheme}://{req.host}{req.path}" - link_headers = await self._header_links(url, page, per_page, total_folders) - LOG.debug(f"Pagination header links: {link_headers}") - LOG.info(f"Querying for user's folders resulted in {total_folders} folders") - return web.Response( - body=result, - status=200, - headers=link_headers, - content_type="application/json", - ) - - async def post_folder(self, req: Request) -> Response: - """Save object folder to database. - - Also assigns the folder to the current user. - - :param req: POST request - :returns: JSON response containing folder ID for submitted folder - """ - db_client = req.app["db_client"] - content = await self._get_data(req) - - JSONValidator(content, "folders").validate - - operator = FolderOperator(db_client) - folder = await operator.create_folder(content) - - user_op = UserOperator(db_client) - current_user = get_session(req)["user_info"] - await user_op.assign_objects(current_user, "folders", [folder]) - - body = ujson.dumps({"folderId": folder}, escape_forward_slashes=False) - - url = f"{req.scheme}://{req.host}{req.path}" - location_headers = CIMultiDict(Location=f"{url}/{folder}") - LOG.info(f"POST new folder with ID {folder} was successful.") - return web.Response(body=body, status=201, headers=location_headers, content_type="application/json") - - async def get_folder(self, req: Request) -> Response: - """Get one object folder by its folder id. - - :param req: GET request - :raises: HTTPNotFound if folder not owned by user - :returns: JSON response containing object folder - """ - folder_id = req.match_info["folderId"] - db_client = req.app["db_client"] - operator = FolderOperator(db_client) - - await operator.check_folder_exists(folder_id) - - await self._handle_check_ownedby_user(req, "folders", folder_id) - - folder = await operator.read_folder(folder_id) - - LOG.info(f"GET folder with ID {folder_id} was successful.") - return web.Response( - body=ujson.dumps(folder, escape_forward_slashes=False), status=200, content_type="application/json" - ) - - async def patch_folder(self, req: Request) -> Response: - """Update object folder with a specific folder id. - - :param req: PATCH request - :returns: JSON response containing folder ID for updated folder - """ - folder_id = req.match_info["folderId"] - db_client = req.app["db_client"] - - operator = FolderOperator(db_client) - - await operator.check_folder_exists(folder_id) - - # Check patch operations in request are valid - patch_ops = await self._get_data(req) - self._check_patch_folder(patch_ops) - - # Validate against folders schema if DOI is being added - for op in patch_ops: - if op["path"] == "/doiInfo": - curr_folder = await operator.read_folder(folder_id) - curr_folder["doiInfo"] = op["value"] - JSONValidator(curr_folder, "folders").validate - - await self._handle_check_ownedby_user(req, "folders", folder_id) - - upd_folder = await operator.update_folder(folder_id, patch_ops if isinstance(patch_ops, list) else [patch_ops]) - - body = ujson.dumps({"folderId": upd_folder}, escape_forward_slashes=False) - LOG.info(f"PATCH folder with ID {upd_folder} was successful.") - return web.Response(body=body, status=200, content_type="application/json") - - async def publish_folder(self, req: Request) -> Response: - """Update object folder specifically into published state. - - :param req: PATCH request - :returns: JSON response containing folder ID for updated folder - """ - folder_id = req.match_info["folderId"] - db_client = req.app["db_client"] - operator = FolderOperator(db_client) - - await operator.check_folder_exists(folder_id) - - await self._handle_check_ownedby_user(req, "folders", folder_id) - - folder = await operator.read_folder(folder_id) - - obj_ops = Operator(db_client) - - # Create draft DOI and delete draft objects from the folder - doi = DOIHandler() - doi_data = await doi.create_draft_doi() - identifier = {"identifierType": "DOI", "doi": doi_data["fullDOI"]} - - for obj in folder["drafts"]: - await obj_ops.delete_metadata_object(obj["schema"], obj["accessionId"]) - - # Patch the folder into a published state - patch = [ - {"op": "replace", "path": "/published", "value": True}, - {"op": "replace", "path": "/drafts", "value": []}, - {"op": "add", "path": "/datePublished", "value": int(datetime.now().timestamp())}, - {"op": "add", "path": "/extraInfo/identifier", "value": identifier}, - {"op": "add", "path": "/extraInfo/url", "value": doi_data["dataset"]}, - {"op": "add", "path": "/extraInfo/publisher", "value": publisher}, - { - "op": "add", - "path": "/extraInfo/types", - "value": { - "ris": "DATA", - "bibtex": "misc", - "citeproc": "dataset", - "schemaOrg": "Dataset", - "resourceTypeGeneral": "Dataset", - }, - }, - {"op": "add", "path": "/extraInfo/publicationYear", "value": date.today().year}, - ] - new_folder = await operator.update_folder(folder_id, patch) - - body = ujson.dumps({"folderId": new_folder}, escape_forward_slashes=False) - LOG.info(f"Patching folder with ID {new_folder} was successful.") - return web.Response(body=body, status=200, content_type="application/json") - - async def delete_folder(self, req: Request) -> Response: - """Delete object folder from database. - - :param req: DELETE request - :returns: HTTP No Content response - """ - folder_id = req.match_info["folderId"] - db_client = req.app["db_client"] - operator = FolderOperator(db_client) - - await operator.check_folder_exists(folder_id) - await operator.check_folder_published(folder_id) - - await self._handle_check_ownedby_user(req, "folders", folder_id) - - obj_ops = Operator(db_client) - - folder = await operator.read_folder(folder_id) - - for obj in folder["drafts"] + folder["metadataObjects"]: - await obj_ops.delete_metadata_object(obj["schema"], obj["accessionId"]) - - _folder_id = await operator.delete_folder(folder_id) - - user_op = UserOperator(db_client) - current_user = get_session(req)["user_info"] - await user_op.remove_objects(current_user, "folders", [folder_id]) - - LOG.info(f"DELETE folder with ID {_folder_id} was successful.") - return web.Response(status=204) - - -class UserAPIHandler(RESTAPIHandler): - """API Handler for users.""" - - def _check_patch_user(self, patch_ops: Any) -> None: - """Check patch operations in request are valid. - - We check that ``folders`` have string values (one or a list) - and ``drafts`` have ``_required_values``. - For tags we check that the ``submissionType`` takes either ``XML`` or - ``Form`` as values. - :param patch_ops: JSON patch request - :raises: HTTPBadRequest if request does not fullfil one of requirements - :raises: HTTPUnauthorized if request tries to do anything else than add or replace - :returns: None - """ - _arrays = ["/templates/-", "/folders/-"] - _required_values = ["schema", "accessionId"] - _tags = re.compile("^/(templates)/[0-9]*/(tags)$") - for op in patch_ops: - if _tags.match(op["path"]): - LOG.info(f"{op['op']} on tags in folder") - if "submissionType" in op["value"].keys() and op["value"]["submissionType"] not in ["XML", "Form"]: - reason = "submissionType is restricted to either 'XML' or 'Form' values." - LOG.error(reason) - raise web.HTTPBadRequest(reason=reason) - pass - else: - if all(i not in op["path"] for i in _arrays): - reason = f"Request contains '{op['path']}' key that cannot be updated to user object" - LOG.error(reason) - raise web.HTTPBadRequest(reason=reason) - if op["op"] in ["remove", "copy", "test", "move", "replace"]: - reason = f"{op['op']} on {op['path']} is not allowed." - LOG.error(reason) - raise web.HTTPUnauthorized(reason=reason) - if op["path"] == "/folders/-": - if not (isinstance(op["value"], str) or isinstance(op["value"], list)): - reason = "We only accept string folder IDs." - LOG.error(reason) - raise web.HTTPBadRequest(reason=reason) - if op["path"] == "/templates/-": - _ops = op["value"] if isinstance(op["value"], list) else [op["value"]] - for item in _ops: - if not all(key in item.keys() for key in _required_values): - reason = "accessionId and schema are required fields." - LOG.error(reason) - raise web.HTTPBadRequest(reason=reason) - if ( - "tags" in item - and "submissionType" in item["tags"] - and item["tags"]["submissionType"] not in ["XML", "Form"] - ): - reason = "submissionType is restricted to either 'XML' or 'Form' values." - LOG.error(reason) - raise web.HTTPBadRequest(reason=reason) - - async def get_user(self, req: Request) -> Response: - """Get one user by its user ID. - - :param req: GET request - :raises: HTTPUnauthorized if not current user - :returns: JSON response containing user object or list of user templates or user folders by id - """ - user_id = req.match_info["userId"] - if user_id != "current": - LOG.info(f"User ID {user_id} was requested") - raise web.HTTPUnauthorized(reason="Only current user retrieval is allowed") - - current_user = get_session(req)["user_info"] - - item_type = req.query.get("items", "").lower() - if item_type: - # Return only list of templates or list of folder IDs owned by the user - result, link_headers = await self._get_user_items(req, current_user, item_type) - return web.Response( - body=ujson.dumps(result, escape_forward_slashes=False), - status=200, - headers=link_headers, - content_type="application/json", - ) - else: - # Return whole user object if templates or folders are not specified in query - db_client = req.app["db_client"] - operator = UserOperator(db_client) - user = await operator.read_user(current_user) - LOG.info(f"GET user with ID {user_id} was successful.") - return web.Response( - body=ujson.dumps(user, escape_forward_slashes=False), status=200, content_type="application/json" - ) - - async def patch_user(self, req: Request) -> Response: - """Update user object with a specific user ID. - - :param req: PATCH request - :raises: HTTPUnauthorized if not current user - :returns: JSON response containing user ID for updated user object - """ - user_id = req.match_info["userId"] - if user_id != "current": - LOG.info(f"User ID {user_id} patch was requested") - raise web.HTTPUnauthorized(reason="Only current user operations are allowed") - db_client = req.app["db_client"] - - patch_ops = await self._get_data(req) - self._check_patch_user(patch_ops) - - operator = UserOperator(db_client) - - current_user = get_session(req)["user_info"] - user = await operator.update_user(current_user, patch_ops if isinstance(patch_ops, list) else [patch_ops]) - - body = ujson.dumps({"userId": user}) - LOG.info(f"PATCH user with ID {user} was successful.") - return web.Response(body=body, status=200, content_type="application/json") - - async def delete_user(self, req: Request) -> Response: - """Delete user from database. - - :param req: DELETE request - :raises: HTTPUnauthorized if not current user - :returns: HTTPNoContent response - """ - user_id = req.match_info["userId"] - if user_id != "current": - LOG.info(f"User ID {user_id} delete was requested") - raise web.HTTPUnauthorized(reason="Only current user deletion is allowed") - db_client = req.app["db_client"] - operator = UserOperator(db_client) - fold_ops = FolderOperator(db_client) - obj_ops = Operator(db_client) - - current_user = get_session(req)["user_info"] - user = await operator.read_user(current_user) - - for folder_id in user["folders"]: - _folder = await fold_ops.read_folder(folder_id) - if "published" in _folder and not _folder["published"]: - for obj in _folder["drafts"] + _folder["metadataObjects"]: - await obj_ops.delete_metadata_object(obj["schema"], obj["accessionId"]) - await fold_ops.delete_folder(folder_id) - - for tmpl in user["templates"]: - await obj_ops.delete_metadata_object(tmpl["schema"], tmpl["accessionId"]) - - await operator.delete_user(current_user) - LOG.info(f"DELETE user with ID {current_user} was successful.") - - cookie = decrypt_cookie(req) - - try: - req.app["Session"].pop(cookie["id"]) - req.app["Cookies"].remove(cookie["id"]) - except KeyError: - pass - - response = web.HTTPSeeOther(f"{aai_config['redirect']}/") - response.headers["Location"] = ( - "/" if aai_config["redirect"] == aai_config["domain"] else f"{aai_config['redirect']}/" - ) - LOG.debug("Logged out user ") - raise response - - async def _get_user_items(self, req: Request, user: Dict, item_type: str) -> Tuple[Dict, CIMultiDict[str]]: - """Get draft templates owned by the user with pagination values. - - :param req: GET request - :param user: User object - :param item_type: Name of the items ("templates" or "folders") - :raises: HTTPUnauthorized if not current user - :returns: Paginated list of user draft templates and link header - """ - # Check item_type parameter is not faulty - if item_type not in ["templates", "folders"]: - reason = f"{item_type} is a faulty item parameter. Should be either folders or templates" - LOG.error(reason) - raise web.HTTPBadRequest(reason=reason) - - page = self._get_page_param(req, "page", 1) - per_page = self._get_page_param(req, "per_page", 5) - - db_client = req.app["db_client"] - operator = UserOperator(db_client) - user_id = req.match_info["userId"] - - query = {"userId": user} - - items, total_items = await operator.filter_user(query, item_type, page, per_page) - LOG.info(f"GET user with ID {user_id} was successful.") - - result = { - "page": { - "page": page, - "size": per_page, - "totalPages": ceil(total_items / per_page), - "total" + item_type.title(): total_items, - }, - item_type: items, - } - - url = f"{req.scheme}://{req.host}{req.path}" - link_headers = await self._header_links(url, page, per_page, total_items) - LOG.debug(f"Pagination header links: {link_headers}") - LOG.info(f"Querying for user's {item_type} resulted in {total_items} {item_type}") - return result, link_headers - - -class SubmissionAPIHandler: - """Handler for non-rest API methods.""" - - async def submit(self, req: Request) -> Response: - """Handle submission.xml containing submissions to server. - - First submission info is parsed and then for every action in submission - (add/modify/validate) corresponding operation is performed. - Finally submission info itself is added. - - :param req: Multipart POST request with submission.xml and files - :raises: HTTPBadRequest if request is missing some parameters or cannot be processed - :returns: XML-based receipt from submission - """ - files = await _extract_xml_upload(req) - schema_types = Counter(file[1] for file in files) - if "submission" not in schema_types: - reason = "There must be a submission.xml file in submission." - LOG.error(reason) - raise web.HTTPBadRequest(reason=reason) - if schema_types["submission"] > 1: - reason = "You should submit only one submission.xml file." - LOG.error(reason) - raise web.HTTPBadRequest(reason=reason) - submission_xml = files[0][0] - submission_json = XMLToJSONParser().parse("submission", submission_xml) - - # Check what actions should be performed, collect them to dictionary - actions: Dict[str, List] = {} - for action_set in submission_json["actions"]["action"]: - for action, attr in action_set.items(): - if not attr: - reason = f"""You also need to provide necessary - information for submission action. - Now {action} was provided without any - extra information.""" - LOG.error(reason) - raise web.HTTPBadRequest(reason=reason) - LOG.debug(f"submission has action {action}") - if attr["schema"] in actions: - set = [] - set.append(actions[attr["schema"]]) - set.append(action) - actions[attr["schema"]] = set - else: - actions[attr["schema"]] = action - - # Go through parsed files and do the actual action - results: List[Dict] = [] - db_client = req.app["db_client"] - for file in files: - content_xml = file[0] - schema_type = file[1] - if schema_type == "submission": - LOG.debug("file has schema of submission type, continuing ...") - continue # No need to use submission xml - action = actions[schema_type] - if isinstance(action, List): - for item in action: - result = await self._execute_action(schema_type, content_xml, db_client, item) - results.append(result) - else: - result = await self._execute_action(schema_type, content_xml, db_client, action) - results.append(result) - - body = ujson.dumps(results, escape_forward_slashes=False) - LOG.info(f"Processed a submission of {len(results)} actions.") - return web.Response(body=body, status=200, content_type="application/json") - - async def validate(self, req: Request) -> Response: - """Handle validating an XML file sent to endpoint. - - :param req: Multipart POST request with submission.xml and files - :returns: JSON response indicating if validation was successful or not - """ - files = await _extract_xml_upload(req, extract_one=True) - xml_content, schema_type = files[0] - validator = await self._perform_validation(schema_type, xml_content) - return web.Response(body=validator.resp_body, content_type="application/json") - - async def _perform_validation(self, schema_type: str, xml_content: str) -> XMLValidator: - """Validate an xml. - - :param schema_type: Schema type of the object to validate. - :param xml_content: Metadata object - :raises: HTTPBadRequest if schema load fails - :returns: JSON response indicating if validation was successful or not - """ - try: - schema = XMLSchemaLoader().get_schema(schema_type) - LOG.info(f"{schema_type} schema loaded.") - return XMLValidator(schema, xml_content) - - except (SchemaNotFoundException, XMLSchemaException) as error: - reason = f"{error} ({schema_type})" - LOG.error(reason) - raise web.HTTPBadRequest(reason=reason) - - async def _execute_action(self, schema: str, content: str, db_client: AsyncIOMotorClient, action: str) -> Dict: - """Complete the command in the action set of the submission file. - - Only "add/modify/validate" actions are supported. - - :param schema: Schema type of the object in question - :param content: Metadata object referred to in submission - :param db_client: Database client for database operations - :param action: Type of action to be done - :raises: HTTPBadRequest if an incorrect or non-supported action is called - :returns: Dict containing specific action that was completed - """ - if action == "add": - result = { - "accessionId": await XMLOperator(db_client).create_metadata_object(schema, content), - "schema": schema, - } - LOG.debug(f"added some content in {schema} ...") - return result - - elif action == "modify": - data_as_json = XMLToJSONParser().parse(schema, content) - if "accessionId" in data_as_json: - accession_id = data_as_json["accessionId"] - else: - alias = data_as_json["alias"] - query = MultiDictProxy(MultiDict([("alias", alias)])) - data, _, _, _ = await Operator(db_client).query_metadata_database(schema, query, 1, 1, []) - if len(data) > 1: - reason = "Alias in provided XML file corresponds with more than one existing metadata object." - LOG.error(reason) - raise web.HTTPBadRequest(reason=reason) - accession_id = data[0]["accessionId"] - data_as_json.pop("accessionId", None) - result = { - "accessionId": await Operator(db_client).update_metadata_object(schema, accession_id, data_as_json), - "schema": schema, - } - LOG.debug(f"modified some content in {schema} ...") - return result - - elif action == "validate": - validator = await self._perform_validation(schema, content) - return ujson.loads(validator.resp_body) - - else: - reason = f"Action {action} in XML is not supported." - LOG.error(reason) - raise web.HTTPBadRequest(reason=reason) - - -class StaticHandler: - """Handler for static routes, mostly frontend and 404.""" - - def __init__(self, frontend_static_files: Path) -> None: - """Initialize path to frontend static files folder.""" - self.path = frontend_static_files - - async def frontend(self, req: Request) -> Response: - """Serve requests related to frontend SPA. - - :param req: GET request - :returns: Response containing frontpage static file - """ - serve_path = self.path.joinpath("./" + req.path) - - if not serve_path.exists() or not serve_path.is_file(): - LOG.debug(f"{serve_path} was not found or is not a file - serving index.html") - serve_path = self.path.joinpath("./index.html") - - LOG.debug(f"Serve Frontend SPA {req.path} by {serve_path}.") - - mime_type = mimetypes.guess_type(serve_path.as_posix()) - - return Response(body=serve_path.read_bytes(), content_type=(mime_type[0] or "text/html")) - - def setup_static(self) -> Path: - """Set path for static js files and correct return mimetypes. - - :returns: Path to static js files folder - """ - mimetypes.init() - mimetypes.types_map[".js"] = "application/javascript" - mimetypes.types_map[".js.map"] = "application/json" - mimetypes.types_map[".svg"] = "image/svg+xml" - mimetypes.types_map[".css"] = "text/css" - mimetypes.types_map[".css.map"] = "application/json" - LOG.debug("static paths for SPA set.") - return self.path / "static" - - -# Private functions shared between handlers -async def _extract_xml_upload(req: Request, extract_one: bool = False) -> List[Tuple[str, str]]: - """Extract submitted xml-file(s) from multi-part request. - - Files are sorted to spesific order by their schema priorities (e.g. - submission should be processed before study). - - :param req: POST request containing "multipart/form-data" upload - :raises: HTTPBadRequest if request is not valid for multipart or multiple files sent. HTTPNotFound if - schema was not found. - :returns: content and schema type for each uploaded file, sorted by schema - type. - """ - files: List[Tuple[str, str]] = [] - try: - reader = await req.multipart() - except AssertionError: - reason = "Request does not have valid multipart/form content" - LOG.error(reason) - raise web.HTTPBadRequest(reason=reason) - while True: - part = await reader.next() - # Following is probably error in aiohttp type hints, fixing so - # mypy doesn't complain about it. No runtime consequences. - part = cast(BodyPartReader, part) - if not part: - break - if extract_one and files: - reason = "Only one file can be sent to this endpoint at a time." - LOG.error(reason) - raise web.HTTPBadRequest(reason=reason) - if part.name: - schema_type = part.name.lower() - if schema_type not in schema_types: - reason = f"Specified schema {schema_type} was not found." - LOG.error(reason) - raise web.HTTPNotFound(reason=reason) - data = [] - while True: - chunk = await part.read_chunk() - if not chunk: - break - data.append(chunk) - xml_content = "".join(x.decode("UTF-8") for x in data) - files.append((xml_content, schema_type)) - LOG.debug(f"processed file in {schema_type}") - return sorted(files, key=lambda x: schema_types[x[1]]["priority"]) diff --git a/metadata_backend/api/handlers/__init__.py b/metadata_backend/api/handlers/__init__.py new file mode 100644 index 000000000..f3b5ffee8 --- /dev/null +++ b/metadata_backend/api/handlers/__init__.py @@ -0,0 +1 @@ +"""API handlers.""" diff --git a/metadata_backend/api/handlers/api_handlers.py b/metadata_backend/api/handlers/api_handlers.py new file mode 100644 index 000000000..e50092292 --- /dev/null +++ b/metadata_backend/api/handlers/api_handlers.py @@ -0,0 +1,307 @@ +"""Handle HTTP methods for server.""" +import json +import mimetypes +from math import ceil +from pathlib import Path +from typing import AsyncGenerator, Dict, List, Tuple, cast + +import ujson +from aiohttp import BodyPartReader, web +from aiohttp.web import Request, Response +from motor.motor_asyncio import AsyncIOMotorClient +from multidict import CIMultiDict + +from ...conf.conf import schema_types +from ...helpers.logger import LOG +from ...helpers.schema_loader import JSONSchemaLoader, SchemaNotFoundException +from ..middlewares import get_session +from ..operators import FolderOperator, UserOperator + + +class RESTAPIHandler: + """Handler for REST API methods.""" + + def _check_schema_exists(self, schema_type: str) -> None: + """Check if schema type exists. + + :param schema_type: schema type. + :raises: HTTPNotFound if schema does not exist. + """ + if schema_type not in schema_types.keys(): + reason = f"Specified schema {schema_type} was not found." + LOG.error(reason) + raise web.HTTPNotFound(reason=reason) + + def _get_page_param(self, req: Request, name: str, default: int) -> int: + """Handle page parameter value extracting. + + :param req: GET Request + :param param_name: Name of the parameter + :param default: Default value in case parameter not specified in request + :returns: Page parameter value + """ + try: + param = int(req.query.get(name, default)) + except ValueError: + reason = f"{name} parameter must be a number, now it is {req.query.get(name)}" + LOG.error(reason) + raise web.HTTPBadRequest(reason=reason) + if param < 1: + reason = f"{name} parameter must be over 0" + LOG.error(reason) + raise web.HTTPBadRequest(reason=reason) + return param + + async def _handle_check_ownedby_user(self, req: Request, collection: str, accession_id: str) -> bool: + """Check if object belongs to user. + + For this we need to check the object is in exactly 1 folder and we need to check + that folder belongs to a user. If the folder is published that means it can be + browsed by other users as well. + + :param req: HTTP request + :param collection: collection or schema of document + :param doc_id: document accession id + :raises: HTTPUnauthorized if accession id does not belong to user + :returns: bool + """ + db_client = req.app["db_client"] + current_user = get_session(req)["user_info"] + user_op = UserOperator(db_client) + _check = False + + if collection != "folders": + + folder_op = FolderOperator(db_client) + check, folder_id, published = await folder_op.check_object_in_folder(collection, accession_id) + if published: + _check = True + elif check: + # if the draft object is found in folder we just need to check if the folder belongs to user + _check = await user_op.check_user_has_doc("folders", current_user, folder_id) + elif collection.startswith("template"): + # if collection is template but not found in a folder + # we also check if object is in templates of the user + # they will be here if they will not be deleted after publish + _check = await user_op.check_user_has_doc(collection, current_user, accession_id) + else: + _check = False + else: + _check = await user_op.check_user_has_doc(collection, current_user, accession_id) + + if not _check: + reason = f"The ID: {accession_id} does not belong to current user." + LOG.error(reason) + raise web.HTTPUnauthorized(reason=reason) + + return _check + + async def _get_collection_objects( + self, folder_op: AsyncIOMotorClient, collection: str, seq: List + ) -> AsyncGenerator: + """Get objects ids based on folder and collection. + + Considering that many objects will be returned good to have a generator. + + :param req: HTTP request + :param collection: collection or schema of document + :param seq: list of folders + :returns: AsyncGenerator + """ + for el in seq: + result = await folder_op.get_collection_objects(el, collection) + + yield result + + async def _handle_user_objects_collection(self, req: Request, collection: str) -> List: + """Retrieve list of objects accession ids belonging to user in collection. + + :param req: HTTP request + :param collection: collection or schema of document + :returns: List + """ + db_client = req.app["db_client"] + current_user = get_session(req)["user_info"] + user_op = UserOperator(db_client) + folder_op = FolderOperator(db_client) + + user = await user_op.read_user(current_user) + res = self._get_collection_objects(folder_op, collection, user["folders"]) + + dt = [] + async for r in res: + dt.extend(r) + + return dt + + async def _filter_by_user(self, req: Request, collection: str, seq: List) -> AsyncGenerator: + """For a list of objects check if these are owned by a user. + + This can be called using a partial from functools. + + :param req: HTTP request + :param collection: collection or schema of document + :param seq: list of folders + :returns: AsyncGenerator + """ + for el in seq: + if await self._handle_check_ownedby_user(req, collection, el["accessionId"]): + yield el + + async def _get_data(self, req: Request) -> Dict: + """Get the data content from a request. + + :param req: POST/PUT/PATCH request + :raises: HTTPBadRequest if request does not have proper JSON data + :returns: JSON content of the request + """ + try: + content = await req.json() + return content + except json.decoder.JSONDecodeError as e: + reason = "JSON is not correctly formatted." f" See: {e}" + LOG.error(reason) + raise web.HTTPBadRequest(reason=reason) + + async def get_schema_types(self, req: Request) -> Response: + """Get all possible metadata schema types from database. + + Basically returns which objects user can submit and query for. + :param req: GET Request + :returns: JSON list of schema types + """ + types_json = ujson.dumps([x["description"] for x in schema_types.values()], escape_forward_slashes=False) + LOG.info(f"GET schema types. Retrieved {len(schema_types)} schemas.") + return web.Response(body=types_json, status=200, content_type="application/json") + + async def get_json_schema(self, req: Request) -> Response: + """Get all JSON Schema for a specific schema type. + + Basically returns which objects user can submit and query for. + :param req: GET Request + :raises: HTTPBadRequest if request does not find the schema + :returns: JSON list of schema types + """ + schema_type = req.match_info["schema"] + self._check_schema_exists(schema_type) + + try: + schema = JSONSchemaLoader().get_schema(schema_type) + LOG.info(f"{schema_type} schema loaded.") + return web.Response( + body=ujson.dumps(schema, escape_forward_slashes=False), status=200, content_type="application/json" + ) + + except SchemaNotFoundException as error: + reason = f"{error} ({schema_type})" + LOG.error(reason) + raise web.HTTPBadRequest(reason=reason) + + async def _header_links(self, url: str, page: int, size: int, total_objects: int) -> CIMultiDict[str]: + """Create link header for pagination. + + :param url: base url for request + :param page: current page + :param size: results per page + :param total_objects: total objects to compute the total pages + :returns: JSON with query results + """ + total_pages = ceil(total_objects / size) + prev_link = f'<{url}?page={page-1}&per_page={size}>; rel="prev", ' if page > 1 else "" + next_link = f'<{url}?page={page+1}&per_page={size}>; rel="next", ' if page < total_pages else "" + last_link = f'<{url}?page={total_pages}&per_page={size}>; rel="last"' if page < total_pages else "" + comma = ", " if page > 1 and page < total_pages else "" + first_link = f'<{url}?page=1&per_page={size}>; rel="first"{comma}' if page > 1 else "" + links = f"{prev_link}{next_link}{first_link}{last_link}" + link_headers = CIMultiDict(Link=f"{links}") + LOG.debug("Link headers created") + return link_headers + + +class StaticHandler: + """Handler for static routes, mostly frontend and 404.""" + + def __init__(self, frontend_static_files: Path) -> None: + """Initialize path to frontend static files folder.""" + self.path = frontend_static_files + + async def frontend(self, req: Request) -> Response: + """Serve requests related to frontend SPA. + + :param req: GET request + :returns: Response containing frontpage static file + """ + serve_path = self.path.joinpath("./" + req.path) + + if not serve_path.exists() or not serve_path.is_file(): + LOG.debug(f"{serve_path} was not found or is not a file - serving index.html") + serve_path = self.path.joinpath("./index.html") + + LOG.debug(f"Serve Frontend SPA {req.path} by {serve_path}.") + + mime_type = mimetypes.guess_type(serve_path.as_posix()) + + return Response(body=serve_path.read_bytes(), content_type=(mime_type[0] or "text/html")) + + def setup_static(self) -> Path: + """Set path for static js files and correct return mimetypes. + + :returns: Path to static js files folder + """ + mimetypes.init() + mimetypes.types_map[".js"] = "application/javascript" + mimetypes.types_map[".js.map"] = "application/json" + mimetypes.types_map[".svg"] = "image/svg+xml" + mimetypes.types_map[".css"] = "text/css" + mimetypes.types_map[".css.map"] = "application/json" + LOG.debug("static paths for SPA set.") + return self.path / "static" + + +# Private functions shared between handlers +async def _extract_xml_upload(req: Request, extract_one: bool = False) -> List[Tuple[str, str]]: + """Extract submitted xml-file(s) from multi-part request. + + Files are sorted to spesific order by their schema priorities (e.g. + submission should be processed before study). + + :param req: POST request containing "multipart/form-data" upload + :raises: HTTPBadRequest if request is not valid for multipart or multiple files sent. HTTPNotFound if + schema was not found. + :returns: content and schema type for each uploaded file, sorted by schema + type. + """ + files: List[Tuple[str, str]] = [] + try: + reader = await req.multipart() + except AssertionError: + reason = "Request does not have valid multipart/form content" + LOG.error(reason) + raise web.HTTPBadRequest(reason=reason) + while True: + part = await reader.next() + # Following is probably error in aiohttp type hints, fixing so + # mypy doesn't complain about it. No runtime consequences. + part = cast(BodyPartReader, part) + if not part: + break + if extract_one and files: + reason = "Only one file can be sent to this endpoint at a time." + LOG.error(reason) + raise web.HTTPBadRequest(reason=reason) + if part.name: + schema_type = part.name.lower() + if schema_type not in schema_types: + reason = f"Specified schema {schema_type} was not found." + LOG.error(reason) + raise web.HTTPNotFound(reason=reason) + data = [] + while True: + chunk = await part.read_chunk() + if not chunk: + break + data.append(chunk) + xml_content = "".join(x.decode("UTF-8") for x in data) + files.append((xml_content, schema_type)) + LOG.debug(f"processed file in {schema_type}") + return sorted(files, key=lambda x: schema_types[x[1]]["priority"]) diff --git a/metadata_backend/api/handlers/folder_handler.py b/metadata_backend/api/handlers/folder_handler.py new file mode 100644 index 000000000..799029109 --- /dev/null +++ b/metadata_backend/api/handlers/folder_handler.py @@ -0,0 +1,322 @@ +"""Handle HTTP methods for server.""" +import re +from datetime import date, datetime +from distutils.util import strtobool +from math import ceil +from typing import Any + +import ujson +from aiohttp import web +from aiohttp.web import Request, Response +from multidict import CIMultiDict + +from ...conf.conf import publisher +from ...helpers.doi import DOIHandler +from ...helpers.logger import LOG +from ...helpers.validator import JSONValidator +from .api_handlers import RESTAPIHandler +from ..middlewares import get_session +from ..operators import FolderOperator, Operator, UserOperator + + +class FolderAPIHandler(RESTAPIHandler): + """API Handler for folders.""" + + def _check_patch_folder(self, patch_ops: Any) -> None: + """Check patch operations in request are valid. + + We check that ``metadataObjects`` and ``drafts`` have ``_required_values``. + For tags we check that the ``submissionType`` takes either ``XML`` or + ``Form`` as values. + :param patch_ops: JSON patch request + :raises: HTTPBadRequest if request does not fullfil one of requirements + :raises: HTTPUnauthorized if request tries to do anything else than add or replace + :returns: None + """ + _required_paths = ["/name", "/description"] + _required_values = ["schema", "accessionId"] + _arrays = ["/metadataObjects/-", "/drafts/-", "/doiInfo"] + _tags = re.compile("^/(metadataObjects|drafts)/[0-9]*/(tags)$") + + for op in patch_ops: + if _tags.match(op["path"]): + LOG.info(f"{op['op']} on tags in folder") + if "submissionType" in op["value"].keys() and op["value"]["submissionType"] not in ["XML", "Form"]: + reason = "submissionType is restricted to either 'XML' or 'Form' values." + LOG.error(reason) + raise web.HTTPBadRequest(reason=reason) + pass + else: + if all(i not in op["path"] for i in _required_paths + _arrays): + reason = f"Request contains '{op['path']}' key that cannot be updated to folders." + LOG.error(reason) + raise web.HTTPBadRequest(reason=reason) + if op["op"] in ["remove", "copy", "test", "move"]: + reason = f"{op['op']} on {op['path']} is not allowed." + LOG.error(reason) + raise web.HTTPUnauthorized(reason=reason) + if op["op"] == "replace" and op["path"] in _arrays: + reason = f"{op['op']} on {op['path']}; replacing all objects is not allowed." + LOG.error(reason) + raise web.HTTPUnauthorized(reason=reason) + if op["path"] in _arrays and op["path"] != "/doiInfo": + _ops = op["value"] if isinstance(op["value"], list) else [op["value"]] + for item in _ops: + if not all(key in item.keys() for key in _required_values): + reason = "accessionId and schema are required fields." + LOG.error(reason) + raise web.HTTPBadRequest(reason=reason) + if ( + "tags" in item + and "submissionType" in item["tags"] + and item["tags"]["submissionType"] not in ["XML", "Form"] + ): + reason = "submissionType is restricted to either 'XML' or 'Form' values." + LOG.error(reason) + raise web.HTTPBadRequest(reason=reason) + + async def get_folders(self, req: Request) -> Response: + """Get a set of folders owned by the user with pagination values. + + :param req: GET Request + :returns: JSON list of folders available for the user + """ + page = self._get_page_param(req, "page", 1) + per_page = self._get_page_param(req, "per_page", 5) + sort = {"date": True, "score": False} + db_client = req.app["db_client"] + + user_operator = UserOperator(db_client) + current_user = get_session(req)["user_info"] + user = await user_operator.read_user(current_user) + + folder_query = {"folderId": {"$in": user["folders"]}} + # Check if only published or draft folders are requestsed + if "published" in req.query: + pub_param = req.query.get("published", "").title() + if pub_param in ["True", "False"]: + folder_query["published"] = {"$eq": bool(strtobool(pub_param))} + else: + reason = "'published' parameter must be either 'true' or 'false'" + LOG.error(reason) + raise web.HTTPBadRequest(reason=reason) + + if "name" in req.query: + name_param = req.query.get("name", "") + if name_param: + folder_query = {"$text": {"$search": name_param}} + sort["score"] = True + sort["date"] = False + + format_incoming = "%Y-%m-%d" + format_query = "%Y-%m-%d %H:%M:%S" + if "date_created_start" in req.query and "date_created_end" in req.query: + date_param_start = req.query.get("date_created_start", "") + date_param_end = req.query.get("date_created_end", "") + + if datetime.strptime(date_param_start, format_incoming) and datetime.strptime( + date_param_end, format_incoming + ): + query_start = datetime.strptime(date_param_start + " 00:00:00", format_query).timestamp() + query_end = datetime.strptime(date_param_end + " 23:59:59", format_query).timestamp() + folder_query["dateCreated"] = {"$gte": query_start, "$lte": query_end} + else: + reason = f"'date_created_start' and 'date_created_end' parameters must be formated as {format_incoming}" + LOG.error(reason) + raise web.HTTPBadRequest(reason=reason) + + if "name" in req.query and "date_created_start" in req.query: + sort["score"] = True + sort["date"] = True + + folder_operator = FolderOperator(db_client) + folders, total_folders = await folder_operator.query_folders(folder_query, page, per_page, sort) + + result = ujson.dumps( + { + "page": { + "page": page, + "size": per_page, + "totalPages": ceil(total_folders / per_page), + "totalFolders": total_folders, + }, + "folders": folders, + }, + escape_forward_slashes=False, + ) + + url = f"{req.scheme}://{req.host}{req.path}" + link_headers = await self._header_links(url, page, per_page, total_folders) + LOG.debug(f"Pagination header links: {link_headers}") + LOG.info(f"Querying for user's folders resulted in {total_folders} folders") + return web.Response( + body=result, + status=200, + headers=link_headers, + content_type="application/json", + ) + + async def post_folder(self, req: Request) -> Response: + """Save object folder to database. + + Also assigns the folder to the current user. + + :param req: POST request + :returns: JSON response containing folder ID for submitted folder + """ + db_client = req.app["db_client"] + content = await self._get_data(req) + + JSONValidator(content, "folders").validate + + operator = FolderOperator(db_client) + folder = await operator.create_folder(content) + + user_op = UserOperator(db_client) + current_user = get_session(req)["user_info"] + await user_op.assign_objects(current_user, "folders", [folder]) + + body = ujson.dumps({"folderId": folder}, escape_forward_slashes=False) + + url = f"{req.scheme}://{req.host}{req.path}" + location_headers = CIMultiDict(Location=f"{url}/{folder}") + LOG.info(f"POST new folder with ID {folder} was successful.") + return web.Response(body=body, status=201, headers=location_headers, content_type="application/json") + + async def get_folder(self, req: Request) -> Response: + """Get one object folder by its folder id. + + :param req: GET request + :raises: HTTPNotFound if folder not owned by user + :returns: JSON response containing object folder + """ + folder_id = req.match_info["folderId"] + db_client = req.app["db_client"] + operator = FolderOperator(db_client) + + await operator.check_folder_exists(folder_id) + + await self._handle_check_ownedby_user(req, "folders", folder_id) + + folder = await operator.read_folder(folder_id) + + LOG.info(f"GET folder with ID {folder_id} was successful.") + return web.Response( + body=ujson.dumps(folder, escape_forward_slashes=False), status=200, content_type="application/json" + ) + + async def patch_folder(self, req: Request) -> Response: + """Update object folder with a specific folder id. + + :param req: PATCH request + :returns: JSON response containing folder ID for updated folder + """ + folder_id = req.match_info["folderId"] + db_client = req.app["db_client"] + + operator = FolderOperator(db_client) + + await operator.check_folder_exists(folder_id) + + # Check patch operations in request are valid + patch_ops = await self._get_data(req) + self._check_patch_folder(patch_ops) + + # Validate against folders schema if DOI is being added + for op in patch_ops: + if op["path"] == "/doiInfo": + curr_folder = await operator.read_folder(folder_id) + curr_folder["doiInfo"] = op["value"] + JSONValidator(curr_folder, "folders").validate + + await self._handle_check_ownedby_user(req, "folders", folder_id) + + upd_folder = await operator.update_folder(folder_id, patch_ops if isinstance(patch_ops, list) else [patch_ops]) + + body = ujson.dumps({"folderId": upd_folder}, escape_forward_slashes=False) + LOG.info(f"PATCH folder with ID {upd_folder} was successful.") + return web.Response(body=body, status=200, content_type="application/json") + + async def publish_folder(self, req: Request) -> Response: + """Update object folder specifically into published state. + + :param req: PATCH request + :returns: JSON response containing folder ID for updated folder + """ + folder_id = req.match_info["folderId"] + db_client = req.app["db_client"] + operator = FolderOperator(db_client) + + await operator.check_folder_exists(folder_id) + + await self._handle_check_ownedby_user(req, "folders", folder_id) + + folder = await operator.read_folder(folder_id) + + obj_ops = Operator(db_client) + + # Create draft DOI and delete draft objects from the folder + doi = DOIHandler() + doi_data = await doi.create_draft_doi() + identifier = {"identifierType": "DOI", "doi": doi_data["fullDOI"]} + + for obj in folder["drafts"]: + await obj_ops.delete_metadata_object(obj["schema"], obj["accessionId"]) + + # Patch the folder into a published state + patch = [ + {"op": "replace", "path": "/published", "value": True}, + {"op": "replace", "path": "/drafts", "value": []}, + {"op": "add", "path": "/datePublished", "value": int(datetime.now().timestamp())}, + {"op": "add", "path": "/extraInfo/identifier", "value": identifier}, + {"op": "add", "path": "/extraInfo/url", "value": doi_data["dataset"]}, + {"op": "add", "path": "/extraInfo/publisher", "value": publisher}, + { + "op": "add", + "path": "/extraInfo/types", + "value": { + "ris": "DATA", + "bibtex": "misc", + "citeproc": "dataset", + "schemaOrg": "Dataset", + "resourceTypeGeneral": "Dataset", + }, + }, + {"op": "add", "path": "/extraInfo/publicationYear", "value": date.today().year}, + ] + new_folder = await operator.update_folder(folder_id, patch) + + body = ujson.dumps({"folderId": new_folder}, escape_forward_slashes=False) + LOG.info(f"Patching folder with ID {new_folder} was successful.") + return web.Response(body=body, status=200, content_type="application/json") + + async def delete_folder(self, req: Request) -> Response: + """Delete object folder from database. + + :param req: DELETE request + :returns: HTTP No Content response + """ + folder_id = req.match_info["folderId"] + db_client = req.app["db_client"] + operator = FolderOperator(db_client) + + await operator.check_folder_exists(folder_id) + await operator.check_folder_published(folder_id) + + await self._handle_check_ownedby_user(req, "folders", folder_id) + + obj_ops = Operator(db_client) + + folder = await operator.read_folder(folder_id) + + for obj in folder["drafts"] + folder["metadataObjects"]: + await obj_ops.delete_metadata_object(obj["schema"], obj["accessionId"]) + + _folder_id = await operator.delete_folder(folder_id) + + user_op = UserOperator(db_client) + current_user = get_session(req)["user_info"] + await user_op.remove_objects(current_user, "folders", [folder_id]) + + LOG.info(f"DELETE folder with ID {_folder_id} was successful.") + return web.Response(status=204) diff --git a/metadata_backend/api/handlers/object_handler.py b/metadata_backend/api/handlers/object_handler.py new file mode 100644 index 000000000..27b627e29 --- /dev/null +++ b/metadata_backend/api/handlers/object_handler.py @@ -0,0 +1,256 @@ +"""Handle HTTP methods for server.""" +from math import ceil +from typing import Dict, Union + +import ujson +from aiohttp import web +from aiohttp.web import Request, Response +from multidict import CIMultiDict + +from ...helpers.logger import LOG +from ...helpers.validator import JSONValidator +from .api_handlers import RESTAPIHandler, _extract_xml_upload +from ..operators import FolderOperator, Operator, XMLOperator + + +class ObjectAPIHandler(RESTAPIHandler): + """API Handler for Objects.""" + + async def _handle_query(self, req: Request) -> Response: + """Handle query results. + + :param req: GET request with query parameters + :returns: JSON with query results + """ + collection = req.match_info["schema"] + req_format = req.query.get("format", "json").lower() + if req_format == "xml": + reason = "xml-formatted query results are not supported" + raise web.HTTPBadRequest(reason=reason) + + page = self._get_page_param(req, "page", 1) + per_page = self._get_page_param(req, "per_page", 10) + db_client = req.app["db_client"] + + filter_list = await self._handle_user_objects_collection(req, collection) + data, page_num, page_size, total_objects = await Operator(db_client).query_metadata_database( + collection, req.query, page, per_page, filter_list + ) + + result = ujson.dumps( + { + "page": { + "page": page_num, + "size": page_size, + "totalPages": ceil(total_objects / per_page), + "totalObjects": total_objects, + }, + "objects": data, + }, + escape_forward_slashes=False, + ) + url = f"{req.scheme}://{req.host}{req.path}" + link_headers = await self._header_links(url, page_num, per_page, total_objects) + LOG.debug(f"Pagination header links: {link_headers}") + LOG.info(f"Querying for objects in {collection} resulted in {total_objects} objects ") + return web.Response( + body=result, + status=200, + headers=link_headers, + content_type="application/json", + ) + + async def get_object(self, req: Request) -> Response: + """Get one metadata object by its accession id. + + Returns original XML object from backup if format query parameter is + set, otherwise JSON. + + :param req: GET request + :returns: JSON or XML response containing metadata object + """ + accession_id = req.match_info["accessionId"] + schema_type = req.match_info["schema"] + self._check_schema_exists(schema_type) + collection = f"draft-{schema_type}" if req.path.startswith("/drafts") else schema_type + + req_format = req.query.get("format", "json").lower() + db_client = req.app["db_client"] + operator = XMLOperator(db_client) if req_format == "xml" else Operator(db_client) + type_collection = f"xml-{collection}" if req_format == "xml" else collection + + await operator.check_exists(collection, accession_id) + + await self._handle_check_ownedby_user(req, collection, accession_id) + + data, content_type = await operator.read_metadata_object(type_collection, accession_id) + + data = data if req_format == "xml" else ujson.dumps(data, escape_forward_slashes=False) + LOG.info(f"GET object with accesssion ID {accession_id} from schema {collection}.") + return web.Response(body=data, status=200, content_type=content_type) + + async def post_object(self, req: Request) -> Response: + """Save metadata object to database. + + For JSON request body we validate it is consistent with the + associated JSON schema. + + :param req: POST request + :returns: JSON response containing accessionId for submitted object + """ + schema_type = req.match_info["schema"] + self._check_schema_exists(schema_type) + collection = f"draft-{schema_type}" if req.path.startswith("/drafts") else schema_type + + db_client = req.app["db_client"] + content: Union[Dict, str] + operator: Union[Operator, XMLOperator] + if req.content_type == "multipart/form-data": + files = await _extract_xml_upload(req, extract_one=True) + content, _ = files[0] + operator = XMLOperator(db_client) + else: + content = await self._get_data(req) + if not req.path.startswith("/drafts"): + JSONValidator(content, schema_type).validate + operator = Operator(db_client) + + accession_id = await operator.create_metadata_object(collection, content) + + body = ujson.dumps({"accessionId": accession_id}, escape_forward_slashes=False) + url = f"{req.scheme}://{req.host}{req.path}" + location_headers = CIMultiDict(Location=f"{url}/{accession_id}") + LOG.info(f"POST object with accesssion ID {accession_id} in schema {collection} was successful.") + return web.Response( + body=body, + status=201, + headers=location_headers, + content_type="application/json", + ) + + async def query_objects(self, req: Request) -> Response: + """Query metadata objects from database. + + :param req: GET request with query parameters (can be empty). + :returns: Query results as JSON + """ + schema_type = req.match_info["schema"] + self._check_schema_exists(schema_type) + return await self._handle_query(req) + + async def delete_object(self, req: Request) -> Response: + """Delete metadata object from database. + + :param req: DELETE request + :raises: HTTPUnauthorized if folder published + :raises: HTTPUnprocessableEntity if object does not belong to current user + :returns: HTTPNoContent response + """ + schema_type = req.match_info["schema"] + self._check_schema_exists(schema_type) + collection = f"draft-{schema_type}" if req.path.startswith("/drafts") else schema_type + + accession_id = req.match_info["accessionId"] + db_client = req.app["db_client"] + + await Operator(db_client).check_exists(collection, accession_id) + + await self._handle_check_ownedby_user(req, collection, accession_id) + + folder_op = FolderOperator(db_client) + exists, folder_id, published = await folder_op.check_object_in_folder(collection, accession_id) + if exists: + if published: + reason = "published objects cannot be deleted." + LOG.error(reason) + raise web.HTTPUnauthorized(reason=reason) + await folder_op.remove_object(folder_id, collection, accession_id) + else: + reason = "This object does not seem to belong to any user." + LOG.error(reason) + raise web.HTTPUnprocessableEntity(reason=reason) + + accession_id = await Operator(db_client).delete_metadata_object(collection, accession_id) + + LOG.info(f"DELETE object with accession ID {accession_id} in schema {collection} was successful.") + return web.Response(status=204) + + async def put_object(self, req: Request) -> Response: + """Replace metadata object in database. + + For JSON request we don't allow replacing in the DB. + + :param req: PUT request + :raises: HTTPUnsupportedMediaType if JSON replace is attempted + :returns: JSON response containing accessionId for submitted object + """ + schema_type = req.match_info["schema"] + accession_id = req.match_info["accessionId"] + self._check_schema_exists(schema_type) + collection = f"draft-{schema_type}" if req.path.startswith("/drafts") else schema_type + + db_client = req.app["db_client"] + content: Union[Dict, str] + operator: Union[Operator, XMLOperator] + if req.content_type == "multipart/form-data": + files = await _extract_xml_upload(req, extract_one=True) + content, _ = files[0] + operator = XMLOperator(db_client) + else: + content = await self._get_data(req) + if not req.path.startswith("/drafts"): + reason = "Replacing objects only allowed for XML." + LOG.error(reason) + raise web.HTTPUnsupportedMediaType(reason=reason) + operator = Operator(db_client) + + await operator.check_exists(collection, accession_id) + + await self._handle_check_ownedby_user(req, collection, accession_id) + + accession_id = await operator.replace_metadata_object(collection, accession_id, content) + + body = ujson.dumps({"accessionId": accession_id}, escape_forward_slashes=False) + LOG.info(f"PUT object with accession ID {accession_id} in schema {collection} was successful.") + return web.Response(body=body, status=200, content_type="application/json") + + async def patch_object(self, req: Request) -> Response: + """Update metadata object in database. + + We do not support patch for XML. + + :param req: PATCH request + :raises: HTTPUnauthorized if object is in published folder + :returns: JSON response containing accessionId for submitted object + """ + schema_type = req.match_info["schema"] + accession_id = req.match_info["accessionId"] + self._check_schema_exists(schema_type) + collection = f"draft-{schema_type}" if req.path.startswith("/drafts") else schema_type + + db_client = req.app["db_client"] + operator: Union[Operator, XMLOperator] + if req.content_type == "multipart/form-data": + reason = "XML patching is not possible." + raise web.HTTPUnsupportedMediaType(reason=reason) + else: + content = await self._get_data(req) + operator = Operator(db_client) + + await operator.check_exists(collection, accession_id) + + await self._handle_check_ownedby_user(req, collection, accession_id) + + folder_op = FolderOperator(db_client) + exists, _, published = await folder_op.check_object_in_folder(collection, accession_id) + if exists: + if published: + reason = "Published objects cannot be updated." + LOG.error(reason) + raise web.HTTPUnauthorized(reason=reason) + + accession_id = await operator.update_metadata_object(collection, accession_id, content) + + body = ujson.dumps({"accessionId": accession_id}, escape_forward_slashes=False) + LOG.info(f"PATCH object with accession ID {accession_id} in schema {collection} was successful.") + return web.Response(body=body, status=200, content_type="application/json") diff --git a/metadata_backend/api/handlers/submission_handler.py b/metadata_backend/api/handlers/submission_handler.py new file mode 100644 index 000000000..d3492e4f2 --- /dev/null +++ b/metadata_backend/api/handlers/submission_handler.py @@ -0,0 +1,166 @@ +"""Handle HTTP methods for server.""" +from collections import Counter +from typing import Dict, List + +import ujson +from aiohttp import web +from aiohttp.web import Request, Response +from motor.motor_asyncio import AsyncIOMotorClient +from multidict import MultiDict, MultiDictProxy +from xmlschema import XMLSchemaException + +from ...helpers.logger import LOG +from ...helpers.parser import XMLToJSONParser +from ...helpers.schema_loader import SchemaNotFoundException, XMLSchemaLoader +from ...helpers.validator import XMLValidator +from .api_handlers import _extract_xml_upload +from ..operators import Operator, XMLOperator + + +class SubmissionAPIHandler: + """Handler for non-rest API methods.""" + + async def submit(self, req: Request) -> Response: + """Handle submission.xml containing submissions to server. + + First submission info is parsed and then for every action in submission + (add/modify/validate) corresponding operation is performed. + Finally submission info itself is added. + + :param req: Multipart POST request with submission.xml and files + :raises: HTTPBadRequest if request is missing some parameters or cannot be processed + :returns: XML-based receipt from submission + """ + files = await _extract_xml_upload(req) + schema_types = Counter(file[1] for file in files) + if "submission" not in schema_types: + reason = "There must be a submission.xml file in submission." + LOG.error(reason) + raise web.HTTPBadRequest(reason=reason) + if schema_types["submission"] > 1: + reason = "You should submit only one submission.xml file." + LOG.error(reason) + raise web.HTTPBadRequest(reason=reason) + submission_xml = files[0][0] + submission_json = XMLToJSONParser().parse("submission", submission_xml) + + # Check what actions should be performed, collect them to dictionary + actions: Dict[str, List] = {} + for action_set in submission_json["actions"]["action"]: + for action, attr in action_set.items(): + if not attr: + reason = f"""You also need to provide necessary + information for submission action. + Now {action} was provided without any + extra information.""" + LOG.error(reason) + raise web.HTTPBadRequest(reason=reason) + LOG.debug(f"submission has action {action}") + if attr["schema"] in actions: + set = [] + set.append(actions[attr["schema"]]) + set.append(action) + actions[attr["schema"]] = set + else: + actions[attr["schema"]] = action + + # Go through parsed files and do the actual action + results: List[Dict] = [] + db_client = req.app["db_client"] + for file in files: + content_xml = file[0] + schema_type = file[1] + if schema_type == "submission": + LOG.debug("file has schema of submission type, continuing ...") + continue # No need to use submission xml + action = actions[schema_type] + if isinstance(action, List): + for item in action: + result = await self._execute_action(schema_type, content_xml, db_client, item) + results.append(result) + else: + result = await self._execute_action(schema_type, content_xml, db_client, action) + results.append(result) + + body = ujson.dumps(results, escape_forward_slashes=False) + LOG.info(f"Processed a submission of {len(results)} actions.") + return web.Response(body=body, status=200, content_type="application/json") + + async def validate(self, req: Request) -> Response: + """Handle validating an XML file sent to endpoint. + + :param req: Multipart POST request with submission.xml and files + :returns: JSON response indicating if validation was successful or not + """ + files = await _extract_xml_upload(req, extract_one=True) + xml_content, schema_type = files[0] + validator = await self._perform_validation(schema_type, xml_content) + return web.Response(body=validator.resp_body, content_type="application/json") + + async def _perform_validation(self, schema_type: str, xml_content: str) -> XMLValidator: + """Validate an xml. + + :param schema_type: Schema type of the object to validate. + :param xml_content: Metadata object + :raises: HTTPBadRequest if schema load fails + :returns: JSON response indicating if validation was successful or not + """ + try: + schema = XMLSchemaLoader().get_schema(schema_type) + LOG.info(f"{schema_type} schema loaded.") + return XMLValidator(schema, xml_content) + + except (SchemaNotFoundException, XMLSchemaException) as error: + reason = f"{error} ({schema_type})" + LOG.error(reason) + raise web.HTTPBadRequest(reason=reason) + + async def _execute_action(self, schema: str, content: str, db_client: AsyncIOMotorClient, action: str) -> Dict: + """Complete the command in the action set of the submission file. + + Only "add/modify/validate" actions are supported. + + :param schema: Schema type of the object in question + :param content: Metadata object referred to in submission + :param db_client: Database client for database operations + :param action: Type of action to be done + :raises: HTTPBadRequest if an incorrect or non-supported action is called + :returns: Dict containing specific action that was completed + """ + if action == "add": + result = { + "accessionId": await XMLOperator(db_client).create_metadata_object(schema, content), + "schema": schema, + } + LOG.debug(f"added some content in {schema} ...") + return result + + elif action == "modify": + data_as_json = XMLToJSONParser().parse(schema, content) + if "accessionId" in data_as_json: + accession_id = data_as_json["accessionId"] + else: + alias = data_as_json["alias"] + query = MultiDictProxy(MultiDict([("alias", alias)])) + data, _, _, _ = await Operator(db_client).query_metadata_database(schema, query, 1, 1, []) + if len(data) > 1: + reason = "Alias in provided XML file corresponds with more than one existing metadata object." + LOG.error(reason) + raise web.HTTPBadRequest(reason=reason) + accession_id = data[0]["accessionId"] + data_as_json.pop("accessionId", None) + result = { + "accessionId": await Operator(db_client).update_metadata_object(schema, accession_id, data_as_json), + "schema": schema, + } + LOG.debug(f"modified some content in {schema} ...") + return result + + elif action == "validate": + validator = await self._perform_validation(schema, content) + return ujson.loads(validator.resp_body) + + else: + reason = f"Action {action} in XML is not supported." + LOG.error(reason) + raise web.HTTPBadRequest(reason=reason) diff --git a/metadata_backend/api/handlers/templates_handler.py b/metadata_backend/api/handlers/templates_handler.py new file mode 100644 index 000000000..f45413142 --- /dev/null +++ b/metadata_backend/api/handlers/templates_handler.py @@ -0,0 +1,163 @@ +"""Handle HTTP methods for server.""" +from typing import Union + +import ujson +from aiohttp import web +from aiohttp.web import Request, Response +from multidict import CIMultiDict + +from ...helpers.logger import LOG +from ..middlewares import get_session +from ..operators import Operator, UserOperator, XMLOperator +from .api_handlers import RESTAPIHandler + + +class TemplatesAPIHandler(RESTAPIHandler): + """API Handler for Templates.""" + + async def get_template(self, req: Request) -> Response: + """Get one metadata template by its accession id. + + Returns JSON. + + :param req: GET request + :returns: JSON response containing template + """ + accession_id = req.match_info["accessionId"] + schema_type = req.match_info["schema"] + self._check_schema_exists(schema_type) + collection = f"template-{schema_type}" + + db_client = req.app["db_client"] + operator = Operator(db_client) + + await operator.check_exists(collection, accession_id) + + await self._handle_check_ownedby_user(req, collection, accession_id) + + data, content_type = await operator.read_metadata_object(collection, accession_id) + + data = ujson.dumps(data, escape_forward_slashes=False) + LOG.info(f"GET template with accesssion ID {accession_id} from schema {collection}.") + return web.Response(body=data, status=200, content_type=content_type) + + async def post_template(self, req: Request) -> Response: + """Save metadata template to database. + + For JSON request body we validate it is consistent with the + associated JSON schema. + + :param req: POST request + :returns: JSON response containing accessionId for submitted template + """ + schema_type = req.match_info["schema"] + self._check_schema_exists(schema_type) + collection = f"template-{schema_type}" + + db_client = req.app["db_client"] + content = await self._get_data(req) + + user_op = UserOperator(db_client) + current_user = get_session(req)["user_info"] + + operator = Operator(db_client) + + if isinstance(content, list): + tmpl_list = [] + for num, tmpl in enumerate(content): + if "template" not in tmpl: + reason = f"template key is missing from request body for element: {num}." + LOG.error(reason) + raise web.HTTPBadRequest(reason=reason) + accession_id = await operator.create_metadata_object(collection, tmpl["template"]) + data = [{"accessionId": accession_id, "schema": collection}] + if "tags" in tmpl: + data[0]["tags"] = tmpl["tags"] + await user_op.assign_objects(current_user, "templates", data) + tmpl_list.append({"accessionId": accession_id}) + + body = ujson.dumps(tmpl_list, escape_forward_slashes=False) + else: + if "template" not in content: + reason = "template key is missing from request body." + LOG.error(reason) + raise web.HTTPBadRequest(reason=reason) + accession_id = await operator.create_metadata_object(collection, content["template"]) + data = [{"accessionId": accession_id, "schema": collection}] + if "tags" in content: + data[0]["tags"] = content["tags"] + await user_op.assign_objects(current_user, "templates", data) + + body = ujson.dumps({"accessionId": accession_id}, escape_forward_slashes=False) + + url = f"{req.scheme}://{req.host}{req.path}" + location_headers = CIMultiDict(Location=f"{url}/{accession_id}") + LOG.info(f"POST template with accesssion ID {accession_id} in schema {collection} was successful.") + return web.Response( + body=body, + status=201, + headers=location_headers, + content_type="application/json", + ) + + async def patch_template(self, req: Request) -> Response: + """Update metadata template in database. + + :param req: PATCH request + :raises: HTTPUnauthorized if template is in published folder + :returns: JSON response containing accessionId for submitted template + """ + schema_type = req.match_info["schema"] + accession_id = req.match_info["accessionId"] + self._check_schema_exists(schema_type) + collection = f"template-{schema_type}" + + db_client = req.app["db_client"] + operator: Union[Operator, XMLOperator] + + content = await self._get_data(req) + operator = Operator(db_client) + + await operator.check_exists(collection, accession_id) + + await self._handle_check_ownedby_user(req, collection, accession_id) + + accession_id = await operator.update_metadata_object(collection, accession_id, content) + + body = ujson.dumps({"accessionId": accession_id}, escape_forward_slashes=False) + LOG.info(f"PATCH template with accession ID {accession_id} in schema {collection} was successful.") + return web.Response(body=body, status=200, content_type="application/json") + + async def delete_template(self, req: Request) -> Response: + """Delete metadata template from database. + + :param req: DELETE request + :raises: HTTPUnauthorized if folder published + :raises: HTTPUnprocessableEntity if template does not belong to current user + :returns: HTTPNoContent response + """ + schema_type = req.match_info["schema"] + self._check_schema_exists(schema_type) + collection = f"template-{schema_type}" + + accession_id = req.match_info["accessionId"] + db_client = req.app["db_client"] + + await Operator(db_client).check_exists(collection, accession_id) + + await self._handle_check_ownedby_user(req, collection, accession_id) + + user_op = UserOperator(db_client) + current_user = get_session(req)["user_info"] + check_user = await user_op.check_user_has_doc(collection, current_user, accession_id) + if check_user: + await user_op.remove_objects(current_user, "templates", [accession_id]) + else: + reason = "This template does not seem to belong to any user." + LOG.error(reason) + raise web.HTTPUnprocessableEntity(reason=reason) + + accession_id = await Operator(db_client).delete_metadata_object(collection, accession_id) + + LOG.info(f"DELETE template with accession ID {accession_id} in schema {collection} was successful.") + return web.Response(status=204) diff --git a/metadata_backend/api/handlers/user_handler.py b/metadata_backend/api/handlers/user_handler.py new file mode 100644 index 000000000..cd3a2bd9b --- /dev/null +++ b/metadata_backend/api/handlers/user_handler.py @@ -0,0 +1,221 @@ +"""Handle HTTP methods for server.""" +import re +from math import ceil +from typing import Any, Dict, Tuple + +import ujson +from aiohttp import web +from aiohttp.web import Request, Response +from multidict import CIMultiDict + +from ...conf.conf import aai_config +from ...helpers.logger import LOG +from .api_handlers import RESTAPIHandler +from ..middlewares import decrypt_cookie, get_session +from ..operators import FolderOperator, Operator, UserOperator + + +class UserAPIHandler(RESTAPIHandler): + """API Handler for users.""" + + def _check_patch_user(self, patch_ops: Any) -> None: + """Check patch operations in request are valid. + + We check that ``folders`` have string values (one or a list) + and ``drafts`` have ``_required_values``. + For tags we check that the ``submissionType`` takes either ``XML`` or + ``Form`` as values. + :param patch_ops: JSON patch request + :raises: HTTPBadRequest if request does not fullfil one of requirements + :raises: HTTPUnauthorized if request tries to do anything else than add or replace + :returns: None + """ + _arrays = ["/templates/-", "/folders/-"] + _required_values = ["schema", "accessionId"] + _tags = re.compile("^/(templates)/[0-9]*/(tags)$") + for op in patch_ops: + if _tags.match(op["path"]): + LOG.info(f"{op['op']} on tags in folder") + if "submissionType" in op["value"].keys() and op["value"]["submissionType"] not in ["XML", "Form"]: + reason = "submissionType is restricted to either 'XML' or 'Form' values." + LOG.error(reason) + raise web.HTTPBadRequest(reason=reason) + pass + else: + if all(i not in op["path"] for i in _arrays): + reason = f"Request contains '{op['path']}' key that cannot be updated to user object" + LOG.error(reason) + raise web.HTTPBadRequest(reason=reason) + if op["op"] in ["remove", "copy", "test", "move", "replace"]: + reason = f"{op['op']} on {op['path']} is not allowed." + LOG.error(reason) + raise web.HTTPUnauthorized(reason=reason) + if op["path"] == "/folders/-": + if not (isinstance(op["value"], str) or isinstance(op["value"], list)): + reason = "We only accept string folder IDs." + LOG.error(reason) + raise web.HTTPBadRequest(reason=reason) + if op["path"] == "/templates/-": + _ops = op["value"] if isinstance(op["value"], list) else [op["value"]] + for item in _ops: + if not all(key in item.keys() for key in _required_values): + reason = "accessionId and schema are required fields." + LOG.error(reason) + raise web.HTTPBadRequest(reason=reason) + if ( + "tags" in item + and "submissionType" in item["tags"] + and item["tags"]["submissionType"] not in ["XML", "Form"] + ): + reason = "submissionType is restricted to either 'XML' or 'Form' values." + LOG.error(reason) + raise web.HTTPBadRequest(reason=reason) + + async def get_user(self, req: Request) -> Response: + """Get one user by its user ID. + + :param req: GET request + :raises: HTTPUnauthorized if not current user + :returns: JSON response containing user object or list of user templates or user folders by id + """ + user_id = req.match_info["userId"] + if user_id != "current": + LOG.info(f"User ID {user_id} was requested") + raise web.HTTPUnauthorized(reason="Only current user retrieval is allowed") + + current_user = get_session(req)["user_info"] + + item_type = req.query.get("items", "").lower() + if item_type: + # Return only list of templates or list of folder IDs owned by the user + result, link_headers = await self._get_user_items(req, current_user, item_type) + return web.Response( + body=ujson.dumps(result, escape_forward_slashes=False), + status=200, + headers=link_headers, + content_type="application/json", + ) + else: + # Return whole user object if templates or folders are not specified in query + db_client = req.app["db_client"] + operator = UserOperator(db_client) + user = await operator.read_user(current_user) + LOG.info(f"GET user with ID {user_id} was successful.") + return web.Response( + body=ujson.dumps(user, escape_forward_slashes=False), status=200, content_type="application/json" + ) + + async def patch_user(self, req: Request) -> Response: + """Update user object with a specific user ID. + + :param req: PATCH request + :raises: HTTPUnauthorized if not current user + :returns: JSON response containing user ID for updated user object + """ + user_id = req.match_info["userId"] + if user_id != "current": + LOG.info(f"User ID {user_id} patch was requested") + raise web.HTTPUnauthorized(reason="Only current user operations are allowed") + db_client = req.app["db_client"] + + patch_ops = await self._get_data(req) + self._check_patch_user(patch_ops) + + operator = UserOperator(db_client) + + current_user = get_session(req)["user_info"] + user = await operator.update_user(current_user, patch_ops if isinstance(patch_ops, list) else [patch_ops]) + + body = ujson.dumps({"userId": user}) + LOG.info(f"PATCH user with ID {user} was successful.") + return web.Response(body=body, status=200, content_type="application/json") + + async def delete_user(self, req: Request) -> Response: + """Delete user from database. + + :param req: DELETE request + :raises: HTTPUnauthorized if not current user + :returns: HTTPNoContent response + """ + user_id = req.match_info["userId"] + if user_id != "current": + LOG.info(f"User ID {user_id} delete was requested") + raise web.HTTPUnauthorized(reason="Only current user deletion is allowed") + db_client = req.app["db_client"] + operator = UserOperator(db_client) + fold_ops = FolderOperator(db_client) + obj_ops = Operator(db_client) + + current_user = get_session(req)["user_info"] + user = await operator.read_user(current_user) + + for folder_id in user["folders"]: + _folder = await fold_ops.read_folder(folder_id) + if "published" in _folder and not _folder["published"]: + for obj in _folder["drafts"] + _folder["metadataObjects"]: + await obj_ops.delete_metadata_object(obj["schema"], obj["accessionId"]) + await fold_ops.delete_folder(folder_id) + + for tmpl in user["templates"]: + await obj_ops.delete_metadata_object(tmpl["schema"], tmpl["accessionId"]) + + await operator.delete_user(current_user) + LOG.info(f"DELETE user with ID {current_user} was successful.") + + cookie = decrypt_cookie(req) + + try: + req.app["Session"].pop(cookie["id"]) + req.app["Cookies"].remove(cookie["id"]) + except KeyError: + pass + + response = web.HTTPSeeOther(f"{aai_config['redirect']}/") + response.headers["Location"] = ( + "/" if aai_config["redirect"] == aai_config["domain"] else f"{aai_config['redirect']}/" + ) + LOG.debug("Logged out user ") + raise response + + async def _get_user_items(self, req: Request, user: Dict, item_type: str) -> Tuple[Dict, CIMultiDict[str]]: + """Get draft templates owned by the user with pagination values. + + :param req: GET request + :param user: User object + :param item_type: Name of the items ("templates" or "folders") + :raises: HTTPUnauthorized if not current user + :returns: Paginated list of user draft templates and link header + """ + # Check item_type parameter is not faulty + if item_type not in ["templates", "folders"]: + reason = f"{item_type} is a faulty item parameter. Should be either folders or templates" + LOG.error(reason) + raise web.HTTPBadRequest(reason=reason) + + page = self._get_page_param(req, "page", 1) + per_page = self._get_page_param(req, "per_page", 5) + + db_client = req.app["db_client"] + operator = UserOperator(db_client) + user_id = req.match_info["userId"] + + query = {"userId": user} + + items, total_items = await operator.filter_user(query, item_type, page, per_page) + LOG.info(f"GET user with ID {user_id} was successful.") + + result = { + "page": { + "page": page, + "size": per_page, + "totalPages": ceil(total_items / per_page), + "total" + item_type.title(): total_items, + }, + item_type: items, + } + + url = f"{req.scheme}://{req.host}{req.path}" + link_headers = await self._header_links(url, page, per_page, total_items) + LOG.debug(f"Pagination header links: {link_headers}") + LOG.info(f"Querying for user's {item_type} resulted in {total_items} {item_type}") + return result, link_headers diff --git a/metadata_backend/server.py b/metadata_backend/server.py index c39d3364e..a600b2d06 100644 --- a/metadata_backend/server.py +++ b/metadata_backend/server.py @@ -1,26 +1,23 @@ """Functions to launch backend server.""" import asyncio +import secrets +import time import uvloop from aiohttp import web from cryptography.fernet import Fernet -import secrets -import time -from .api.handlers import ( - RESTAPIHandler, - StaticHandler, - SubmissionAPIHandler, - FolderAPIHandler, - UserAPIHandler, - ObjectAPIHandler, - TemplatesAPIHandler, -) from .api.auth import AccessHandler -from .api.middlewares import http_error_handler, check_login +from .api.handlers.api_handlers import RESTAPIHandler, StaticHandler +from .api.handlers.folder_handler import FolderAPIHandler +from .api.handlers.object_handler import ObjectAPIHandler +from .api.handlers.submission_handler import SubmissionAPIHandler +from .api.handlers.templates_handler import TemplatesAPIHandler +from .api.handlers.user_handler import UserAPIHandler from .api.health import HealthHandler -from .conf.conf import create_db_client, frontend_static_files, aai_config +from .api.middlewares import check_login, http_error_handler +from .conf.conf import aai_config, create_db_client, frontend_static_files from .helpers.logger import LOG asyncio.set_event_loop_policy(uvloop.EventLoopPolicy()) diff --git a/tests/test_handlers.py b/tests/test_handlers.py index b7d62a419..6e62bb1b1 100644 --- a/tests/test_handlers.py +++ b/tests/test_handlers.py @@ -4,12 +4,13 @@ from unittest.mock import patch from aiohttp import FormData -from aiohttp.test_utils import AioHTTPTestCase - +from aiohttp.test_utils import AioHTTPTestCase, make_mocked_coro +from metadata_backend.api.handlers.api_handlers import RESTAPIHandler from metadata_backend.api.middlewares import generate_cookie -from .mockups import get_request_with_fernet from metadata_backend.server import init +from .mockups import get_request_with_fernet + class HandlersTestCase(AioHTTPTestCase): """API endpoint class test cases.""" @@ -22,6 +23,14 @@ async def get_application(self): server["Session"] = {"user_info": ["value", "value"]} return server + def authenticate(self, client): + """Authenticate client.""" + request = get_request_with_fernet() + request.app["Crypt"] = client.app["Crypt"] + cookie, cookiestring = generate_cookie(request) + client.app["Session"] = {cookie["id"]: {"access_token": "mock_token_value", "user_info": {}}} + client._session.cookie_jar.update_cookies({"MTD_SESSION": cookiestring}) + async def setUpAsync(self): """Configure default values for testing and other modules. @@ -34,6 +43,7 @@ async def setUpAsync(self): self.client = await self.get_client(self.server) await self.client.start_server() + self.authenticate(self.client) self.test_ega_string = "EGA123456" self.query_accessionId = ("EDAG3991701442770179",) @@ -66,15 +76,8 @@ async def setUpAsync(self): "templates": [], "folders": ["FOL12345678"], } - self.test_draft_doi = {"fullDOI": "10.xxxx/yyyyy", "dataset": "https://doi.org/10.xxxx/yyyyy"} - class_parser = "metadata_backend.api.handlers.XMLToJSONParser" - class_operator = "metadata_backend.api.handlers.Operator" - class_xmloperator = "metadata_backend.api.handlers.XMLOperator" - class_folderoperator = "metadata_backend.api.handlers.FolderOperator" - class_useroperator = "metadata_backend.api.handlers.UserOperator" - class_doihandler = "metadata_backend.api.handlers.DOIHandler" - operator_config = { + self.operator_config = { "read_metadata_object.side_effect": self.fake_operator_read_metadata_object, "query_metadata_database.side_effect": self.fake_operator_query_metadata_object, "create_metadata_object.side_effect": self.fake_operator_create_metadata_object, @@ -82,53 +85,27 @@ async def setUpAsync(self): "update_metadata_object.side_effect": self.fake_operator_update_metadata_object, "replace_metadata_object.side_effect": self.fake_operator_replace_metadata_object, } - xmloperator_config = { + self.xmloperator_config = { "read_metadata_object.side_effect": self.fake_xmloperator_read_metadata_object, "create_metadata_object.side_effect": self.fake_xmloperator_create_metadata_object, "replace_metadata_object.side_effect": self.fake_xmloperator_replace_metadata_object, } - folderoperator_config = { + self.folderoperator_config = { "create_folder.side_effect": self.fake_folderoperator_create_folder, "read_folder.side_effect": self.fake_folderoperator_read_folder, "delete_folder.side_effect": self.fake_folderoperator_delete_folder, "check_object_in_folder.side_effect": self.fake_folderoperator_check_object, - "get_collection_objects.side_effect": self.fake_folderoperator_get_collection_objects, } - useroperator_config = { + self.useroperator_config = { "create_user.side_effect": self.fake_useroperator_create_user, "read_user.side_effect": self.fake_useroperator_read_user, "filter_user.side_effect": self.fake_useroperator_filter_user, - "check_user_has_doc.side_effect": self.fake_useroperator_user_has_folder, } - self.patch_parser = patch(class_parser, spec=True) - self.patch_operator = patch(class_operator, **operator_config, spec=True) - self.patch_xmloperator = patch(class_xmloperator, **xmloperator_config, spec=True) - self.patch_folderoperator = patch(class_folderoperator, **folderoperator_config, spec=True) - self.patch_useroperator = patch(class_useroperator, **useroperator_config, spec=True) - self.patch_doihandler = patch(class_doihandler, spec=True) - self.MockedParser = self.patch_parser.start() - self.MockedOperator = self.patch_operator.start() - self.MockedXMLOperator = self.patch_xmloperator.start() - self.MockedFolderOperator = self.patch_folderoperator.start() - self.MockedUserOperator = self.patch_useroperator.start() - self.MockedDoiHandler = self.patch_doihandler.start() - # Set up authentication - request = get_request_with_fernet() - request.app["Crypt"] = self.client.app["Crypt"] - cookie, cookiestring = generate_cookie(request) - self.client.app["Session"] = {cookie["id"]: {"access_token": "mock_token_value", "user_info": {}}} - self.client._session.cookie_jar.update_cookies({"MTD_SESSION": cookiestring}) + RESTAPIHandler._handle_check_ownedby_user = make_mocked_coro(True) async def tearDownAsync(self): """Cleanup mocked stuff.""" - self.patch_parser.stop() - self.patch_operator.stop() - self.patch_xmloperator.stop() - self.patch_folderoperator.stop() - self.patch_useroperator.stop() - self.patch_doihandler.stop() - await self.client.close() def create_submission_data(self, files): @@ -195,14 +172,6 @@ async def fake_folderoperator_check_object(self, schema_type, accession_id): data = True, self.folder_id, False return data - async def fake_folderoperator_get_collection_objects(self, schema_type, accession_id): - """Fake get collection of objects in folder.""" - return ["EDAG3991701442770179", "EGA123456"] - - async def fake_useroperator_user_has_folder(self, schema_type, user_id, folder_id): - """Fake check object in folder.""" - return True - async def fake_useroperator_create_user(self, content): """Fake user operation to return mocked userId.""" return self.user_id @@ -215,37 +184,9 @@ async def fake_useroperator_filter_user(self, query, item_type, page, per_page): """Fake read operation to return mocked user.""" return self.test_user[item_type], len(self.test_user[item_type]) - async def test_submit_endpoint_submission_does_not_fail(self): - """Test that submission with valid SUBMISSION.xml does not fail.""" - files = [("submission", "ERA521986_valid.xml")] - data = self.create_submission_data(files) - response = await self.client.post("/submit", data=data) - self.assertEqual(response.status, 200) - self.assertEqual(response.content_type, "application/json") - async def test_submit_endpoint_fails_without_submission_xml(self): - """Test that basic POST submission fails with no submission.xml. - - User should also be notified for missing file. - """ - files = [("analysis", "ERZ266973.xml")] - data = self.create_submission_data(files) - response = await self.client.post("/submit", data=data) - failure_text = "There must be a submission.xml file in submission." - self.assertEqual(response.status, 400) - self.assertIn(failure_text, await response.text()) - - async def test_submit_endpoint_fails_with_many_submission_xmls(self): - """Test submission fails when there's too many submission.xml -files. - - User should be notified for submitting too many files. - """ - files = [("submission", "ERA521986_valid.xml"), ("submission", "ERA521986_valid2.xml")] - data = self.create_submission_data(files) - response = await self.client.post("/submit", data=data) - failure_text = "You should submit only one submission.xml file." - self.assertEqual(response.status, 400) - self.assertIn(failure_text, await response.text()) +class APIHandlerTestCase(HandlersTestCase): + """Schema API endpoint class test cases.""" async def test_correct_schema_types_are_returned(self): """Test api endpoint for all schema types.""" @@ -285,6 +226,138 @@ async def test_raises_not_found_schema(self): resp_json = await response.json() self.assertEqual(resp_json["detail"], "The provided schema type could not be found. (project)") + +class SubmissionHandlerTestCase(HandlersTestCase): + """Submission API endpoint class test cases.""" + + async def setUpAsync(self): + """Configure default values for testing and other modules. + + This patches used modules and sets default return values for their + methods. + """ + + await super().setUpAsync() + class_parser = "metadata_backend.api.handlers.submission_handler.XMLToJSONParser" + self.patch_parser = patch(class_parser, spec=True) + self.MockedParser = self.patch_parser.start() + + class_xmloperator = "metadata_backend.api.handlers.submission_handler.XMLOperator" + self.patch_xmloperator = patch(class_xmloperator, **self.xmloperator_config, spec=True) + self.MockedXMLOperator = self.patch_xmloperator.start() + + async def tearDownAsync(self): + """Cleanup mocked stuff.""" + await super().tearDownAsync() + self.patch_parser.stop() + self.patch_xmloperator.stop() + + async def test_submit_endpoint_submission_does_not_fail(self): + """Test that submission with valid SUBMISSION.xml does not fail.""" + files = [("submission", "ERA521986_valid.xml")] + data = self.create_submission_data(files) + response = await self.client.post("/submit", data=data) + self.assertEqual(response.status, 200) + self.assertEqual(response.content_type, "application/json") + + async def test_submit_endpoint_fails_without_submission_xml(self): + """Test that basic POST submission fails with no submission.xml. + + User should also be notified for missing file. + """ + files = [("analysis", "ERZ266973.xml")] + data = self.create_submission_data(files) + response = await self.client.post("/submit", data=data) + failure_text = "There must be a submission.xml file in submission." + self.assertEqual(response.status, 400) + self.assertIn(failure_text, await response.text()) + + async def test_submit_endpoint_fails_with_many_submission_xmls(self): + """Test submission fails when there's too many submission.xml -files. + + User should be notified for submitting too many files. + """ + files = [("submission", "ERA521986_valid.xml"), ("submission", "ERA521986_valid2.xml")] + data = self.create_submission_data(files) + response = await self.client.post("/submit", data=data) + failure_text = "You should submit only one submission.xml file." + self.assertEqual(response.status, 400) + self.assertIn(failure_text, await response.text()) + + async def test_validation_passes_for_valid_xml(self): + """Test validation endpoint for valid xml.""" + files = [("study", "SRP000539.xml")] + data = self.create_submission_data(files) + response = await self.client.post("/validate", data=data) + self.assertEqual(response.status, 200) + self.assertIn('{"isValid":true}', await response.text()) + + async def test_validation_fails_bad_schema(self): + """Test validation fails for bad schema and valid xml.""" + files = [("fake", "SRP000539.xml")] + data = self.create_submission_data(files) + response = await self.client.post("/validate", data=data) + self.assertEqual(response.status, 404) + + async def test_validation_fails_for_invalid_xml_syntax(self): + """Test validation endpoint for XML with bad syntax.""" + files = [("study", "SRP000539_invalid.xml")] + data = self.create_submission_data(files) + response = await self.client.post("/validate", data=data) + resp_dict = await response.json() + self.assertEqual(response.status, 200) + self.assertIn("Faulty XML file was given, mismatched tag", resp_dict["detail"]["reason"]) + + async def test_validation_fails_for_invalid_xml(self): + """Test validation endpoint for invalid xml.""" + files = [("study", "SRP000539_invalid2.xml")] + data = self.create_submission_data(files) + response = await self.client.post("/validate", data=data) + resp_dict = await response.json() + self.assertEqual(response.status, 200) + self.assertIn("value must be one of", resp_dict["detail"]["reason"]) + + async def test_validation_fails_with_too_many_files(self): + """Test validation endpoint for too many files.""" + files = [("submission", "ERA521986_valid.xml"), ("submission", "ERA521986_valid2.xml")] + data = self.create_submission_data(files) + response = await self.client.post("/validate", data=data) + reason = "Only one file can be sent to this endpoint at a time." + self.assertEqual(response.status, 400) + self.assertIn(reason, await response.text()) + + +class ObjectHandlerTestCase(HandlersTestCase): + """Object API endpoint class test cases.""" + + async def setUpAsync(self): + """Configure default values for testing and other modules. + + This patches used modules and sets default return values for their + methods. + """ + + await super().setUpAsync() + + class_xmloperator = "metadata_backend.api.handlers.object_handler.XMLOperator" + self.patch_xmloperator = patch(class_xmloperator, **self.xmloperator_config, spec=True) + self.MockedXMLOperator = self.patch_xmloperator.start() + + class_operator = "metadata_backend.api.handlers.object_handler.Operator" + self.patch_operator = patch(class_operator, **self.operator_config, spec=True) + self.MockedOperator = self.patch_operator.start() + + class_folderoperator = "metadata_backend.api.handlers.object_handler.FolderOperator" + self.patch_folderoperator = patch(class_folderoperator, **self.folderoperator_config, spec=True) + self.MockedFolderOperator = self.patch_folderoperator.start() + + async def tearDownAsync(self): + """Cleanup mocked stuff.""" + await super().tearDownAsync() + self.patch_xmloperator.stop() + self.patch_folderoperator.stop() + self.patch_operator.stop() + async def test_submit_object_works(self): """Test that submission is handled, XMLOperator is called.""" files = [("study", "SRP000539.xml")] @@ -421,6 +494,7 @@ async def test_submit_object_fails_with_too_many_files(self): self.assertEqual(response.status, 400) self.assertIn(reason, await response.text()) + # handle_check_ownedby_user.return_value = True async def test_get_object(self): """Test that accessionId returns correct JSON object.""" url = f"/objects/study/{self.query_accessionId}" @@ -447,6 +521,7 @@ async def test_get_object_as_xml(self): async def test_query_is_called_and_returns_json_in_correct_format(self): """Test query method calls operator and returns mocked JSON object.""" + RESTAPIHandler._handle_user_objects_collection = make_mocked_coro(["EDAG3991701442770179", "EGA123456"]) url = f"/objects/study?studyType=foo&name=bar&page={self.page_num}" f"&per_page={self.page_size}" response = await self.client.get(url) self.assertEqual(response.status, 200) @@ -479,48 +554,6 @@ async def test_query_fails_with_xml_format(self): self.assertEqual(response.status, 400) self.assertIn("xml-formatted query results are not supported", json_resp["detail"]) - async def test_validation_passes_for_valid_xml(self): - """Test validation endpoint for valid xml.""" - files = [("study", "SRP000539.xml")] - data = self.create_submission_data(files) - response = await self.client.post("/validate", data=data) - self.assertEqual(response.status, 200) - self.assertIn('{"isValid":true}', await response.text()) - - async def test_validation_fails_bad_schema(self): - """Test validation fails for bad schema and valid xml.""" - files = [("fake", "SRP000539.xml")] - data = self.create_submission_data(files) - response = await self.client.post("/validate", data=data) - self.assertEqual(response.status, 404) - - async def test_validation_fails_for_invalid_xml_syntax(self): - """Test validation endpoint for XML with bad syntax.""" - files = [("study", "SRP000539_invalid.xml")] - data = self.create_submission_data(files) - response = await self.client.post("/validate", data=data) - resp_dict = await response.json() - self.assertEqual(response.status, 200) - self.assertIn("Faulty XML file was given, mismatched tag", resp_dict["detail"]["reason"]) - - async def test_validation_fails_for_invalid_xml(self): - """Test validation endpoint for invalid xml.""" - files = [("study", "SRP000539_invalid2.xml")] - data = self.create_submission_data(files) - response = await self.client.post("/validate", data=data) - resp_dict = await response.json() - self.assertEqual(response.status, 200) - self.assertIn("value must be one of", resp_dict["detail"]["reason"]) - - async def test_validation_fails_with_too_many_files(self): - """Test validation endpoint for too many files.""" - files = [("submission", "ERA521986_valid.xml"), ("submission", "ERA521986_valid2.xml")] - data = self.create_submission_data(files) - response = await self.client.post("/validate", data=data) - reason = "Only one file can be sent to this endpoint at a time." - self.assertEqual(response.status, 400) - self.assertIn(reason, await response.text()) - async def test_operations_fail_for_wrong_schema_type(self): """Test 404 error is raised if incorrect schema name is given.""" get_resp = await self.client.get("/objects/bad_scehma_name/some_id") @@ -557,6 +590,174 @@ async def test_query_with_invalid_pagination_params(self): get_resp = await self.client.get("/objects/study?per_page=0") self.assertEqual(get_resp.status, 400) + +class UserHandlerTestCase(HandlersTestCase): + """User API endpoint class test cases.""" + + async def setUpAsync(self): + """Configure default values for testing and other modules. + + This patches used modules and sets default return values for their + methods. + """ + + await super().setUpAsync() + class_useroperator = "metadata_backend.api.handlers.user_handler.UserOperator" + self.patch_useroperator = patch(class_useroperator, **self.useroperator_config, spec=True) + self.MockedUserOperator = self.patch_useroperator.start() + + class_folderoperator = "metadata_backend.api.handlers.user_handler.FolderOperator" + self.patch_folderoperator = patch(class_folderoperator, **self.folderoperator_config, spec=True) + self.MockedFolderOperator = self.patch_folderoperator.start() + + class_operator = "metadata_backend.api.handlers.user_handler.Operator" + self.patch_operator = patch(class_operator, **self.operator_config, spec=True) + self.MockedOperator = self.patch_operator.start() + + async def tearDownAsync(self): + """Cleanup mocked stuff.""" + await super().tearDownAsync() + self.patch_useroperator.stop() + self.patch_folderoperator.stop() + self.patch_operator.stop() + + async def test_get_user_works(self): + """Test user object is returned when correct user id is given.""" + response = await self.client.get("/users/current") + self.assertEqual(response.status, 200) + self.MockedUserOperator().read_user.assert_called_once() + json_resp = await response.json() + self.assertEqual(self.test_user, json_resp) + + async def test_get_user_drafts_with_no_drafts(self): + """Test getting user drafts when user has no drafts.""" + response = await self.client.get("/users/current?items=templates") + self.assertEqual(response.status, 200) + self.MockedUserOperator().filter_user.assert_called_once() + json_resp = await response.json() + result = { + "page": { + "page": 1, + "size": 5, + "totalPages": 0, + "totalTemplates": 0, + }, + "templates": [], + } + self.assertEqual(json_resp, result) + + async def test_get_user_templates_with_1_template(self): + """Test getting user templates when user has 1 draft.""" + user = self.test_user + user["templates"].append(self.metadata_json) + self.MockedUserOperator().filter_user.return_value = (user["templates"], 1) + response = await self.client.get("/users/current?items=templates") + self.assertEqual(response.status, 200) + self.MockedUserOperator().filter_user.assert_called_once() + json_resp = await response.json() + result = { + "page": { + "page": 1, + "size": 5, + "totalPages": 1, + "totalTemplates": 1, + }, + "templates": [self.metadata_json], + } + self.assertEqual(json_resp, result) + + async def test_get_user_folder_list(self): + """Test get user with folders url returns a folder ID.""" + self.MockedUserOperator().filter_user.return_value = (self.test_user["folders"], 1) + response = await self.client.get("/users/current?items=folders") + self.assertEqual(response.status, 200) + self.MockedUserOperator().filter_user.assert_called_once() + json_resp = await response.json() + result = { + "page": { + "page": 1, + "size": 5, + "totalPages": 1, + "totalFolders": 1, + }, + "folders": ["FOL12345678"], + } + self.assertEqual(json_resp, result) + + async def test_get_user_items_with_bad_param(self): + """Test that error is raised if items parameter in query is not templates or folders.""" + response = await self.client.get("/users/current?items=wrong_thing") + self.assertEqual(response.status, 400) + json_resp = await response.json() + self.assertEqual( + json_resp["detail"], "wrong_thing is a faulty item parameter. Should be either folders or templates" + ) + + async def test_user_deletion_is_called(self): + """Test that user object would be deleted.""" + self.MockedUserOperator().read_user.return_value = self.test_user + self.MockedUserOperator().delete_user.return_value = None + await self.client.delete("/users/current") + self.MockedUserOperator().read_user.assert_called_once() + self.MockedUserOperator().delete_user.assert_called_once() + + async def test_update_user_fails_with_wrong_key(self): + """Test that user object does not update when forbidden keys are provided.""" + data = [{"op": "add", "path": "/userId"}] + response = await self.client.patch("/users/current", json=data) + self.assertEqual(response.status, 400) + json_resp = await response.json() + reason = "Request contains '/userId' key that cannot be updated to user object" + self.assertEqual(reason, json_resp["detail"]) + + async def test_update_user_passes(self): + """Test that user object would update with correct keys.""" + self.MockedUserOperator().update_user.return_value = self.user_id + data = [{"op": "add", "path": "/templates/-", "value": [{"accessionId": "3", "schema": "sample"}]}] + response = await self.client.patch("/users/current", json=data) + self.MockedUserOperator().update_user.assert_called_once() + self.assertEqual(response.status, 200) + json_resp = await response.json() + self.assertEqual(json_resp["userId"], self.user_id) + + +class FolderHandlerTestCase(HandlersTestCase): + """Folder API endpoint class test cases.""" + + async def setUpAsync(self): + """Configure default values for testing and other modules. + + This patches used modules and sets default return values for their + methods. + """ + + await super().setUpAsync() + + self.test_draft_doi = {"fullDOI": "10.xxxx/yyyyy", "dataset": "https://doi.org/10.xxxx/yyyyy"} + class_doihandler = "metadata_backend.api.handlers.folder_handler.DOIHandler" + self.patch_doihandler = patch(class_doihandler, spec=True) + self.MockedDoiHandler = self.patch_doihandler.start() + + class_folderoperator = "metadata_backend.api.handlers.folder_handler.FolderOperator" + self.patch_folderoperator = patch(class_folderoperator, **self.folderoperator_config, spec=True) + self.MockedFolderOperator = self.patch_folderoperator.start() + + class_useroperator = "metadata_backend.api.handlers.folder_handler.UserOperator" + self.patch_useroperator = patch(class_useroperator, **self.useroperator_config, spec=True) + self.MockedUserOperator = self.patch_useroperator.start() + + class_operator = "metadata_backend.api.handlers.folder_handler.Operator" + self.patch_operator = patch(class_operator, **self.operator_config, spec=True) + self.MockedOperator = self.patch_operator.start() + + async def tearDownAsync(self): + """Cleanup mocked stuff.""" + await super().tearDownAsync() + self.patch_doihandler.stop() + self.patch_folderoperator.stop() + self.patch_useroperator.stop() + self.patch_operator.stop() + async def test_folder_creation_works(self): """Test that folder is created and folder ID returned.""" json_req = {"name": "test", "description": "test folder"} @@ -634,6 +835,8 @@ async def test_get_folders_with_bad_params(self): async def test_get_folder_works(self): """Test folder is returned when correct folder id is given.""" + # RESTAPIHandler._handle_check_ownedby_user = make_mocked_coro(True) + response = await self.client.get("/folders/FOL12345678") self.assertEqual(response.status, 200) self.MockedFolderOperator().read_folder.assert_called_once() @@ -677,102 +880,3 @@ async def test_folder_deletion_is_called(self): self.MockedFolderOperator().read_folder.assert_called_once() self.MockedFolderOperator().delete_folder.assert_called_once() self.assertEqual(response.status, 204) - - async def test_get_user_works(self): - """Test user object is returned when correct user id is given.""" - response = await self.client.get("/users/current") - self.assertEqual(response.status, 200) - self.MockedUserOperator().read_user.assert_called_once() - json_resp = await response.json() - self.assertEqual(self.test_user, json_resp) - - async def test_get_user_drafts_with_no_drafts(self): - """Test getting user drafts when user has no drafts.""" - response = await self.client.get("/users/current?items=templates") - self.assertEqual(response.status, 200) - self.MockedUserOperator().filter_user.assert_called_once() - json_resp = await response.json() - result = { - "page": { - "page": 1, - "size": 5, - "totalPages": 0, - "totalTemplates": 0, - }, - "templates": [], - } - self.assertEqual(json_resp, result) - - async def test_get_user_templates_with_1_template(self): - """Test getting user templates when user has 1 draft.""" - user = self.test_user - user["templates"].append(self.metadata_json) - self.MockedUserOperator().filter_user.return_value = (user["templates"], 1) - response = await self.client.get("/users/current?items=templates") - self.assertEqual(response.status, 200) - self.MockedUserOperator().filter_user.assert_called_once() - json_resp = await response.json() - result = { - "page": { - "page": 1, - "size": 5, - "totalPages": 1, - "totalTemplates": 1, - }, - "templates": [self.metadata_json], - } - self.assertEqual(json_resp, result) - - async def test_get_user_folder_list(self): - """Test get user with folders url returns a folder ID.""" - self.MockedUserOperator().filter_user.return_value = (self.test_user["folders"], 1) - response = await self.client.get("/users/current?items=folders") - self.assertEqual(response.status, 200) - self.MockedUserOperator().filter_user.assert_called_once() - json_resp = await response.json() - result = { - "page": { - "page": 1, - "size": 5, - "totalPages": 1, - "totalFolders": 1, - }, - "folders": ["FOL12345678"], - } - self.assertEqual(json_resp, result) - - async def test_get_user_items_with_bad_param(self): - """Test that error is raised if items parameter in query is not templates or folders.""" - response = await self.client.get("/users/current?items=wrong_thing") - self.assertEqual(response.status, 400) - json_resp = await response.json() - self.assertEqual( - json_resp["detail"], "wrong_thing is a faulty item parameter. Should be either folders or templates" - ) - - async def test_user_deletion_is_called(self): - """Test that user object would be deleted.""" - self.MockedUserOperator().read_user.return_value = self.test_user - self.MockedUserOperator().delete_user.return_value = None - await self.client.delete("/users/current") - self.MockedUserOperator().read_user.assert_called_once() - self.MockedUserOperator().delete_user.assert_called_once() - - async def test_update_user_fails_with_wrong_key(self): - """Test that user object does not update when forbidden keys are provided.""" - data = [{"op": "add", "path": "/userId"}] - response = await self.client.patch("/users/current", json=data) - self.assertEqual(response.status, 400) - json_resp = await response.json() - reason = "Request contains '/userId' key that cannot be updated to user object" - self.assertEqual(reason, json_resp["detail"]) - - async def test_update_user_passes(self): - """Test that user object would update with correct keys.""" - self.MockedUserOperator().update_user.return_value = self.user_id - data = [{"op": "add", "path": "/templates/-", "value": [{"accessionId": "3", "schema": "sample"}]}] - response = await self.client.patch("/users/current", json=data) - self.MockedUserOperator().update_user.assert_called_once() - self.assertEqual(response.status, 200) - json_resp = await response.json() - self.assertEqual(json_resp["userId"], self.user_id) From 77fe2a7dcd02ce56228898674b01af57ade64893 Mon Sep 17 00:00:00 2001 From: Evgenia Lyjina Date: Wed, 29 Dec 2021 08:55:03 +0000 Subject: [PATCH 2/2] Update naming scheme --- metadata_backend/api/handlers/common.py | 56 +++++++++++ .../handlers/{folder_handler.py => folder.py} | 2 +- .../handlers/{object_handler.py => object.py} | 7 +- .../handlers/{api_handlers.py => restapi.py} | 95 +------------------ metadata_backend/api/handlers/static.py | 47 +++++++++ .../{submission_handler.py => submission.py} | 6 +- .../{templates_handler.py => template.py} | 2 +- .../api/handlers/{user_handler.py => user.py} | 2 +- metadata_backend/server.py | 13 +-- tests/test_handlers.py | 26 ++--- 10 files changed, 135 insertions(+), 121 deletions(-) create mode 100644 metadata_backend/api/handlers/common.py rename metadata_backend/api/handlers/{folder_handler.py => folder.py} (99%) rename metadata_backend/api/handlers/{object_handler.py => object.py} (98%) rename metadata_backend/api/handlers/{api_handlers.py => restapi.py} (69%) create mode 100644 metadata_backend/api/handlers/static.py rename metadata_backend/api/handlers/{submission_handler.py => submission.py} (97%) rename metadata_backend/api/handlers/{templates_handler.py => template.py} (99%) rename metadata_backend/api/handlers/{user_handler.py => user.py} (99%) diff --git a/metadata_backend/api/handlers/common.py b/metadata_backend/api/handlers/common.py new file mode 100644 index 000000000..ff1470dd0 --- /dev/null +++ b/metadata_backend/api/handlers/common.py @@ -0,0 +1,56 @@ +"""Functions shared between handlers.""" +from typing import List, Tuple, cast + +from aiohttp import BodyPartReader, web +from aiohttp.web import Request + +from ...conf.conf import schema_types +from ...helpers.logger import LOG + + +async def extract_xml_upload(req: Request, extract_one: bool = False) -> List[Tuple[str, str]]: + """Extract submitted xml-file(s) from multi-part request. + + Files are sorted to spesific order by their schema priorities (e.g. + submission should be processed before study). + + :param req: POST request containing "multipart/form-data" upload + :raises: HTTPBadRequest if request is not valid for multipart or multiple files sent. HTTPNotFound if + schema was not found. + :returns: content and schema type for each uploaded file, sorted by schema + type. + """ + files: List[Tuple[str, str]] = [] + try: + reader = await req.multipart() + except AssertionError: + reason = "Request does not have valid multipart/form content" + LOG.error(reason) + raise web.HTTPBadRequest(reason=reason) + while True: + part = await reader.next() + # Following is probably error in aiohttp type hints, fixing so + # mypy doesn't complain about it. No runtime consequences. + part = cast(BodyPartReader, part) + if not part: + break + if extract_one and files: + reason = "Only one file can be sent to this endpoint at a time." + LOG.error(reason) + raise web.HTTPBadRequest(reason=reason) + if part.name: + schema_type = part.name.lower() + if schema_type not in schema_types: + reason = f"Specified schema {schema_type} was not found." + LOG.error(reason) + raise web.HTTPNotFound(reason=reason) + data = [] + while True: + chunk = await part.read_chunk() + if not chunk: + break + data.append(chunk) + xml_content = "".join(x.decode("UTF-8") for x in data) + files.append((xml_content, schema_type)) + LOG.debug(f"processed file in {schema_type}") + return sorted(files, key=lambda x: schema_types[x[1]]["priority"]) diff --git a/metadata_backend/api/handlers/folder_handler.py b/metadata_backend/api/handlers/folder.py similarity index 99% rename from metadata_backend/api/handlers/folder_handler.py rename to metadata_backend/api/handlers/folder.py index 799029109..cc5910fc9 100644 --- a/metadata_backend/api/handlers/folder_handler.py +++ b/metadata_backend/api/handlers/folder.py @@ -14,7 +14,7 @@ from ...helpers.doi import DOIHandler from ...helpers.logger import LOG from ...helpers.validator import JSONValidator -from .api_handlers import RESTAPIHandler +from .restapi import RESTAPIHandler from ..middlewares import get_session from ..operators import FolderOperator, Operator, UserOperator diff --git a/metadata_backend/api/handlers/object_handler.py b/metadata_backend/api/handlers/object.py similarity index 98% rename from metadata_backend/api/handlers/object_handler.py rename to metadata_backend/api/handlers/object.py index 27b627e29..486649991 100644 --- a/metadata_backend/api/handlers/object_handler.py +++ b/metadata_backend/api/handlers/object.py @@ -9,8 +9,9 @@ from ...helpers.logger import LOG from ...helpers.validator import JSONValidator -from .api_handlers import RESTAPIHandler, _extract_xml_upload from ..operators import FolderOperator, Operator, XMLOperator +from .common import extract_xml_upload +from .restapi import RESTAPIHandler class ObjectAPIHandler(RESTAPIHandler): @@ -106,7 +107,7 @@ async def post_object(self, req: Request) -> Response: content: Union[Dict, str] operator: Union[Operator, XMLOperator] if req.content_type == "multipart/form-data": - files = await _extract_xml_upload(req, extract_one=True) + files = await extract_xml_upload(req, extract_one=True) content, _ = files[0] operator = XMLOperator(db_client) else: @@ -193,7 +194,7 @@ async def put_object(self, req: Request) -> Response: content: Union[Dict, str] operator: Union[Operator, XMLOperator] if req.content_type == "multipart/form-data": - files = await _extract_xml_upload(req, extract_one=True) + files = await extract_xml_upload(req, extract_one=True) content, _ = files[0] operator = XMLOperator(db_client) else: diff --git a/metadata_backend/api/handlers/api_handlers.py b/metadata_backend/api/handlers/restapi.py similarity index 69% rename from metadata_backend/api/handlers/api_handlers.py rename to metadata_backend/api/handlers/restapi.py index e50092292..943d58742 100644 --- a/metadata_backend/api/handlers/api_handlers.py +++ b/metadata_backend/api/handlers/restapi.py @@ -1,12 +1,10 @@ """Handle HTTP methods for server.""" import json -import mimetypes from math import ceil -from pathlib import Path -from typing import AsyncGenerator, Dict, List, Tuple, cast +from typing import AsyncGenerator, Dict, List import ujson -from aiohttp import BodyPartReader, web +from aiohttp import web from aiohttp.web import Request, Response from motor.motor_asyncio import AsyncIOMotorClient from multidict import CIMultiDict @@ -216,92 +214,3 @@ async def _header_links(self, url: str, page: int, size: int, total_objects: int link_headers = CIMultiDict(Link=f"{links}") LOG.debug("Link headers created") return link_headers - - -class StaticHandler: - """Handler for static routes, mostly frontend and 404.""" - - def __init__(self, frontend_static_files: Path) -> None: - """Initialize path to frontend static files folder.""" - self.path = frontend_static_files - - async def frontend(self, req: Request) -> Response: - """Serve requests related to frontend SPA. - - :param req: GET request - :returns: Response containing frontpage static file - """ - serve_path = self.path.joinpath("./" + req.path) - - if not serve_path.exists() or not serve_path.is_file(): - LOG.debug(f"{serve_path} was not found or is not a file - serving index.html") - serve_path = self.path.joinpath("./index.html") - - LOG.debug(f"Serve Frontend SPA {req.path} by {serve_path}.") - - mime_type = mimetypes.guess_type(serve_path.as_posix()) - - return Response(body=serve_path.read_bytes(), content_type=(mime_type[0] or "text/html")) - - def setup_static(self) -> Path: - """Set path for static js files and correct return mimetypes. - - :returns: Path to static js files folder - """ - mimetypes.init() - mimetypes.types_map[".js"] = "application/javascript" - mimetypes.types_map[".js.map"] = "application/json" - mimetypes.types_map[".svg"] = "image/svg+xml" - mimetypes.types_map[".css"] = "text/css" - mimetypes.types_map[".css.map"] = "application/json" - LOG.debug("static paths for SPA set.") - return self.path / "static" - - -# Private functions shared between handlers -async def _extract_xml_upload(req: Request, extract_one: bool = False) -> List[Tuple[str, str]]: - """Extract submitted xml-file(s) from multi-part request. - - Files are sorted to spesific order by their schema priorities (e.g. - submission should be processed before study). - - :param req: POST request containing "multipart/form-data" upload - :raises: HTTPBadRequest if request is not valid for multipart or multiple files sent. HTTPNotFound if - schema was not found. - :returns: content and schema type for each uploaded file, sorted by schema - type. - """ - files: List[Tuple[str, str]] = [] - try: - reader = await req.multipart() - except AssertionError: - reason = "Request does not have valid multipart/form content" - LOG.error(reason) - raise web.HTTPBadRequest(reason=reason) - while True: - part = await reader.next() - # Following is probably error in aiohttp type hints, fixing so - # mypy doesn't complain about it. No runtime consequences. - part = cast(BodyPartReader, part) - if not part: - break - if extract_one and files: - reason = "Only one file can be sent to this endpoint at a time." - LOG.error(reason) - raise web.HTTPBadRequest(reason=reason) - if part.name: - schema_type = part.name.lower() - if schema_type not in schema_types: - reason = f"Specified schema {schema_type} was not found." - LOG.error(reason) - raise web.HTTPNotFound(reason=reason) - data = [] - while True: - chunk = await part.read_chunk() - if not chunk: - break - data.append(chunk) - xml_content = "".join(x.decode("UTF-8") for x in data) - files.append((xml_content, schema_type)) - LOG.debug(f"processed file in {schema_type}") - return sorted(files, key=lambda x: schema_types[x[1]]["priority"]) diff --git a/metadata_backend/api/handlers/static.py b/metadata_backend/api/handlers/static.py new file mode 100644 index 000000000..5f93d6fa2 --- /dev/null +++ b/metadata_backend/api/handlers/static.py @@ -0,0 +1,47 @@ +"""Handle HTTP methods for server.""" +import mimetypes +from pathlib import Path + +from aiohttp.web import Request, Response + +from ...helpers.logger import LOG + + +class StaticHandler: + """Handler for static routes, mostly frontend and 404.""" + + def __init__(self, frontend_static_files: Path) -> None: + """Initialize path to frontend static files folder.""" + self.path = frontend_static_files + + async def frontend(self, req: Request) -> Response: + """Serve requests related to frontend SPA. + + :param req: GET request + :returns: Response containing frontpage static file + """ + serve_path = self.path.joinpath("./" + req.path) + + if not serve_path.exists() or not serve_path.is_file(): + LOG.debug(f"{serve_path} was not found or is not a file - serving index.html") + serve_path = self.path.joinpath("./index.html") + + LOG.debug(f"Serve Frontend SPA {req.path} by {serve_path}.") + + mime_type = mimetypes.guess_type(serve_path.as_posix()) + + return Response(body=serve_path.read_bytes(), content_type=(mime_type[0] or "text/html")) + + def setup_static(self) -> Path: + """Set path for static js files and correct return mimetypes. + + :returns: Path to static js files folder + """ + mimetypes.init() + mimetypes.types_map[".js"] = "application/javascript" + mimetypes.types_map[".js.map"] = "application/json" + mimetypes.types_map[".svg"] = "image/svg+xml" + mimetypes.types_map[".css"] = "text/css" + mimetypes.types_map[".css.map"] = "application/json" + LOG.debug("static paths for SPA set.") + return self.path / "static" diff --git a/metadata_backend/api/handlers/submission_handler.py b/metadata_backend/api/handlers/submission.py similarity index 97% rename from metadata_backend/api/handlers/submission_handler.py rename to metadata_backend/api/handlers/submission.py index d3492e4f2..5a95fa804 100644 --- a/metadata_backend/api/handlers/submission_handler.py +++ b/metadata_backend/api/handlers/submission.py @@ -13,8 +13,8 @@ from ...helpers.parser import XMLToJSONParser from ...helpers.schema_loader import SchemaNotFoundException, XMLSchemaLoader from ...helpers.validator import XMLValidator -from .api_handlers import _extract_xml_upload from ..operators import Operator, XMLOperator +from .common import extract_xml_upload class SubmissionAPIHandler: @@ -31,7 +31,7 @@ async def submit(self, req: Request) -> Response: :raises: HTTPBadRequest if request is missing some parameters or cannot be processed :returns: XML-based receipt from submission """ - files = await _extract_xml_upload(req) + files = await extract_xml_upload(req) schema_types = Counter(file[1] for file in files) if "submission" not in schema_types: reason = "There must be a submission.xml file in submission." @@ -92,7 +92,7 @@ async def validate(self, req: Request) -> Response: :param req: Multipart POST request with submission.xml and files :returns: JSON response indicating if validation was successful or not """ - files = await _extract_xml_upload(req, extract_one=True) + files = await extract_xml_upload(req, extract_one=True) xml_content, schema_type = files[0] validator = await self._perform_validation(schema_type, xml_content) return web.Response(body=validator.resp_body, content_type="application/json") diff --git a/metadata_backend/api/handlers/templates_handler.py b/metadata_backend/api/handlers/template.py similarity index 99% rename from metadata_backend/api/handlers/templates_handler.py rename to metadata_backend/api/handlers/template.py index f45413142..c2bccc2cc 100644 --- a/metadata_backend/api/handlers/templates_handler.py +++ b/metadata_backend/api/handlers/template.py @@ -9,7 +9,7 @@ from ...helpers.logger import LOG from ..middlewares import get_session from ..operators import Operator, UserOperator, XMLOperator -from .api_handlers import RESTAPIHandler +from .restapi import RESTAPIHandler class TemplatesAPIHandler(RESTAPIHandler): diff --git a/metadata_backend/api/handlers/user_handler.py b/metadata_backend/api/handlers/user.py similarity index 99% rename from metadata_backend/api/handlers/user_handler.py rename to metadata_backend/api/handlers/user.py index cd3a2bd9b..e77ce3d3d 100644 --- a/metadata_backend/api/handlers/user_handler.py +++ b/metadata_backend/api/handlers/user.py @@ -10,7 +10,7 @@ from ...conf.conf import aai_config from ...helpers.logger import LOG -from .api_handlers import RESTAPIHandler +from .restapi import RESTAPIHandler from ..middlewares import decrypt_cookie, get_session from ..operators import FolderOperator, Operator, UserOperator diff --git a/metadata_backend/server.py b/metadata_backend/server.py index a600b2d06..8b0be93d1 100644 --- a/metadata_backend/server.py +++ b/metadata_backend/server.py @@ -9,12 +9,13 @@ from cryptography.fernet import Fernet from .api.auth import AccessHandler -from .api.handlers.api_handlers import RESTAPIHandler, StaticHandler -from .api.handlers.folder_handler import FolderAPIHandler -from .api.handlers.object_handler import ObjectAPIHandler -from .api.handlers.submission_handler import SubmissionAPIHandler -from .api.handlers.templates_handler import TemplatesAPIHandler -from .api.handlers.user_handler import UserAPIHandler +from .api.handlers.restapi import RESTAPIHandler +from .api.handlers.static import StaticHandler +from .api.handlers.folder import FolderAPIHandler +from .api.handlers.object import ObjectAPIHandler +from .api.handlers.submission import SubmissionAPIHandler +from .api.handlers.template import TemplatesAPIHandler +from .api.handlers.user import UserAPIHandler from .api.health import HealthHandler from .api.middlewares import check_login, http_error_handler from .conf.conf import aai_config, create_db_client, frontend_static_files diff --git a/tests/test_handlers.py b/tests/test_handlers.py index 6e62bb1b1..ccc7346b3 100644 --- a/tests/test_handlers.py +++ b/tests/test_handlers.py @@ -5,7 +5,7 @@ from aiohttp import FormData from aiohttp.test_utils import AioHTTPTestCase, make_mocked_coro -from metadata_backend.api.handlers.api_handlers import RESTAPIHandler +from metadata_backend.api.handlers.restapi import RESTAPIHandler from metadata_backend.api.middlewares import generate_cookie from metadata_backend.server import init @@ -238,11 +238,11 @@ async def setUpAsync(self): """ await super().setUpAsync() - class_parser = "metadata_backend.api.handlers.submission_handler.XMLToJSONParser" + class_parser = "metadata_backend.api.handlers.submission.XMLToJSONParser" self.patch_parser = patch(class_parser, spec=True) self.MockedParser = self.patch_parser.start() - class_xmloperator = "metadata_backend.api.handlers.submission_handler.XMLOperator" + class_xmloperator = "metadata_backend.api.handlers.submission.XMLOperator" self.patch_xmloperator = patch(class_xmloperator, **self.xmloperator_config, spec=True) self.MockedXMLOperator = self.patch_xmloperator.start() @@ -339,15 +339,15 @@ async def setUpAsync(self): await super().setUpAsync() - class_xmloperator = "metadata_backend.api.handlers.object_handler.XMLOperator" + class_xmloperator = "metadata_backend.api.handlers.object.XMLOperator" self.patch_xmloperator = patch(class_xmloperator, **self.xmloperator_config, spec=True) self.MockedXMLOperator = self.patch_xmloperator.start() - class_operator = "metadata_backend.api.handlers.object_handler.Operator" + class_operator = "metadata_backend.api.handlers.object.Operator" self.patch_operator = patch(class_operator, **self.operator_config, spec=True) self.MockedOperator = self.patch_operator.start() - class_folderoperator = "metadata_backend.api.handlers.object_handler.FolderOperator" + class_folderoperator = "metadata_backend.api.handlers.object.FolderOperator" self.patch_folderoperator = patch(class_folderoperator, **self.folderoperator_config, spec=True) self.MockedFolderOperator = self.patch_folderoperator.start() @@ -602,15 +602,15 @@ async def setUpAsync(self): """ await super().setUpAsync() - class_useroperator = "metadata_backend.api.handlers.user_handler.UserOperator" + class_useroperator = "metadata_backend.api.handlers.user.UserOperator" self.patch_useroperator = patch(class_useroperator, **self.useroperator_config, spec=True) self.MockedUserOperator = self.patch_useroperator.start() - class_folderoperator = "metadata_backend.api.handlers.user_handler.FolderOperator" + class_folderoperator = "metadata_backend.api.handlers.user.FolderOperator" self.patch_folderoperator = patch(class_folderoperator, **self.folderoperator_config, spec=True) self.MockedFolderOperator = self.patch_folderoperator.start() - class_operator = "metadata_backend.api.handlers.user_handler.Operator" + class_operator = "metadata_backend.api.handlers.user.Operator" self.patch_operator = patch(class_operator, **self.operator_config, spec=True) self.MockedOperator = self.patch_operator.start() @@ -734,19 +734,19 @@ async def setUpAsync(self): await super().setUpAsync() self.test_draft_doi = {"fullDOI": "10.xxxx/yyyyy", "dataset": "https://doi.org/10.xxxx/yyyyy"} - class_doihandler = "metadata_backend.api.handlers.folder_handler.DOIHandler" + class_doihandler = "metadata_backend.api.handlers.folder.DOIHandler" self.patch_doihandler = patch(class_doihandler, spec=True) self.MockedDoiHandler = self.patch_doihandler.start() - class_folderoperator = "metadata_backend.api.handlers.folder_handler.FolderOperator" + class_folderoperator = "metadata_backend.api.handlers.folder.FolderOperator" self.patch_folderoperator = patch(class_folderoperator, **self.folderoperator_config, spec=True) self.MockedFolderOperator = self.patch_folderoperator.start() - class_useroperator = "metadata_backend.api.handlers.folder_handler.UserOperator" + class_useroperator = "metadata_backend.api.handlers.folder.UserOperator" self.patch_useroperator = patch(class_useroperator, **self.useroperator_config, spec=True) self.MockedUserOperator = self.patch_useroperator.start() - class_operator = "metadata_backend.api.handlers.folder_handler.Operator" + class_operator = "metadata_backend.api.handlers.folder.Operator" self.patch_operator = patch(class_operator, **self.operator_config, spec=True) self.MockedOperator = self.patch_operator.start()