Merge pull request #471 from Ouranosinc/svc-type

CHANGES.rst

@@ -9,6 +9,35 @@ Changes
 
 Features / Changes
 ~~~~~~~~~~~~~~~~~~~~~
+* Add ``type`` query parameter to multiple requests returning ``Services`` or ``Resources`` regrouped
+  by ``ServiceType``, either in general or for a given ``User`` or ``Group`` in order to limit listing in responses
+  and optimise some operations where only a subset of details are needed.
+* When requesting specific ``type`` with new query parameters, the relevant sections will always be added to the
+  response content, even when no ``Service`` are to be returned when ``User`` as no `Direct` or `Inherited` permissions
+  on it. This is to better illustrate that ``type`` was properly interpreted and indicate that nothing was found.
+* Using new ``type`` query to filter ``ServiceType``, improve ``Permissions`` listing in UI pages with faster processing
+  because ``Services`` that are not required (since they are not currently being displayed by the tab-panel view) can
+  be skipped entirely, removing the need to compute their underlying ``Resource`` and ``Permissions`` tree hierarchy.
+* Add various test utility improvements to parse and retrieve ``Permissions`` from HTML pages combo-boxes to facilitate
+  development and increase validation of UI functionalities.
+  This will also help for futures tests (relates to `#193 <https://github.com/Ouranosinc/Magpie/issues/193>`_).
+* Reapply ``list`` (prior name in ``2.x`` releases) as permitted alternative query parameter name to official
+  query parameter ``flatten`` for requests that support it.
+* Sort items by type and name for better readability of returned content by the various ``Service`` endpoints.
+
+Bug Fixes
+~~~~~~~~~~~~~~~~~~~~~
+* Replace invalid schema definitions using old ``combined`` query parameter by ``resolve`` query parameter actually
+  employed by request views in order to properly report this query parameter in the `OpenAPI` specification.
+* Apply ``resolve=true`` query parameter to UI page sub-request when resolving inherited user/group permissions in
+  order to display the highest priority ``Permission`` for each corresponding ``Resource`` in the tree hierarchy.
+  Without this option, the first permission was displayed based on naming ordering methodology, which made it more
+  confusing for administrators to understand how effective permissions could be obtained
+  (fixes `#463 <https://github.com/Ouranosinc/Magpie/issues/463>`_).
+* Fix a situation where the response from the API for ``GET /users/{}/resources`` endpoint would not correctly
+  list `Resolved Permissions` only for the top-most ``Resource`` in the hierarchy (i.e.: ``Service``) due to different
+  resolution methodologies applied between both types. This does **NOT** affect `Effective Resolution` which has its
+  own algorithm for access resolution to ``Resources``.
 * Add links to `Magpie's ReadTheDocs Terms <https://pavics-magpie.readthedocs.io/en/latest/glossary.html>`_ for
   all corresponding ``Permissions`` definitions rendered in information note within the UI ``User`` edit page.
   Notes indicate the resolution priority and methodology from the documentation to remind the administrator about what
@@ -17,9 +46,6 @@ Features / Changes
   within `Magpie's ReadTheDocs Permissions <https://pavics-magpie.readthedocs.io/en/latest/permissions.html>`_ page
   from a ``term`` glossary reference to corresponding detailed section reference in `Types of Permissions` chapter
   to avoid back and forth redirects between the `Permissions` page and their generic term glossary.
-
-Bug Fixes
-~~~~~~~~~~~~~~~~~~~~~
 * Fix incorrectly generated references from `Permissions` terms in glossary to detailed descriptions in `ReadTheDocs`.
 
 `3.15.1 <https://github.com/Ouranosinc/Magpie/tree/3.15.1>`_ (2021-09-29)

magpie/api/management/group/group_utils.py

@@ -20,7 +20,7 @@
 from magpie.api.management.group.group_formats import format_group
 from magpie.api.management.resource.resource_formats import format_resource
 from magpie.api.management.resource.resource_utils import check_valid_service_or_resource_permission
-from magpie.api.management.service.service_formats import format_service, format_service_resources
+from magpie.api.management.service import service_formats as sf
 from magpie.api.webhooks import WebhookAction, get_permission_update_params, process_webhook_requests
 from magpie.permissions import PermissionSet, PermissionType, format_permissions
 from magpie.services import SERVICE_TYPE_DICT
@@ -46,20 +46,24 @@ def get_all_group_names(db_session):
     return group_names
 
 
-def get_group_resources(group, db_session):
-    # type: (models.Group, Session) -> JSON
+def get_group_resources(group, db_session, service_types=None):
+    # type: (models.Group, Session, Optional[List[Str]]) -> JSON
     """
     Get formatted JSON body describing all service resources the ``group`` as permissions on.
     """
     json_response = {}
+    if service_types is None:
+        service_types = list(SERVICE_TYPE_DICT)
     for svc in list(ResourceService.all(models.Service, db_session=db_session)):
+        svc_type = str(svc.type)
+        if svc_type not in service_types:
+            continue
         svc_perms = get_group_service_permissions(group=group, service=svc, db_session=db_session)
         svc_name = str(svc.resource_name)
-        svc_type = str(svc.type)
         if svc_type not in json_response:
             json_response[svc_type] = {}
         res_perm_dict = get_group_service_resources_permissions_dict(group=group, service=svc, db_session=db_session)
-        json_response[svc_type][svc_name] = format_service_resources(
+        json_response[svc_type][svc_name] = sf.format_service_resources(
             svc,
             db_session=db_session,
             service_perms=svc_perms,
@@ -274,24 +278,28 @@ def delete_group_resource_permission_response(group, resource, permission, db_se
     return ax.valid_http(http_success=HTTPOk, detail=s.GroupServicePermission_DELETE_OkResponseSchema.description)
 
 
-def get_group_services(resources_permissions_dict, db_session):
-    # type: (JSON, Session) -> JSON
+def get_group_services(resources_permissions_dict, db_session, service_types=None):
+    # type: (JSON, Session, Optional[List[Str]]) -> JSON
     """
     Nest and regroup the resource permissions under corresponding root service types.
     """
     grp_svc_dict = {}
+    if service_types is None:
+        service_types = list(SERVICE_TYPE_DICT)
     for res_id, perms in resources_permissions_dict.items():
         svc = ResourceService.by_resource_id(resource_id=res_id, db_session=db_session)
         svc_type = str(svc.type)
+        if svc_type not in service_types:
+            continue
         svc_name = str(svc.resource_name)
         if svc_type not in grp_svc_dict:
             grp_svc_dict[svc_type] = {}
-        grp_svc_dict[svc_type][svc_name] = format_service(svc, perms, show_private_url=False)
+        grp_svc_dict[svc_type][svc_name] = sf.format_service(svc, perms, show_private_url=False)
     return grp_svc_dict
 
 
-def get_group_services_response(group, db_session):
-    # type: (models.Group, Session) -> HTTPException
+def get_group_services_response(group, db_session, service_types=None):
+    # type: (models.Group, Session, Optional[List[Str]]) -> HTTPException
     """
     Get validated response of services the group has permissions on.
 
@@ -301,7 +309,7 @@ def get_group_services_response(group, db_session):
     res_perm_dict = get_group_resources_permissions_dict(group,
                                                          resource_types=[models.Service.resource_type_name],
                                                          db_session=db_session)
-    grp_svc_json = ax.evaluate_call(lambda: get_group_services(res_perm_dict, db_session),
+    grp_svc_json = ax.evaluate_call(lambda: get_group_services(res_perm_dict, db_session, service_types=service_types),
                                     http_error=HTTPInternalServerError,
                                     msg_on_fail=s.GroupServices_InternalServerErrorResponseSchema.description,
                                     content={"group": format_group(group, basic_info=True)})
@@ -343,7 +351,7 @@ def get_group_service_permissions_response(group, service, db_session):
         lambda: format_permissions(get_group_service_permissions(group, service, db_session), PermissionType.APPLIED),
         http_error=HTTPInternalServerError,
         msg_on_fail=s.GroupServicePermissions_GET_InternalServerErrorResponseSchema.description,
-        content={"group": format_group(group, basic_info=True), "service": format_service(service)})
+        content={"group": format_group(group, basic_info=True), "service": sf.format_service(service)})
     return ax.valid_http(http_success=HTTPOk, content=svc_perms_found,
                          detail=s.GroupServicePermissions_GET_OkResponseSchema.description)
 
@@ -369,7 +377,7 @@ def get_group_service_resources_response(group, service, db_session):
     """
     svc_perms = get_group_service_permissions(group=group, service=service, db_session=db_session)
     res_perms = get_group_service_resources_permissions_dict(group=group, service=service, db_session=db_session)
-    svc_res_json = format_service_resources(
+    svc_res_json = sf.format_service_resources(
         service=service,
         db_session=db_session,
         service_perms=svc_perms,

magpie/api/management/group/group_views.py

@@ -8,6 +8,7 @@
 from magpie.api import schemas as s
 from magpie.api.management.group import group_formats as gf
 from magpie.api.management.group import group_utils as gu
+from magpie.api.management.service import service_utils as su
 from magpie.constants import get_constant
 
 
@@ -133,7 +134,9 @@ def get_group_services_view(request):
     List all services a group has permission on.
     """
     group = ar.get_group_matchdict_checked(request)
-    return gu.get_group_services_response(group, request.db)
+    service_types = ar.get_query_param(request, ["type", "types"], default="")
+    service_types = su.filter_service_types(service_types)
+    return gu.get_group_services_response(group, request.db, service_types=service_types)
 
 
 @s.GroupServicePermissionsAPI.get(schema=s.GroupServicePermissions_GET_RequestSchema,
@@ -215,7 +218,9 @@ def get_group_resources_view(request):
     List all resources a group has permission on.
     """
     group = ar.get_group_matchdict_checked(request)
-    grp_res_json = ax.evaluate_call(lambda: gu.get_group_resources(group, request.db),
+    service_types = ar.get_query_param(request, ["type", "types"], default="")
+    service_types = su.filter_service_types(service_types)
+    grp_res_json = ax.evaluate_call(lambda: gu.get_group_resources(group, request.db, service_types=service_types),
                                     fallback=lambda: request.db.rollback(),
                                     http_error=HTTPInternalServerError, content={"group": repr(group)},
                                     msg_on_fail=s.GroupResources_GET_InternalServerErrorResponseSchema.description)

magpie/api/management/service/service_utils.py

@@ -25,7 +25,7 @@
 
 if TYPE_CHECKING:
     # pylint: disable=W0611,unused-import
-    from typing import Iterable, Optional
+    from typing import Iterable, List, Optional
 
     from pyramid.httpexceptions import HTTPException
     from sqlalchemy.orm.session import Session
@@ -120,3 +120,23 @@ def add_service_getcapabilities_perms(service, db_session, group_name=None):
                                                            Permission.GET_CAPABILITIES.value, db_session)
         if perm is None:  # not set, create it
             create_group_resource_permission_response(group, service, Permission.GET_CAPABILITIES.value, db_session)
+
+
+def filter_service_types(service_query, default_services=False):
+    # type: (Optional[Str], bool) -> Optional[List[Str]]
+    """
+    Obtains all valid case-insensitive service-type names from a filtered comma-separated list.
+
+    :param service_query: query string or service type(s) comma-separated to parse.
+    :param default_services: specify if the complete list of known service-types must be returned if no query to parse.
+    :returns: parsed service-types if query was provided, or None by default, or all known service-types if requested.
+    """
+    if service_query:
+        service_types = [
+            svc_type.lower() for svc_type in service_query.split(",")
+            if svc_type.strip() and svc_type.lower() in SERVICE_TYPE_DICT
+        ]
+        return service_types
+    if default_services:
+        return list(SERVICE_TYPE_DICT)
+    return None

magpie/api/management/service/service_views.py

@@ -67,18 +67,24 @@ def get_services_runner(request):
     and query parameters.
     """
     service_type_filter = request.matchdict.get("service_type")  # no check because None/empty is for 'all services'
-    services_as_list = asbool(ar.get_query_param(request, "flatten", False))
+    services_as_list = asbool(ar.get_query_param(request, ["flatten", "list"], False))
+    known_service_types = list(SERVICE_TYPE_DICT)
 
+    # using '/services?type={}' query (allow many, no error if unknown)
     if not service_type_filter:
-        service_types = SERVICE_TYPE_DICT.keys()
+        service_types = ar.get_query_param(request, ["type", "types"], default="")
+        service_types = su.filter_service_types(service_types, default_services=True)
+    # using '/services/types/{}' path (error if unknown)
     else:
-        ax.verify_param(service_type_filter, param_compare=SERVICE_TYPE_DICT.keys(), is_in=True,
+        ax.verify_param(service_type_filter, param_compare=known_service_types, is_in=True,
                         http_error=HTTPBadRequest, msg_on_fail=s.Services_GET_BadRequestResponseSchema.description,
                         content={"service_type": str(service_type_filter)}, content_type=CONTENT_TYPE_JSON)
         service_types = [service_type_filter]
 
     svc_content = [] if services_as_list else {}  # type: Union[List[JSON], JSON]
     for service_type in service_types:
+        if service_type not in known_service_types:
+            continue
         services = su.get_services_by_type(service_type, db_session=request.db)
         if not services_as_list:
             svc_content[service_type] = {}
@@ -89,6 +95,12 @@ def get_services_runner(request):
             else:
                 svc_content[service_type][service.resource_name] = svc_fmt
 
+    # sort result
+    if services_as_list:
+        svc_content = list(sorted(svc_content, key=lambda svc: svc["service_name"]))
+    else:
+        svc_content = {svc_type: dict(sorted(svc_items.items())) for svc_type, svc_items in sorted(svc_content.items())}
+
     return ax.valid_http(http_success=HTTPOk, content={"services": svc_content},
                          detail=s.Services_GET_OkResponseSchema.description)