From 56022a5ab78c75193e191b791422f2eff5a1b841 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Mon, 22 Oct 2018 19:34:18 -0400 Subject: [PATCH 01/13] target ziggurat 0.8.1 + fix broken tests --- magpie/api/api_rest_schemas.py | 58 ++++++++++----- magpie/api/login/login.py | 4 +- magpie/api/management/group/group_utils.py | 7 +- magpie/api/management/group/group_views.py | 2 +- .../management/resource/resource_formats.py | 5 +- .../api/management/resource/resource_utils.py | 4 +- .../api/management/service/service_formats.py | 1 - magpie/api/management/user/user_utils.py | 10 +-- magpie/api/management/user/user_views.py | 16 ++-- magpie/db.py | 1 + magpie/helpers/register_default_users.py | 8 +- magpie/models.py | 3 +- magpie/register.py | 5 +- requirements.txt | 5 +- tests/interfaces.py | 73 ++++++++++--------- tests/test_magpie_api.py | 10 ++- tests/utils.py | 20 ++--- 17 files changed, 130 insertions(+), 102 deletions(-) diff --git a/magpie/api/api_rest_schemas.py b/magpie/api/api_rest_schemas.py index 08cac2190..3282d7331 100644 --- a/magpie/api/api_rest_schemas.py +++ b/magpie/api/api_rest_schemas.py @@ -1381,19 +1381,39 @@ class UserGroups_POST_CreatedResponseSchema(colander.MappingSchema): class UserGroups_POST_GroupNotFoundResponseSchema(colander.MappingSchema): description = "Can't find the group to assign to." header = HeaderResponseSchema() - body = BaseResponseBodySchema(code=HTTPNotFound.code, description=description) + body = ErrorResponseBodySchema(code=HTTPNotFound.code, description=description) class UserGroups_POST_ForbiddenResponseSchema(colander.MappingSchema): description = "Group query by name refused by db." header = HeaderResponseSchema() - body = BaseResponseBodySchema(code=HTTPForbidden.code, description=description) + body = ErrorResponseBodySchema(code=HTTPForbidden.code, description=description) + + +class UserGroups_POST_RelationshipForbiddenResponseSchema(colander.MappingSchema): + description = "User-Group relationship creation refused by db." + header = HeaderResponseSchema() + body = ErrorResponseBodySchema(code=HTTPForbidden.code, description=description) + + +class UserGroups_POST_ConflictResponseBodySchema(ErrorResponseBodySchema): + param = ErrorVerifyParamBodySchema() + user_name = colander.SchemaNode( + colander.String(), + description="Name of the user in the user-group relationship", + example="toto", + ) + group_name = colander.SchemaNode( + colander.String(), + description="Name of the group in the user-group relationship", + example="users", + ) class UserGroups_POST_ConflictResponseSchema(colander.MappingSchema): description = "User already belongs to this group." header = HeaderResponseSchema() - body = BaseResponseBodySchema(code=HTTPConflict.code, description=description) + body = UserGroups_POST_ConflictResponseBodySchema(code=HTTPConflict.code, description=description) class UserGroup_DELETE_RequestSchema(colander.MappingSchema): @@ -1410,7 +1430,7 @@ class UserGroup_DELETE_OkResponseSchema(colander.MappingSchema): class UserGroup_DELETE_NotFoundResponseSchema(colander.MappingSchema): description = "Invalid user-group combination for delete." header = HeaderResponseSchema() - body = BaseResponseBodySchema(code=HTTPNotFound.code, description=description) + body = ErrorResponseBodySchema(code=HTTPNotFound.code, description=description) class UserResources_GET_ResponseBodySchema(BaseResponseBodySchema): @@ -1479,7 +1499,7 @@ class UserResourcePermissions_GET_NotAcceptableResourceTypeResponseSchema(coland class UserResourcePermissions_GET_NotFoundResponseSchema(colander.MappingSchema): description = "Specified user not found to obtain resource permissions." header = HeaderResponseSchema() - body = BaseResponseBodySchema(code=HTTPNotFound.code, description=description) + body = ErrorResponseBodySchema(code=HTTPNotFound.code, description=description) class UserResourcePermissions_POST_RequestBodySchema(colander.MappingSchema): @@ -1645,31 +1665,31 @@ class UserServicePermissions_GET_OkResponseSchema(colander.MappingSchema): class UserServicePermissions_GET_NotFoundResponseSchema(colander.MappingSchema): description = "Could not find permissions using specified `service_name` and `user_name`." header = HeaderResponseSchema() - body = BaseResponseBodySchema(code=HTTPNotFound.code, description=description) + body = ErrorResponseBodySchema(code=HTTPNotFound.code, description=description) class Group_MatchDictCheck_ForbiddenResponseSchema(colander.MappingSchema): description = "Group query by name refused by db." header = HeaderResponseSchema() - body = BaseResponseBodySchema(code=HTTPForbidden.code, description=description) + body = ErrorResponseBodySchema(code=HTTPForbidden.code, description=description) class Group_MatchDictCheck_NotFoundResponseSchema(colander.MappingSchema): description = "Group name not found in db." header = HeaderResponseSchema() - body = BaseResponseBodySchema(code=HTTPNotFound.code, description=description) + body = ErrorResponseBodySchema(code=HTTPNotFound.code, description=description) class Groups_CheckInfo_NotFoundResponseSchema(colander.MappingSchema): description = "User name not found in db." header = HeaderResponseSchema() - body = BaseResponseBodySchema(code=HTTPNotFound.code, description=description) + body = ErrorResponseBodySchema(code=HTTPNotFound.code, description=description) class Groups_CheckInfo_ForbiddenResponseSchema(colander.MappingSchema): description = "Failed to obtain groups of user." header = HeaderResponseSchema() - body = BaseResponseBodySchema(code=HTTPForbidden.code, description=description) + body = ErrorResponseBodySchema(code=HTTPForbidden.code, description=description) class Groups_GET_ResponseBodySchema(BaseResponseBodySchema): @@ -1685,7 +1705,7 @@ class Groups_GET_OkResponseSchema(colander.MappingSchema): class Groups_GET_ForbiddenResponseSchema(colander.MappingSchema): description = "Obtain group names refused by db." header = HeaderResponseSchema() - body = BaseResponseBodySchema(code=HTTPForbidden.code, description=description) + body = ErrorResponseBodySchema(code=HTTPForbidden.code, description=description) class Groups_POST_RequestSchema(colander.MappingSchema): @@ -1717,7 +1737,7 @@ class Groups_POST_ForbiddenAddResponseSchema(colander.MappingSchema): class Groups_POST_ConflictResponseSchema(colander.MappingSchema): description = "Group name matches an already existing group name." header = HeaderResponseSchema() - body = BaseResponseBodySchema(code=HTTPConflict.code, description=description) + body = ErrorResponseBodySchema(code=HTTPConflict.code, description=description) class Group_GET_ResponseBodySchema(BaseResponseBodySchema): @@ -1733,7 +1753,7 @@ class Group_GET_OkResponseSchema(colander.MappingSchema): class Group_GET_NotFoundResponseSchema(colander.MappingSchema): description = "Group name was not found." header = HeaderResponseSchema() - body = BaseResponseBodySchema(code=HTTPNotFound.code, description=description) + body = ErrorResponseBodySchema(code=HTTPNotFound.code, description=description) class Group_PUT_RequestSchema(colander.MappingSchema): @@ -1749,26 +1769,26 @@ class Group_PUT_OkResponseSchema(colander.MappingSchema): class Group_PUT_Name_NotAcceptableResponseSchema(colander.MappingSchema): description = "Invalid `group_name` value specified." header = HeaderResponseSchema() - body = BaseResponseBodySchema(code=HTTPNotAcceptable.code, description=description) + body = ErrorResponseBodySchema(code=HTTPNotAcceptable.code, description=description) class Group_PUT_Size_NotAcceptableResponseSchema(colander.MappingSchema): description = "Invalid `group_name` length specified (>{length} characters)." \ .format(length=MAGPIE_USER_NAME_MAX_LENGTH) header = HeaderResponseSchema() - body = BaseResponseBodySchema(code=HTTPNotAcceptable.code, description=description) + body = ErrorResponseBodySchema(code=HTTPNotAcceptable.code, description=description) class Group_PUT_Same_NotAcceptableResponseSchema(colander.MappingSchema): description = "Invalid `group_name` must be different than current name." header = HeaderResponseSchema() - body = BaseResponseBodySchema(code=HTTPNotAcceptable.code, description=description) + body = ErrorResponseBodySchema(code=HTTPNotAcceptable.code, description=description) class Group_PUT_ConflictResponseSchema(colander.MappingSchema): description = "Group name already exists." header = HeaderResponseSchema() - body = BaseResponseBodySchema(code=HTTPConflict.code, description=description) + body = ErrorResponseBodySchema(code=HTTPConflict.code, description=description) class Group_DELETE_RequestSchema(colander.MappingSchema): @@ -1785,7 +1805,7 @@ class Group_DELETE_OkResponseSchema(colander.MappingSchema): class Group_DELETE_ForbiddenResponseSchema(colander.MappingSchema): description = "Delete group forbidden by db." header = HeaderResponseSchema() - body = BaseResponseBodySchema(code=HTTPOk.code, description=description) + body = ErrorResponseBodySchema(code=HTTPForbidden.code, description=description) class GroupUsers_GET_OkResponseSchema(colander.MappingSchema): @@ -1797,7 +1817,7 @@ class GroupUsers_GET_OkResponseSchema(colander.MappingSchema): class GroupUsers_GET_ForbiddenResponseSchema(colander.MappingSchema): description = "Failed to obtain group user names from db." header = HeaderResponseSchema() - body = BaseResponseBodySchema(code=HTTPForbidden.code, description=description) + body = ErrorResponseBodySchema(code=HTTPForbidden.code, description=description) class GroupServices_GET_ResponseBodySchema(BaseResponseBodySchema): diff --git a/magpie/api/login/login.py b/magpie/api/login/login.py index ef8ac4a00..173fa4128 100644 --- a/magpie/api/login/login.py +++ b/magpie/api/login/login.py @@ -52,9 +52,9 @@ def sign_in(request): verify_provider(provider_name) if provider_name in MAGPIE_INTERNAL_PROVIDERS.keys(): - signin_internal_url = '{host}{path}'.format(host=request.application_url, path='/signin_internal') + signin_internal_url = request.route_url('ziggurat.routes.sign_in') signin_internal_data = {u'user_name': user_name, u'password': password, u'provider_name': provider_name} - signin_response = requests.post(signin_internal_url, data=signin_internal_data, allow_redirects=True) + signin_response = requests.post(signin_internal_url, data=signin_internal_data, allow_redirects=True, verify=False) if signin_response.status_code == HTTPOk.code: pyramid_response = Response(body=signin_response.content, headers=signin_response.headers) diff --git a/magpie/api/management/group/group_utils.py b/magpie/api/management/group/group_utils.py index 8a462c684..560422b68 100644 --- a/magpie/api/management/group/group_utils.py +++ b/magpie/api/management/group/group_utils.py @@ -61,7 +61,8 @@ def create_group_resource_permission(permission_name, resource, group, db_sessio def get_group_resources_permissions_dict(group, db_session, resource_ids=None, resource_types=None): def get_grp_res_perm(grp, db, res_ids, res_types): - res_perms_tup = grp.resources_with_possible_perms(resource_ids=res_ids, resource_types=res_types, db_session=db) + res_perms_tup = GroupService.resources_with_possible_perms( + grp, resource_ids=res_ids, resource_types=res_types, db_session=db) res_perms_dict = {} for res_perm in res_perms_tup: if res_perm.resource.resource_id not in res_perms_dict: @@ -115,7 +116,7 @@ def delete_group_resource_permission(permission_name, resource, group, db_sessio def get_group_services(resources_permissions_dict, db_session): grp_svc_dict = {} for res_id, perms in resources_permissions_dict.items(): - svc = models.Service.by_resource_id(resource_id=res_id, db_session=db_session) + svc = ResourceService.by_resource_id(resource_id=res_id, db_session=db_session) svc_type = str(svc.type) svc_name = str(svc.resource_name) if svc_type not in grp_svc_dict: @@ -146,7 +147,7 @@ def get_grp_svc_perms(grp, db_ses, res_ids): db_session=db_ses, resource_ids=res_ids) grp_svc_perms = [] for res_id, res_perm in res_perms.items(): - svc = models.Service.by_resource_id(res_id, db_session=db_ses) + svc = ResourceService.by_resource_id(res_id, db_session=db_ses) grp_svc_perms.append((svc, res_perm)) return grp_svc_perms diff --git a/magpie/api/management/group/group_views.py b/magpie/api/management/group/group_views.py index 979234ba6..dabf308cb 100644 --- a/magpie/api/management/group/group_views.py +++ b/magpie/api/management/group/group_views.py @@ -54,7 +54,7 @@ def edit_group(request): msgOnFail=Group_PUT_Size_NotAcceptableResponseSchema.description) verify_param(new_group_name, notEqual=True, httpError=HTTPNotAcceptable, paramCompare=group.group_name, msgOnFail=Group_PUT_Same_NotAcceptableResponseSchema.description) - verify_param(models.Group.by_group_name(new_group_name, db_session=request.db), isNone=True, httpError=HTTPConflict, + verify_param(GroupService.by_group_name(new_group_name, db_session=request.db), isNone=True, httpError=HTTPConflict, msgOnFail=Group_PUT_ConflictResponseSchema.description) group.group_name = new_group_name return valid_http(httpSuccess=HTTPOk, detail=Group_PUT_OkResponseSchema.description) diff --git a/magpie/api/management/resource/resource_formats.py b/magpie/api/management/resource/resource_formats.py index 013cabeae..6a2ab252a 100644 --- a/magpie/api/management/resource/resource_formats.py +++ b/magpie/api/management/resource/resource_formats.py @@ -1,5 +1,6 @@ from magpie.definitions.pyramid_definitions import * -from magpie.models import resource_tree_service, Service +from magpie.definitions.ziggurat_definitions import ResourceService +from magpie.models import resource_tree_service from magpie.services import service_type_dict from magpie.api.api_except import evaluate_call @@ -70,7 +71,7 @@ def format_resource_tree(children, db_session, resources_perms_dict=None, intern else: service_id = resource.root_service_id if service_id not in internal_svc_res_perm_dict: - service = Service.by_resource_id(service_id, db_session=db_session) + service = ResourceService.by_resource_id(service_id, db_session=db_session) internal_svc_res_perm_dict[service_id] = service_type_dict[service.type].resource_types_permissions perms = internal_svc_res_perm_dict[service_id][resource.resource_type] diff --git a/magpie/api/management/resource/resource_utils.py b/magpie/api/management/resource/resource_utils.py index 7486341a8..ff60025fe 100644 --- a/magpie/api/management/resource/resource_utils.py +++ b/magpie/api/management/resource/resource_utils.py @@ -2,9 +2,7 @@ from magpie.models import resource_factory, resource_type_dict, resource_tree_service from magpie.services import service_type_dict from magpie.register import sync_services_phoenix -from magpie.definitions.pyramid_definitions import * from magpie.definitions.ziggurat_definitions import * -from magpie.api.api_rest_schemas import * from magpie.api.api_requests import * from magpie.api.api_except import verify_param, evaluate_call, raise_http, valid_http from magpie.api.management.resource.resource_formats import format_resource @@ -98,7 +96,7 @@ def get_resource_permissions(resource, db_session): return service_type_dict[service.type].permission_names # otherwise obtain root level service to infer sub-resource permissions - service = models.Service.by_resource_id(resource.root_service_id, db_session=db_session) + service = ResourceService.by_resource_id(resource.root_service_id, db_session=db_session) verify_param(service.resource_type, isEqual=True, httpError=HTTPNotAcceptable, paramName=u'resource_type', paramCompare=u'service', msgOnFail=UserResourcePermissions_GET_NotAcceptableRootServiceResponseSchema.description) diff --git a/magpie/api/management/service/service_formats.py b/magpie/api/management/service/service_formats.py index f67a7cb44..4b5e1f4a7 100644 --- a/magpie/api/management/service/service_formats.py +++ b/magpie/api/management/service/service_formats.py @@ -10,7 +10,6 @@ def format_service(service, permissions=None, show_private_url=False): def fmt_svc(svc, perms): svc_info = { u'public_url': str(get_twitcher_protected_service_url(svc.resource_name)), - u'service_url': str(svc.url), u'service_name': str(svc.resource_name), u'service_type': str(svc.type), u'service_sync_type': svc.sync_type, diff --git a/magpie/api/management/user/user_utils.py b/magpie/api/management/user/user_utils.py index 4f2514269..2d1192404 100644 --- a/magpie/api/management/user/user_utils.py +++ b/magpie/api/management/user/user_utils.py @@ -109,9 +109,9 @@ def get_user_services(user, db_session, cascade_resources=False, services = {} for resource_id, perms in res_perm_dict.items(): - svc = models.Service.by_resource_id(resource_id=resource_id, db_session=db_session) + svc = ResourceService.by_resource_id(resource_id=resource_id, db_session=db_session) if svc.resource_type != 'service' and cascade_resources: - svc = models.Service.by_resource_id(resource_id=svc.root_service_id, db_session=db_session) + svc = ResourceService.by_resource_id(resource_id=svc.root_service_id, db_session=db_session) perms = service_type_dict[svc.type].permission_names if svc.type not in services: services[svc.type] = {} @@ -132,7 +132,7 @@ def get_user_service_permissions(user, service, db_session, inherit_groups_permi if service.owner_user_id == user.id: permission_names = service_type_dict[service.type].permission_names else: - svc_perm_tuple_list = service.perms_for_user(user, db_session=db_session) + svc_perm_tuple_list = ResourceService.perms_for_user(service, user, db_session=db_session) if not inherit_groups_permissions: svc_perm_tuple_list = filter_user_permission(svc_perm_tuple_list, user) permission_names = [permission.perm_name for permission in svc_perm_tuple_list] @@ -155,8 +155,8 @@ def get_user_resources_permissions_dict(user, db_session, resource_types=None, """ verify_param(user, notNone=True, httpError=HTTPNotFound, msgOnFail=UserResourcePermissions_GET_NotFoundResponseSchema.description) - res_perm_tuple_list = user.resources_with_possible_perms(resource_ids=resource_ids, - resource_types=resource_types, db_session=db_session) + res_perm_tuple_list = UserService.resources_with_possible_perms( + user, resource_ids=resource_ids,resource_types=resource_types, db_session=db_session) if not inherit_groups_permissions: res_perm_tuple_list = filter_user_permission(res_perm_tuple_list, user) resources_permissions_dict = {} diff --git a/magpie/api/management/user/user_views.py b/magpie/api/management/user/user_views.py index 4b8ba3074..a3ee584c7 100644 --- a/magpie/api/management/user/user_views.py +++ b/magpie/api/management/user/user_views.py @@ -48,7 +48,7 @@ def update_user_view(request): check_user_info(new_user_name, new_email, new_password, group_name=new_user_name) if user.user_name != new_user_name: - evaluate_call(lambda: models.User.by_user_name(new_user_name, db_session=request.db), + evaluate_call(lambda: UserService.by_user_name(new_user_name, db_session=request.db), fallback=lambda: request.db.rollback(), httpError=HTTPConflict, msgOnFail=User_PUT_ConflictResponseSchema.description) user.user_name = new_user_name @@ -102,7 +102,6 @@ def get_user_groups(request): @view_config(route_name=UserGroupsAPI.name, request_method='POST') def assign_user_group(request): """Assign a user to a group.""" - db = request.db user = get_user_matchdict_checked_or_logged(request) group_name = get_value_multiformat_post_checked(request, 'group_name') @@ -111,13 +110,16 @@ def assign_user_group(request): httpError=HTTPForbidden, msgOnFail=UserGroups_POST_ForbiddenResponseSchema.description) verify_param(group, notNone=True, httpError=HTTPNotFound, msgOnFail=UserGroups_POST_GroupNotFoundResponseSchema.description) + verify_param(user.id, paramCompare=[usr.id for usr in group.users], notIn=True, httpError=HTTPConflict, + content={u'user_name': user.user_name, u'group_name': group.group_name}, + msgOnFail=UserGroups_POST_ConflictResponseSchema.description) - new_user_group = models.UserGroup(group_id=group.id, user_id=user.id) - - evaluate_call(lambda: db.add(new_user_group), fallback=lambda: db.rollback(), - httpError=HTTPConflict, msgOnFail=UserGroups_POST_ConflictResponseSchema.description, + evaluate_call(lambda: request.db.add(models.UserGroup(group_id=group.id, user_id=user.id)), + fallback=lambda: request.db.rollback(), + httpError=HTTPForbidden, msgOnFail=UserGroups_POST_RelationshipForbiddenResponseSchema.description, content={u'user_name': user.user_name, u'group_name': group.group_name}) - return valid_http(httpSuccess=HTTPCreated, detail=UserGroups_POST_CreatedResponseSchema.description) + return valid_http(httpSuccess=HTTPCreated, detail=UserGroups_POST_CreatedResponseSchema.description, + content={u'user_name': user.user_name, u'group_name': group.group_name}) @UserGroupAPI.delete(schema=UserGroup_DELETE_RequestSchema(), tags=[UsersTag], diff --git a/magpie/db.py b/magpie/db.py index 5ece78e89..d8bf4ed1e 100644 --- a/magpie/db.py +++ b/magpie/db.py @@ -80,6 +80,7 @@ def get_db_session_from_config_ini(config_ini_path, ini_main_section_name='app:m def get_settings_from_config_ini(config_ini_path, ini_main_section_name='app:magpie_app'): parser = configparser.ConfigParser() + parser.optionxform = lambda option: option # preserve case of config (ziggurat requires it for 'User' model) parser.read([config_ini_path]) settings = dict(parser.items(ini_main_section_name)) return settings diff --git a/magpie/helpers/register_default_users.py b/magpie/helpers/register_default_users.py index ff5b4592a..627db30f2 100644 --- a/magpie/helpers/register_default_users.py +++ b/magpie/helpers/register_default_users.py @@ -12,17 +12,17 @@ def register_user_with_group(user_name, group_name, email, password, db_session) if not GroupService.by_group_name(group_name, db_session=db_session): new_group = models.Group(group_name=group_name) db_session.add(new_group) - registered_group = models.Group.by_group_name(group_name=group_name, db_session=db_session) + registered_group = GroupService.by_group_name(group_name=group_name, db_session=db_session) registered_user = UserService.by_user_name(user_name, db_session=db_session) if not registered_user: new_user = models.User(user_name=user_name, email=email) - new_user.set_password(password) - new_user.regenerate_security_code() + UserService.set_password(new_user, password) + UserService.regenerate_security_code(new_user) db_session.add(new_user) registered_user = UserService.by_user_name(user_name, db_session=db_session) else: - print_log(user_name+' already exist', level=logging.DEBUG) + print_log('User `{}` already exist'.format(user_name), level=logging.DEBUG) try: # ensure the reference between user/group exists (user joined the group) diff --git a/magpie/models.py b/magpie/models.py index 3e9347439..37c63c901 100644 --- a/magpie/models.py +++ b/magpie/models.py @@ -85,7 +85,8 @@ class RootFactory(object): def __init__(self, request): self.__acl__ = [] if request.user: - for outcome, perm_user, perm_name in permission_to_pyramid_acls(request.user.permissions): + permissions = UserService.permissions(request.user, request.db) + for outcome, perm_user, perm_name in permission_to_pyramid_acls(permissions): self.__acl__.append((outcome, perm_user, perm_name)) diff --git a/magpie/register.py b/magpie/register.py index 05268da3f..b342ad332 100644 --- a/magpie/register.py +++ b/magpie/register.py @@ -2,6 +2,7 @@ from magpie.common import make_dirs, print_log, raise_log, bool2str from magpie.constants import get_constant from magpie import models +from magpie.definitions.ziggurat_definitions import UserService, UserResourcePermissionService import os import time import yaml @@ -355,7 +356,7 @@ def magpie_register_services_with_db_session(services_dict, db_session, push_to_ force_update=False, update_getcapabilities_permissions=False): existing_services_names = [n[0] for n in db_session.query(models.Service.resource_name)] magpie_anonymous_user = get_constant('MAGPIE_ANONYMOUS_USER') - anonymous_user = models.User.by_user_name(magpie_anonymous_user, db_session=db_session) + anonymous_user = UserService.by_user_name(magpie_anonymous_user, db_session=db_session) for svc_name, svc_values in services_dict.items(): svc_new_url = os.path.expandvars(svc_values['url']) @@ -386,7 +387,7 @@ def magpie_register_services_with_db_session(services_dict, db_session, push_to_ print_log("Cannot update 'getcapabilities' permission of non existing anonymous user", level=logging.WARN) elif update_getcapabilities_permissions and 'getcapabilities' in service_type_dict[svc_type].permission_names: svc = db_session.query(models.Service.resource_id).filter_by(resource_name=svc_name).first() - svc_perm_getcapabilities = models.UserResourcePermissionService.by_resource_user_and_perm( + svc_perm_getcapabilities = UserResourcePermissionService.by_resource_user_and_perm( user_id=anonymous_user.id, perm_name='getcapabilities', resource_id=svc.resource_id, db_session=db_session ) diff --git a/requirements.txt b/requirements.txt index a05e86668..6cab63a28 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,7 +9,7 @@ Sphinx==1.3.1 #cryptography==1.9 PyYAML>=3.11 pyramid==1.8.3 -ziggurat-foundations==0.7.1 +ziggurat-foundations==0.8.1 pyramid_tm==2.2 pyramid_chameleon==0.3 pyramid_mako>=1.0.2 @@ -20,7 +20,7 @@ psycopg2>=2.7.1 lxml>=3.7 bcrypt==3.1.3 futures==3.1.1 -zope.sqlalchemy +zope.sqlalchemy==1.0 gunicorn==19.8.1 alembic==0.9.6 paste @@ -32,3 +32,4 @@ colander threddsclient==0.3.4 humanize requests_file +webtest diff --git a/tests/interfaces.py b/tests/interfaces.py index 6462b21ba..3577737c7 100644 --- a/tests/interfaces.py +++ b/tests/interfaces.py @@ -33,7 +33,7 @@ def tearDownClass(cls): @unittest.skipUnless(runner.MAGPIE_TEST_LOGIN, reason=runner.MAGPIE_TEST_DISABLED_MESSAGE('login')) def test_GetSession_Anonymous(self): resp = utils.test_request(self.url, 'GET', '/session', headers=self.json_headers) - json_body = utils.check_response_basic_info(resp, 200) + json_body = utils.check_response_basic_info(resp, 200, expected_method='GET') utils.check_val_equal(json_body['authenticated'], False) if LooseVersion(self.version) >= LooseVersion('0.6.3'): utils.check_val_not_in('user', json_body) @@ -44,7 +44,7 @@ def test_GetSession_Anonymous(self): def test_GetVersion(self): resp = utils.test_request(self.url, 'GET', '/version', headers=self.json_headers) - json_body = utils.check_response_basic_info(resp, 200) + json_body = utils.check_response_basic_info(resp, 200, expected_method='GET') utils.check_val_is_in('db_version', json_body) utils.check_val_is_in('version', json_body) # server not necessarily at latest version, ensure at least format @@ -58,7 +58,7 @@ def test_GetVersion(self): def test_GetCurrentUser(self): logged_user = get_constant('MAGPIE_LOGGED_USER') resp = utils.test_request(self.url, 'GET', '/users/{}'.format(logged_user), headers=self.json_headers) - json_body = utils.check_response_basic_info(resp, 200) + json_body = utils.check_response_basic_info(resp, 200, expected_method='GET') if LooseVersion(self.version) >= LooseVersion('0.6.3'): utils.check_val_equal(json_body['user']['user_name'], self.usr) else: @@ -125,11 +125,10 @@ def get_test_values(cls): cls.test_service_name = u'project-api' cls.test_service_type = cls.test_services_info[cls.test_service_name]['type'] - utils.check_val_is_in(cls.test_service_type, cls.test_services_info) - resp = utils.test_request(cls.url, 'GET', '/services/project-api', + resp = utils.test_request(cls.url, 'GET', '/services/{}'.format(cls.test_service_name), headers=cls.json_headers, cookies=cls.cookies) - json_body = utils.check_response_basic_info(resp, 200) + json_body = utils.check_response_basic_info(resp, 200, expected_method='GET') cls.test_service_resource_id = json_body[cls.test_service_name]['resource_id'] cls.test_resource_name = u'magpie-unittest-resource' @@ -167,7 +166,7 @@ def test_GetAPI(self): @unittest.skipUnless(runner.MAGPIE_TEST_LOGIN, reason=runner.MAGPIE_TEST_DISABLED_MESSAGE('login')) def test_GetSession_Administrator(self): resp = utils.test_request(self.url, 'GET', '/session', headers=self.json_headers, cookies=self.cookies) - json_body = utils.check_response_basic_info(resp, 200) + json_body = utils.check_response_basic_info(resp, 200, expected_method='GET') utils.check_val_equal(json_body['authenticated'], True) if LooseVersion(self.version) >= LooseVersion('0.6.3'): utils.check_val_is_in('user', json_body) @@ -185,7 +184,7 @@ def test_GetSession_Administrator(self): @unittest.skipUnless(runner.MAGPIE_TEST_USERS, reason=runner.MAGPIE_TEST_DISABLED_MESSAGE('users')) def test_GetUsers(self): resp = utils.test_request(self.url, 'GET', '/users', headers=self.json_headers, cookies=self.cookies) - json_body = utils.check_response_basic_info(resp, 200) + json_body = utils.check_response_basic_info(resp, 200, expected_method='GET') utils.check_val_is_in('user_names', json_body) utils.check_val_type(json_body['user_names'], list) utils.check_val_equal(len(json_body['user_names']) > 1, True) # should have more than only 'anonymous' @@ -198,7 +197,7 @@ def test_GetUsers(self): @unittest.skipUnless(runner.MAGPIE_TEST_DEFAULTS, reason=runner.MAGPIE_TEST_DISABLED_MESSAGE('defaults')) def test_ValidateDefaultUsers(self): resp = utils.test_request(self.url, 'GET', '/users', headers=self.json_headers, cookies=self.cookies) - json_body = utils.check_response_basic_info(resp, 200) + json_body = utils.check_response_basic_info(resp, 200, expected_method='GET') users = json_body['user_names'] utils.check_val_is_in(get_constant('MAGPIE_ANONYMOUS_USER'), users) utils.check_val_is_in(get_constant('MAGPIE_ADMIN_USER'), users) @@ -207,7 +206,7 @@ def test_ValidateDefaultUsers(self): def check_GetUserResourcesPermissions(cls, user_name): route = '/users/{usr}/resources/{res_id}/permissions'.format(res_id=cls.test_service_resource_id, usr=user_name) resp = utils.test_request(cls.url, 'GET', route, headers=cls.json_headers, cookies=cls.cookies) - json_body = utils.check_response_basic_info(resp, 200) + json_body = utils.check_response_basic_info(resp, 200, expected_method='GET') utils.check_val_is_in('permission_names', json_body) utils.check_val_type(json_body['permission_names'], list) @@ -226,7 +225,7 @@ def test_GetUserResourcesPermissions(self): def test_GetCurrentUserGroups(self): resp = utils.test_request(self.url, 'GET', '/users/current/groups', headers=self.json_headers, cookies=self.cookies) - json_body = utils.check_response_basic_info(resp, 200) + json_body = utils.check_response_basic_info(resp, 200, expected_method='GET') utils.check_val_is_in('group_names', json_body) utils.check_val_type(json_body['group_names'], list) utils.check_val_is_in(get_constant('MAGPIE_ADMIN_GROUP'), json_body['group_names']) @@ -236,7 +235,7 @@ def test_GetCurrentUserGroups(self): def test_GetUserInheritedResources(self): route = '/users/{usr}/inherited_resources'.format(usr=self.usr) resp = utils.test_request(self.url, 'GET', route, headers=self.json_headers, cookies=self.cookies) - json_body = utils.check_response_basic_info(resp, 200) + json_body = utils.check_response_basic_info(resp, 200, expected_method='GET') utils.check_val_is_in('resources', json_body) utils.check_val_type(json_body['resources'], dict) service_types = utils.get_service_types_for_version(self.version) @@ -268,13 +267,17 @@ def test_GetUserInheritedResources(self): def test_GetUserServices(self): route = '/users/{usr}/services'.format(usr=self.usr) resp = utils.test_request(self.url, 'GET', route, headers=self.json_headers, cookies=self.cookies) - json_body = utils.check_response_basic_info(resp, 200) + json_body = utils.check_response_basic_info(resp, 200, expected_method='GET') utils.check_val_is_in('services', json_body) services = json_body['services'] utils.check_val_type(services, dict) service_types = utils.get_service_types_for_version(self.version) - utils.check_all_equal(services.keys(), service_types, any_order=True) + # as of version '0.7.0', visible services depend on the connected user permissions, + # so all services types not necessarily returned in the response + if LooseVersion(self.version) < LooseVersion('0.7.0'): + utils.check_all_equal(services.keys(), service_types, any_order=True) for svc_type in services: + utils.check_val_is_in(svc_type, service_types) # one of valid service types for svc in services[svc_type]: svc_dict = services[svc_type][svc] utils.check_val_type(svc_dict, dict) @@ -304,7 +307,7 @@ def test_GetUserServices(self): def test_GetUserServiceResources(self): route = '/users/{usr}/services/{svc}/resources'.format(usr=self.usr, svc=self.test_service_name) resp = utils.test_request(self.url, 'GET', route, headers=self.json_headers, cookies=self.cookies) - json_body = utils.check_response_basic_info(resp, 200) + json_body = utils.check_response_basic_info(resp, 200, expected_method='GET') utils.check_val_is_in('service', json_body) svc_dict = json_body['service'] utils.check_val_type(svc_dict, dict) @@ -352,7 +355,7 @@ def test_GetUser_existing(self): route = '/users/{usr}'.format(usr=self.test_user_name) resp = utils.test_request(self.url, 'GET', route, headers=self.json_headers, cookies=self.cookies) - json_body = utils.check_response_basic_info(resp, 200) + json_body = utils.check_response_basic_info(resp, 200, expected_method='GET') if LooseVersion(self.version) >= LooseVersion('0.6.3'): utils.check_val_is_in('user', json_body) utils.check_val_is_in('user_name', json_body['user']) @@ -376,7 +379,7 @@ def test_GetUser_missing(self): route = '/users/{usr}'.format(usr=self.test_user_name) resp = utils.test_request(self.url, 'GET', route, headers=self.json_headers, cookies=self.cookies, expect_errors=True) - utils.check_response_basic_info(resp, 404) + utils.check_response_basic_info(resp, 404, expected_method='GET') @pytest.mark.users @unittest.skipUnless(runner.MAGPIE_TEST_USERS, reason=runner.MAGPIE_TEST_DISABLED_MESSAGE('users')) @@ -385,7 +388,7 @@ def test_GetCurrentUser(self): logged_user = get_constant('MAGPIE_LOGGED_USER') resp = utils.test_request(self.url, 'GET', '/users/{}'.format(logged_user), headers=self.json_headers, cookies=self.cookies) - json_body = utils.check_response_basic_info(resp, 200) + json_body = utils.check_response_basic_info(resp, 200, expected_method='GET') if LooseVersion(self.version) >= LooseVersion('0.6.3'): utils.check_val_equal(json_body['user']['user_name'], self.usr) else: @@ -397,7 +400,7 @@ def test_GetCurrentUser(self): @unittest.skipUnless(runner.MAGPIE_TEST_DEFAULTS, reason=runner.MAGPIE_TEST_DISABLED_MESSAGE('defaults')) def test_ValidateDefaultGroups(self): resp = utils.test_request(self.url, 'GET', '/groups', headers=self.json_headers, cookies=self.cookies) - json_body = utils.check_response_basic_info(resp, 200) + json_body = utils.check_response_basic_info(resp, 200, expected_method='GET') groups = json_body['group_names'] utils.check_val_is_in(get_constant('MAGPIE_ANONYMOUS_GROUP'), groups) utils.check_val_is_in(get_constant('MAGPIE_USERS_GROUP'), groups) @@ -409,7 +412,7 @@ def test_PostUserGroup_assign(self): route = '/users/{usr}/groups'.format(usr=get_constant('MAGPIE_ADMIN_USER')) data = {'group_name': get_constant('MAGPIE_ANONYMOUS_GROUP')} resp = utils.test_request(self.url, 'POST', route, headers=self.json_headers, cookies=self.cookies, data=data) - utils.check_response_basic_info(resp, 201) + utils.check_response_basic_info(resp, 201, expected_method='POST') @pytest.mark.groups @unittest.skipUnless(runner.MAGPIE_TEST_GROUPS, reason=runner.MAGPIE_TEST_DISABLED_MESSAGE('groups')) @@ -417,7 +420,7 @@ def test_PostUserGroup_not_found(self): route = '/users/{usr}/groups'.format(usr=get_constant('MAGPIE_ADMIN_USER')) data = {'group_name': 'not_found'} resp = utils.test_request(self.url, 'POST', route, headers=self.json_headers, cookies=self.cookies, data=data) - utils.check_response_basic_info(resp, 404) + utils.check_response_basic_info(resp, 404, expected_method='POST') @pytest.mark.groups @unittest.skipUnless(runner.MAGPIE_TEST_GROUPS, reason=runner.MAGPIE_TEST_DISABLED_MESSAGE('groups')) @@ -425,14 +428,14 @@ def test_PostUserGroup_conflict(self): route = '/users/{usr}/groups'.format(usr=get_constant('MAGPIE_ADMIN_USER')) data = {'group_name': get_constant('MAGPIE_ADMIN_GROUP')} resp = utils.test_request(self.url, 'POST', route, headers=self.json_headers, cookies=self.cookies, data=data) - utils.check_response_basic_info(resp, 409) + utils.check_response_basic_info(resp, 409, expected_method='POST') @pytest.mark.groups @unittest.skipUnless(runner.MAGPIE_TEST_GROUPS, reason=runner.MAGPIE_TEST_DISABLED_MESSAGE('groups')) def test_GetGroupUsers(self): route = '/groups/{grp}/users'.format(grp=get_constant('MAGPIE_ADMIN_GROUP')) resp = utils.test_request(self.url, 'GET', route, headers=self.json_headers, cookies=self.cookies) - json_body = utils.check_response_basic_info(resp, 200) + json_body = utils.check_response_basic_info(resp, 200, expected_method='GET') utils.check_val_is_in('user_names', json_body) utils.check_val_type(json_body['user_names'], list) utils.check_val_is_in(get_constant('MAGPIE_ADMIN_USER'), json_body['user_names']) @@ -443,7 +446,7 @@ def test_GetGroupUsers(self): def test_GetGroupServices(self): route = '/users/{grp}/services'.format(grp=self.grp) resp = utils.test_request(self.url, 'GET', route, headers=self.json_headers, cookies=self.cookies) - json_body = utils.check_response_basic_info(resp, 200) + json_body = utils.check_response_basic_info(resp, 200, expected_method='GET') utils.check_val_is_in('services', json_body) services = json_body['services'] utils.check_val_type(services, dict) @@ -479,7 +482,7 @@ def test_GetGroupServices(self): def test_GetGroupServiceResources(self): route = '/groups/{grp}/services/{svc}/resources'.format(grp=self.grp, svc=self.test_service_name) resp = utils.test_request(self.url, 'GET', route, headers=self.json_headers, cookies=self.cookies) - json_body = utils.check_response_basic_info(resp, 200) + json_body = utils.check_response_basic_info(resp, 200, expected_method='GET') utils.check_val_is_in('service', json_body) svc_dict = json_body['service'] utils.check_val_type(svc_dict, dict) @@ -509,7 +512,7 @@ def test_GetGroupServiceResources(self): def test_GetServiceResources(self): route = '/services/{svc}/resources'.format(svc=self.test_service_name) resp = utils.test_request(self.url, 'GET', route, headers=self.json_headers, cookies=self.cookies) - json_body = utils.check_response_basic_info(resp, 200) + json_body = utils.check_response_basic_info(resp, 200, expected_method='GET') svc_dict = json_body[self.test_service_name] utils.check_val_is_in(self.test_service_name, json_body) utils.check_val_type(json_body[self.test_service_name], dict) @@ -541,7 +544,7 @@ def test_GetServicePermissions(self): service_perms = service_type_dict[svc['service_type']].permission_names route = '/services/{svc}/permissions'.format(svc=svc_name) resp = utils.test_request(self.url, 'GET', route, headers=self.json_headers, cookies=self.cookies) - json_body = utils.check_response_basic_info(resp, 200) + json_body = utils.check_response_basic_info(resp, 200, expected_method='GET') utils.check_val_is_in('permission_names', json_body) utils.check_val_type(json_body['permission_names'], list) utils.check_all_equal(json_body['permission_names'], service_perms, any_order=True) @@ -618,7 +621,7 @@ def test_PostServiceResources_ChildrenResource_ParentID(self): child_resource_id = json_body['resource_id'] route = '/resources/{res_id}'.format(res_id=child_resource_id) resp = utils.test_request(self.url, 'GET', route, headers=self.json_headers, cookies=self.cookies) - json_body = utils.check_response_basic_info(resp, 200) + json_body = utils.check_response_basic_info(resp, 200, expected_method='GET') utils.check_val_is_in(str(child_resource_id), json_body) resource_body = json_body[str(child_resource_id)] utils.check_val_equal(resource_body['root_service_id'], service_root_id) @@ -637,7 +640,7 @@ def test_PostServiceResources_DirectResource_Conflict(self): data = {"resource_name": self.test_resource_name, "resource_type": self.test_resource_type} resp = utils.test_request(self.url, 'POST', route, headers=self.json_headers, cookies=self.cookies, json=data, expect_errors=True) - json_body = utils.check_response_basic_info(resp, 409) + json_body = utils.check_response_basic_info(resp, 409, expected_method='POST') utils.check_error_param_structure(json_body, version=self.version, isParamValueLiteralUnicode=True, paramCompareExists=True, paramValue=self.test_resource_name, paramName=u'resource_name') @@ -670,7 +673,7 @@ def test_ValidateDefaultServiceProviders(self): services_list_getcap = [svc for svc in services_list if 'getcapabilities' in svc['permission_names']] route = '/users/{usr}/services'.format(usr=get_constant('MAGPIE_ANONYMOUS_USER')) resp = utils.test_request(self.url, 'GET', route, headers=self.json_headers, cookies=self.cookies) - json_body = utils.check_response_basic_info(resp, 200) + json_body = utils.check_response_basic_info(resp, 200, expected_method='GET') services_body = json_body['services'] for svc in services_list_getcap: svc_name = svc['service_name'] @@ -697,7 +700,7 @@ def test_PostResources_DirectServiceResource(self): } resp = utils.test_request(self.url, 'POST', '/resources', headers=self.json_headers, cookies=self.cookies, data=data) - json_body = utils.check_response_basic_info(resp, 201) + json_body = utils.check_response_basic_info(resp, 201, expected_method='POST') utils.check_post_resource_structure(json_body, self.test_resource_name, self.test_resource_type, self.test_resource_name, self.version) @@ -715,7 +718,7 @@ def test_PostResources_DirectServiceResourceOptional(self): } resp = utils.test_request(self.url, 'POST', '/resources', headers=self.json_headers, cookies=self.cookies, data=data) - json_body = utils.check_response_basic_info(resp, 201) + json_body = utils.check_response_basic_info(resp, 201, expected_method='POST') utils.check_post_resource_structure(json_body, self.test_resource_name, self.test_resource_type, self.test_resource_name, self.version) @@ -736,7 +739,7 @@ def test_PostResources_ChildrenResource(self): } resp = utils.test_request(self.url, 'POST', '/resources', headers=self.json_headers, cookies=self.cookies, data=data) - json_body = utils.check_response_basic_info(resp, 201) + json_body = utils.check_response_basic_info(resp, 201, expected_method='POST') utils.check_post_resource_structure(json_body, self.test_resource_name, self.test_resource_type, self.test_resource_name, self.version) @@ -749,7 +752,7 @@ def test_PostResources_MissingParentID(self): } resp = utils.test_request(self.url, 'POST', '/resources', headers=self.json_headers, cookies=self.cookies, data=data, expect_errors=True) - json_body = utils.check_response_basic_info(resp, 422) + json_body = utils.check_response_basic_info(resp, 422, expected_method='POST') utils.check_error_param_structure(json_body, paramName='parent_id', paramValue=repr(None), version=self.version) @pytest.mark.resources @@ -763,7 +766,7 @@ def test_DeleteResource(self): route = '/resources/{res_id}'.format(res_id=resource_id) resp = utils.test_request(self.url, 'DELETE', route, headers=self.json_headers, cookies=self.cookies) - utils.check_response_basic_info(resp, 200) + utils.check_response_basic_info(resp, 200, expected_method='DELETE') utils.TestSetup.check_NonExistingTestResource(self) diff --git a/tests/test_magpie_api.py b/tests/test_magpie_api.py index 489bfcc99..0ef65e7ae 100644 --- a/tests/test_magpie_api.py +++ b/tests/test_magpie_api.py @@ -36,6 +36,7 @@ def setUpClass(cls): cls.version = __meta__.__version__ cls.cookies = None cls.usr = get_constant('MAGPIE_ANONYMOUS_USER') + cls.grp = get_constant('MAGPIE_ANONYMOUS_GROUP') @pytest.mark.api @@ -67,6 +68,7 @@ class TestMagpieAPI_AdminAuth_Local(ti.TestMagpieAPI_AdminAuth_Interface): def setUpClass(cls): cls.app = utils.get_test_magpie_app() cls.url = cls.app # to simplify calls of TestSetup (all use .url) + cls.grp = get_constant('MAGPIE_ADMIN_GROUP') cls.usr = get_constant('MAGPIE_TEST_ADMIN_USERNAME') cls.pwd = get_constant('MAGPIE_TEST_ADMIN_PASSWORD') cls.json_headers = utils.get_headers_content_type(cls.app, 'application/json') @@ -75,8 +77,8 @@ def setUpClass(cls): # TODO: fix UI views so that they can be 'found' directly in the WebTest.TestApp # NOTE: localhost magpie has to be running for following login call to work cls.headers, cls.cookies = utils.check_or_try_login_user(cls.app, cls.usr, cls.pwd, - use_ui_form_submit=True, version=cls.version) - cls.require = "cannot run tests without logged in '{}' user".format(get_constant('MAGPIE_ADMIN_GROUP')) + use_ui_form_submit=False, version=cls.version) + cls.require = "cannot run tests without logged in '{}' user".format(cls.grp) cls.check_requirements() cls.get_test_values() @@ -97,6 +99,7 @@ def setUpClass(cls): cls.json_headers = utils.get_headers_content_type(cls.url, 'application/json') cls.cookies = None cls.usr = get_constant('MAGPIE_ANONYMOUS_USER') + cls.grp = get_constant('MAGPIE_ANONYMOUS_GROUP') cls.version = utils.TestSetup.get_Version(cls) @@ -127,11 +130,12 @@ class TestMagpieAPI_AdminAuth_Remote(ti.TestMagpieAPI_AdminAuth_Interface): @classmethod def setUpClass(cls): + cls.grp = get_constant('MAGPIE_ADMIN_GROUP') cls.usr = get_constant('MAGPIE_TEST_ADMIN_USERNAME') cls.pwd = get_constant('MAGPIE_TEST_ADMIN_PASSWORD') cls.url = get_constant('MAGPIE_TEST_REMOTE_SERVER_URL') cls.headers, cls.cookies = utils.check_or_try_login_user(cls.url, cls.usr, cls.pwd) - cls.require = "cannot run tests without logged in '{}' user".format(get_constant('MAGPIE_ADMIN_GROUP')) + cls.require = "cannot run tests without logged in '{}' user".format(cls.grp) cls.json_headers = utils.get_headers_content_type(cls.url, 'application/json') cls.version = utils.TestSetup.get_Version(cls) cls.check_requirements() diff --git a/tests/utils.py b/tests/utils.py index 5e82b337a..437dda0c5 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -1,8 +1,8 @@ -import pyramid import requests import six from six.moves.urllib.parse import urlparse from distutils.version import LooseVersion +from pyramid.testing import setUp as PyramidSetUp from webtest import TestApp from webtest.response import TestResponse from magpie import __meta__, db, services, magpiectl @@ -11,16 +11,13 @@ def config_setup_from_ini(config_ini_file_path): settings = db.get_settings_from_config_ini(config_ini_file_path) - config = pyramid.testing.setUp(settings=settings) + config = PyramidSetUp(settings=settings) return config def get_test_magpie_app(): # parse settings from ini file to pass them to the application config = config_setup_from_ini(get_constant('MAGPIE_INI_FILE_PATH')) - # required redefinition because root models' location is not the same from within this test file - config.add_settings({'ziggurat_foundations.model_locations.User': 'models:User', - 'ziggurat_foundations.model_locations.user': 'models:User', }) config.include('ziggurat_foundations.ext.pyramid.sign_in') config.registry.settings['magpie.db_migration_disabled'] = True # scan dependencies @@ -385,8 +382,7 @@ def check_Unauthorized(test_class, method, path, content_type='application/json' def get_AnyServiceOfTestServiceType(test_class): route = '/services/types/{}'.format(test_class.test_service_type) resp = test_request(test_class.url, 'GET', route, headers=test_class.json_headers, cookies=test_class.cookies) - check_val_equal(resp.status_code, 200) - json_body = get_json_body(resp) + json_body = check_response_basic_info(resp, 200, expected_method='GET') check_val_is_in('services', json_body) check_val_is_in(test_class.test_service_type, json_body['services']) check_val_not_equal(len(json_body['services'][test_class.test_service_type]), 0, @@ -406,7 +402,7 @@ def create_TestServiceResource(test_class, data_override=None): resp = test_request(test_class.url, 'POST', route, headers=test_class.json_headers, cookies=test_class.cookies, json=data) - return check_response_basic_info(resp, 201) + return check_response_basic_info(resp, 201, expected_method='POST') @staticmethod def get_ExistingTestServiceInfo(test_class): @@ -448,7 +444,7 @@ def get_RegisteredServicesList(test_class): resp = test_request(test_class.url, 'GET', '/services', headers=test_class.json_headers, cookies=test_class.cookies) - json_body = check_response_basic_info(resp, 200) + json_body = check_response_basic_info(resp, 200, expected_method='GET') # prepare a flat list of registered services services_list = list() @@ -462,7 +458,7 @@ def get_RegisteredUsersList(test_class): resp = test_request(test_class.url, 'GET', '/users', headers=test_class.json_headers, cookies=test_class.cookies) - json_body = check_response_basic_info(resp, 200) + json_body = check_response_basic_info(resp, 200, expected_method='GET') return json_body['user_names'] @staticmethod @@ -481,7 +477,7 @@ def create_TestUser(test_class): resp = test_request(test_class.url, 'POST', '/users', headers=test_class.json_headers, cookies=test_class.cookies, json=data) - return check_response_basic_info(resp, 201) + return check_response_basic_info(resp, 201, expected_method='POST') @staticmethod def delete_TestUser(test_class): @@ -492,5 +488,5 @@ def delete_TestUser(test_class): resp = test_request(test_class.url, 'DELETE', route, headers=test_class.json_headers, cookies=test_class.cookies) - check_val_equal(resp.status_code, 200) + check_response_basic_info(resp.status_code, 200, expected_method='DELETE') TestSetup.check_NonExistingTestUser(test_class) From 71bf4454f78c1db90e528532fb390d6a901e7115 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Mon, 22 Oct 2018 19:34:58 -0400 Subject: [PATCH 02/13] version 0.7.3 --- magpie/__meta__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/magpie/__meta__.py b/magpie/__meta__.py index 9da6bf761..4e8bd1e4c 100644 --- a/magpie/__meta__.py +++ b/magpie/__meta__.py @@ -2,7 +2,7 @@ General meta information on the magpie package. """ -__version__ = '0.7.2' +__version__ = '0.7.3' __author__ = "Francois-Xavier Derue, Francis Charette-Migneault" __maintainer__ = "Francis Charette-Migneault" __email__ = 'francis.charette-migneault@crim.ca' From c0541c8bc7384075c07ad83e486ebd0319228b09 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Tue, 23 Oct 2018 15:08:43 -0400 Subject: [PATCH 03/13] fix UI redirect new user name --- magpie/definitions/pyramid_definitions.py | 1 + magpie/ui/management/views.py | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/magpie/definitions/pyramid_definitions.py b/magpie/definitions/pyramid_definitions.py index c1b72d6b4..00d37273a 100644 --- a/magpie/definitions/pyramid_definitions.py +++ b/magpie/definitions/pyramid_definitions.py @@ -7,6 +7,7 @@ HTTPCreated, HTTPFound, HTTPTemporaryRedirect, + HTTPMovedPermanently, HTTPBadRequest, HTTPUnauthorized, HTTPForbidden, diff --git a/magpie/ui/management/views.py b/magpie/ui/management/views.py index 296b194cb..c1bebc4f5 100644 --- a/magpie/ui/management/views.py +++ b/magpie/ui/management/views.py @@ -325,9 +325,11 @@ def edit_user(self): if requires_update_name: # re-fetch user groups as current user-group will have changed on new user_name - user_info[u'own_groups'] = self.get_user_groups(user_info[u'user_name']) + user_name = user_info[u'user_name'] + user_info[u'own_groups'] = self.get_user_groups(user_name) # return immediately with updated URL to user with new name - return HTTPFound(self.request.route_url('edit_user', **user_info)) + users_url = self.request.route_url('edit_user', user_name=user_name, cur_svc_type=cur_svc_type) + return HTTPMovedPermanently(location=users_url) # edits to groups checkboxes if is_edit_group_membership: From 7f15c9fc40f6f60181ecf34e359863f9facbb5cf Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Tue, 23 Oct 2018 16:00:42 -0400 Subject: [PATCH 04/13] more ziggurat services fixes --- magpie/models.py | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/magpie/models.py b/magpie/models.py index 37c63c901..cc0f608d3 100644 --- a/magpie/models.py +++ b/magpie/models.py @@ -14,7 +14,7 @@ def get_user(request): userid = request.unauthenticated_userid db_session = get_session_callable(request) if userid is not None: - return User.by_id(userid, db_session=db_session) + return UserService.by_id(userid, db_session=db_session) class Group(GroupMixin, Base): @@ -36,7 +36,7 @@ class GroupResourcePermission(GroupResourcePermissionMixin, Base): class Resource(ResourceMixin, Base): # ... your own properties.... - # example implementation of ACLS for pyramid application + # example implementation of ACL for pyramid application permission_names = [] child_resource_allowed = True @@ -53,15 +53,14 @@ def root_service_id(self): @property def __acl__(self): - acls = [] + acl = [] if self.owner_user_id: - acls.extend([(ALLOW, self.owner_user_id, ALL_PERMISSIONS,), ]) + acl.extend([(ALLOW, self.owner_user_id, ALL_PERMISSIONS,), ]) if self.owner_group_id: - acls.extend([(ALLOW, "group:%s" % self.owner_group_id, - ALL_PERMISSIONS,), ]) - return acls + acl.extend([(ALLOW, "group:%s" % self.owner_group_id, ALL_PERMISSIONS,), ]) + return acl class UserPermission(UserPermissionMixin, Base): @@ -216,7 +215,7 @@ def __repr__(self): return '' % info -class RemoteResourcesSyncInfo(Base): +class RemoteResourcesSyncInfo(BaseModel, Base): __tablename__ = "remote_resources_sync_info" id = sa.Column(sa.Integer(), primary_key=True, nullable=False, autoincrement=True) @@ -250,9 +249,9 @@ def __init__(self, service_cls): super(RemoteResourceTreeService, self).__init__(service_cls) -class RemoteResourceTreeServicePostgreSQL(ResourceTreeServicePostgreSQL): +class RemoteResourceTreeServicePostgresSQL(ResourceTreeServicePostgreSQL): """ - This is necessary, because ResourceTreeServicePostgreSQL.model is the Resource class. + This is necessary, because ResourceTreeServicePostgresSQL.model is the Resource class. If we want to change it for a RemoteResource, we need this class. The ResourceTreeService.__init__ call sets the model. @@ -265,7 +264,7 @@ class RemoteResourceTreeServicePostgreSQL(ResourceTreeServicePostgreSQL): ExternalIdentity, passwordmanager=None) resource_tree_service = ResourceTreeService(ResourceTreeServicePostgreSQL) -remote_resource_tree_service = RemoteResourceTreeService(RemoteResourceTreeServicePostgreSQL) +remote_resource_tree_service = RemoteResourceTreeService(RemoteResourceTreeServicePostgresSQL) resource_type_dict = { Service.resource_type_name: Service, @@ -292,9 +291,9 @@ def get_all_resource_permission_names(): return all_permission_names_list -def find_children_by_name(name, parent_id, db_session): +def find_children_by_name(child_name, parent_id, db_session): tree_struct = resource_tree_service.from_parent_deeper(parent_id=parent_id, limit_depth=1, db_session=db_session) tree_level_entries = [node for node in tree_struct] tree_level_filtered = [node.Resource for node in tree_level_entries if - node.Resource.resource_name.lower() == name.lower()] + node.Resource.resource_name.lower() == child_name.lower()] return tree_level_filtered.pop() if len(tree_level_filtered) else None From 29e88a9555c7b419f143cf60f64e34b8f12c8e64 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Tue, 23 Oct 2018 17:14:08 -0400 Subject: [PATCH 05/13] allow admin view to see private service_url --- magpie/api/management/service/service_views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/magpie/api/management/service/service_views.py b/magpie/api/management/service/service_views.py index ee907c46d..3e3c60b22 100644 --- a/magpie/api/management/service/service_views.py +++ b/magpie/api/management/service/service_views.py @@ -39,7 +39,7 @@ def get_services_runner(request): services = get_services_by_type(service_type, db_session=request.db) json_response[service_type] = {} for service in services: - json_response[service_type][service.resource_name] = format_service(service, show_private_url=False) + json_response[service_type][service.resource_name] = format_service(service, show_private_url=True) return valid_http(httpSuccess=HTTPOk, detail=Services_GET_OkResponseSchema.description, content={u'services': json_response}) From 4314a164a43e6dc2b8ad22d34ccb99ed260fac3a Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Tue, 23 Oct 2018 19:06:04 -0400 Subject: [PATCH 06/13] fix tests and user-services ziggurat (validated localhost) --- magpie/api/api_rest_schemas.py | 7 ++- magpie/api/management/user/user_utils.py | 17 ++++---- magpie/api/management/user/user_views.py | 6 +-- tests/interfaces.py | 42 +++++++++--------- tests/utils.py | 54 ++++++++++++++++++++---- 5 files changed, 84 insertions(+), 42 deletions(-) diff --git a/magpie/api/api_rest_schemas.py b/magpie/api/api_rest_schemas.py index 3282d7331..795774cfd 100644 --- a/magpie/api/api_rest_schemas.py +++ b/magpie/api/api_rest_schemas.py @@ -1708,10 +1708,15 @@ class Groups_GET_ForbiddenResponseSchema(colander.MappingSchema): body = ErrorResponseBodySchema(code=HTTPForbidden.code, description=description) -class Groups_POST_RequestSchema(colander.MappingSchema): +class Groups_POST_RequestBodySchema(colander.MappingSchema): group_name = colander.SchemaNode(colander.String(), description="Name of the group to create.") +class Groups_POST_RequestSchema(colander.MappingSchema): + header = HeaderRequestSchema() + body = Groups_POST_RequestBodySchema() + + class Groups_POST_ResponseBodySchema(BaseResponseBodySchema): group = GroupBodySchema() diff --git a/magpie/api/management/user/user_utils.py b/magpie/api/management/user/user_utils.py index 2d1192404..36d3467ed 100644 --- a/magpie/api/management/user/user_utils.py +++ b/magpie/api/management/user/user_utils.py @@ -24,16 +24,17 @@ def create_user(user_name, password, email, group_name, db_session): msgOnFail=User_Check_ConflictResponseSchema.description) # Create user with specified name and group to assign - user_model = models.User(user_name=user_name, email=email) + new_user = models.User(user_name=user_name, email=email) if password: - user_model.set_password(password) - user_model.regenerate_security_code() - evaluate_call(lambda: db.add(user_model), fallback=lambda: db.rollback(), + UserService.set_password(new_user, password) + UserService.regenerate_security_code(new_user) + evaluate_call(lambda: db.add(new_user), fallback=lambda: db.rollback(), httpError=HTTPForbidden, msgOnFail=Users_POST_ForbiddenResponseSchema.description) - - # Assign user to default group and own group + # Fetch user to update fields new_user = evaluate_call(lambda: UserService.by_user_name(user_name, db_session=db), httpError=HTTPForbidden, msgOnFail=UserNew_POST_ForbiddenResponseSchema.description) + + # Assign user to group group_entry = models.UserGroup(group_id=group_check.id, user_id=new_user.id) evaluate_call(lambda: db.add(group_entry), fallback=lambda: db.rollback(), httpError=HTTPForbidden, msgOnFail=UserGroup_GET_ForbiddenResponseSchema.description) @@ -76,7 +77,7 @@ def get_user_resource_permissions(user, resource, db_session, inherit_groups_per if resource.owner_user_id == user.id: permission_names = models.resource_type_dict[resource.type].permission_names else: - res_perm_tuple_list = resource.perms_for_user(user, db_session=db_session) + res_perm_tuple_list = ResourceService.perms_for_user(resource, user, db_session=db_session) if not inherit_groups_permissions: res_perm_tuple_list = filter_user_permission(res_perm_tuple_list, user) permission_names = [permission.perm_name for permission in res_perm_tuple_list] @@ -156,7 +157,7 @@ def get_user_resources_permissions_dict(user, db_session, resource_types=None, verify_param(user, notNone=True, httpError=HTTPNotFound, msgOnFail=UserResourcePermissions_GET_NotFoundResponseSchema.description) res_perm_tuple_list = UserService.resources_with_possible_perms( - user, resource_ids=resource_ids,resource_types=resource_types, db_session=db_session) + user, resource_ids=resource_ids, resource_types=resource_types, db_session=db_session) if not inherit_groups_permissions: res_perm_tuple_list = filter_user_permission(res_perm_tuple_list, user) resources_permissions_dict = {} diff --git a/magpie/api/management/user/user_views.py b/magpie/api/management/user/user_views.py index a3ee584c7..8fa61c494 100644 --- a/magpie/api/management/user/user_views.py +++ b/magpie/api/management/user/user_views.py @@ -78,8 +78,7 @@ def get_user_view(request): def delete_user(request): """Delete a user by name.""" user = get_user_matchdict_checked_or_logged(request) - db = request.db - evaluate_call(lambda: db.delete(user), fallback=lambda: db.rollback(), + evaluate_call(lambda: request.db.delete(user), fallback=lambda: request.db.rollback(), httpError=HTTPForbidden, msgOnFail=User_DELETE_ForbiddenResponseSchema.description) return valid_http(httpSuccess=HTTPOk, detail=User_DELETE_OkResponseSchema.description) @@ -301,7 +300,8 @@ def get_user_inherited_services_view(request): @LoggedUserServiceInheritedPermissionsAPI.get(schema=UserServicePermissions_GET_RequestSchema, tags=[LoggedUserTag], api_security=SecurityEveryoneAPI, response_schemas=LoggedUserServicePermissions_GET_responses) -@view_config(route_name=UserServiceInheritedPermissionsAPI.name, request_method='GET', permission=NO_PERMISSION_REQUIRED) +@view_config(route_name=UserServiceInheritedPermissionsAPI.name, request_method='GET', + permission=NO_PERMISSION_REQUIRED) def get_user_service_inherited_permissions_view(request): """List all permissions a user has on a service using all his inherited user and groups permissions.""" LOGGER.warn("Route deprecated: [{0}], Instead Use: [{1}]" diff --git a/tests/interfaces.py b/tests/interfaces.py index 3577737c7..94ad29368 100644 --- a/tests/interfaces.py +++ b/tests/interfaces.py @@ -136,6 +136,7 @@ def get_test_values(cls): assert len(test_service_resource_types), "test service should allow at least 1 sub-resource for test execution" cls.test_resource_type = test_service_resource_types[0] + cls.test_group_name = u'magpie-unittest-dummy-group' cls.test_user_name = u'magpie-unittest-toto' cls.test_user_group = u'users' @@ -247,20 +248,23 @@ def test_GetUserInheritedResources(self): utils.check_val_is_in('resource_id', svc_dict) utils.check_val_is_in('service_name', svc_dict) utils.check_val_is_in('service_type', svc_dict) - utils.check_val_is_in('service_url', svc_dict) utils.check_val_is_in('public_url', svc_dict) utils.check_val_is_in('permission_names', svc_dict) utils.check_val_is_in('resources', svc_dict) utils.check_val_type(svc_dict['resource_id'], int) utils.check_val_type(svc_dict['service_name'], six.string_types) - utils.check_val_type(svc_dict['service_url'], six.string_types) utils.check_val_type(svc_dict['service_type'], six.string_types) utils.check_val_type(svc_dict['public_url'], six.string_types) utils.check_val_type(svc_dict['permission_names'], list) utils.check_val_type(svc_dict['resources'], dict) if LooseVersion(self.version) >= LooseVersion('0.7.0'): utils.check_val_is_in('service_sync_type', svc_dict) - utils.check_val_type(svc_dict['service_sync_type'], six.string_types) + utils.check_val_type(svc_dict['service_sync_type'], six.string_types + tuple([type(None)])) + utils.check_val_not_in('service_url', svc_dict, + msg="Services under user routes shouldn't show private url.") + else: + utils.check_val_is_in('service_url', svc_dict) + utils.check_val_type(svc_dict['service_url'], six.string_types) @pytest.mark.users @unittest.skipUnless(runner.MAGPIE_TEST_USERS, reason=runner.MAGPIE_TEST_DISABLED_MESSAGE('users')) @@ -296,8 +300,8 @@ def test_GetUserServices(self): if LooseVersion(self.version) >= LooseVersion('0.7.0'): utils.check_val_is_in('service_sync_type', svc_dict) utils.check_val_type(svc_dict['service_sync_type'], six.string_types) - if LooseVersion(self.version) >= LooseVersion('0.7.1'): - utils.check_val_not_in('service_url', svc_dict) + utils.check_val_not_in('service_url', svc_dict, + msg="Services under user routes shouldn't show private url.") else: utils.check_val_is_in('service_url', svc_dict) utils.check_val_type(svc_dict['service_url'], six.string_types) @@ -326,7 +330,6 @@ def test_GetUserServiceResources(self): if LooseVersion(self.version) >= LooseVersion('0.7.0'): utils.check_val_is_in('service_sync_type', svc_dict) utils.check_val_type(svc_dict['service_sync_type'], six.string_types) - if LooseVersion(self.version) >= LooseVersion('0.7.1'): utils.check_val_not_in('service_url', svc_dict) else: utils.check_val_is_in('service_url', svc_dict) @@ -409,10 +412,9 @@ def test_ValidateDefaultGroups(self): @pytest.mark.groups @unittest.skipUnless(runner.MAGPIE_TEST_GROUPS, reason=runner.MAGPIE_TEST_DISABLED_MESSAGE('groups')) def test_PostUserGroup_assign(self): - route = '/users/{usr}/groups'.format(usr=get_constant('MAGPIE_ADMIN_USER')) - data = {'group_name': get_constant('MAGPIE_ANONYMOUS_GROUP')} - resp = utils.test_request(self.url, 'POST', route, headers=self.json_headers, cookies=self.cookies, data=data) - utils.check_response_basic_info(resp, 201, expected_method='POST') + utils.TestSetup.delete_TestGroup(self) # setup as required + utils.TestSetup.create_TestGroup(self) # actual test + utils.TestSetup.delete_TestGroup(self) # cleanup @pytest.mark.groups @unittest.skipUnless(runner.MAGPIE_TEST_GROUPS, reason=runner.MAGPIE_TEST_DISABLED_MESSAGE('groups')) @@ -444,15 +446,19 @@ def test_GetGroupUsers(self): @pytest.mark.groups @unittest.skipUnless(runner.MAGPIE_TEST_USERS, reason=runner.MAGPIE_TEST_DISABLED_MESSAGE('groups')) def test_GetGroupServices(self): - route = '/users/{grp}/services'.format(grp=self.grp) + route = '/groups/{grp}/services'.format(grp=self.grp) resp = utils.test_request(self.url, 'GET', route, headers=self.json_headers, cookies=self.cookies) json_body = utils.check_response_basic_info(resp, 200, expected_method='GET') utils.check_val_is_in('services', json_body) services = json_body['services'] utils.check_val_type(services, dict) service_types = utils.get_service_types_for_version(self.version) - utils.check_all_equal(services.keys(), service_types, any_order=True) + # as of version '0.7.0', visible services depend on the connected user permissions, + # so all services types not necessarily returned in the response + if LooseVersion(self.version) < LooseVersion('0.7.0'): + utils.check_all_equal(services.keys(), service_types, any_order=True) for svc_type in services: + utils.check_val_is_in(svc_type, service_types) # one of valid service types for svc in services[svc_type]: svc_dict = services[svc_type][svc] utils.check_val_type(svc_dict, dict) @@ -461,17 +467,14 @@ def test_GetGroupServices(self): utils.check_val_is_in('service_type', svc_dict) utils.check_val_is_in('public_url', svc_dict) utils.check_val_is_in('permission_names', svc_dict) - utils.check_val_is_in('resources', svc_dict) utils.check_val_type(svc_dict['resource_id'], int) utils.check_val_type(svc_dict['service_name'], six.string_types) utils.check_val_type(svc_dict['service_type'], six.string_types) utils.check_val_type(svc_dict['public_url'], six.string_types) utils.check_val_type(svc_dict['permission_names'], list) - utils.check_val_type(svc_dict['resources'], dict) if LooseVersion(self.version) >= LooseVersion('0.7.0'): utils.check_val_is_in('service_sync_type', svc_dict) - utils.check_val_type(svc_dict['service_sync_type'], six.string_types) - if LooseVersion(self.version) >= LooseVersion('0.7.1'): + utils.check_val_type(svc_dict['service_sync_type'], six.string_types + tuple([type(None)])) utils.check_val_not_in('service_url', svc_dict) else: utils.check_val_is_in('service_url', svc_dict) @@ -501,7 +504,6 @@ def test_GetGroupServiceResources(self): if LooseVersion(self.version) >= LooseVersion('0.7.0'): utils.check_val_is_in('service_sync_type', svc_dict) utils.check_val_type(svc_dict['service_sync_type'], six.string_types) - if LooseVersion(self.version) >= LooseVersion('0.7.1'): utils.check_val_not_in('service_url', svc_dict) else: utils.check_val_is_in('service_url', svc_dict) @@ -670,17 +672,15 @@ def test_ValidateDefaultServiceProviders(self): utils.check_val_is_in(svc_name, registered_svc_names) # ensure that 'getcapabilities' permission is given to anonymous for applicable services + anonymous = get_constant('MAGPIE_ANONYMOUS_USER') services_list_getcap = [svc for svc in services_list if 'getcapabilities' in svc['permission_names']] - route = '/users/{usr}/services'.format(usr=get_constant('MAGPIE_ANONYMOUS_USER')) + route = '/users/{usr}/services'.format(usr=anonymous) resp = utils.test_request(self.url, 'GET', route, headers=self.json_headers, cookies=self.cookies) json_body = utils.check_response_basic_info(resp, 200, expected_method='GET') services_body = json_body['services'] for svc in services_list_getcap: svc_name = svc['service_name'] svc_type = svc['service_type'] - if LooseVersion(self.version) >= LooseVersion('0.7.1'): - utils.check_val_not_in('service_url', svc, msg="Services under user routes shouldn't show private url.") - anonymous = get_constant('MAGPIE_ANONYMOUS_USER') msg = "Service `{name}` of type `{type}` is expected to have `{perm}` permissions for user `{usr}`" \ .format(name=svc_name, type=svc_type, perm='getcapabilities', usr=anonymous) utils.check_val_is_in(svc_name, services_body[svc_type], msg=msg) diff --git a/tests/utils.py b/tests/utils.py index 437dda0c5..deea6992e 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -170,8 +170,11 @@ def check_or_try_login_user(app_or_url, username=None, password=None, provider=' return resp.headers, resp_cookies -def format_test_val_ref(val, ref, pre='Fail'): - return '({0}) Test value: `{1}`, Reference value: `{2}`'.format(pre, val, ref) +def format_test_val_ref(val, ref, pre='Fail', msg=None): + _msg = '({0}) Test value: `{1}`, Reference value: `{2}`'.format(pre, val, ref) + if isinstance(msg, six.string_types): + _msg = '{}\n{}'.format(msg, _msg) + return _msg def all_equal(iter_val, iter_ref, any_order=False): @@ -187,27 +190,27 @@ def all_equal(iter_val, iter_ref, any_order=False): def check_all_equal(iter_val, iter_ref, any_order=False, msg=None): r_it_val = repr(iter_val) r_it_ref = repr(iter_ref) - assert all_equal(iter_val, iter_ref, any_order), msg or format_test_val_ref(r_it_val, r_it_ref, pre='Equal Fail') + assert all_equal(iter_val, iter_ref, any_order), format_test_val_ref(r_it_val, r_it_ref, pre='Equal Fail', msg=msg) def check_val_equal(val, ref, msg=None): - assert isinstance(ref, null) or val == ref, msg or format_test_val_ref(val, ref, pre='Equal Fail') + assert isinstance(ref, null) or val == ref, format_test_val_ref(val, ref, pre='Equal Fail', msg=msg) def check_val_not_equal(val, ref, msg=None): - assert isinstance(ref, null) or val != ref, msg or format_test_val_ref(val, ref, pre='Equal Fail') + assert isinstance(ref, null) or val != ref, format_test_val_ref(val, ref, pre='Equal Fail', msg=msg) def check_val_is_in(val, ref, msg=None): - assert isinstance(ref, null) or val in ref, msg or format_test_val_ref(val, ref, pre='Is In Fail') + assert isinstance(ref, null) or val in ref, format_test_val_ref(val, ref, pre='Is In Fail', msg=msg) def check_val_not_in(val, ref, msg=None): - assert isinstance(ref, null) or val not in ref, msg or format_test_val_ref(val, ref, pre='Not In Fail') + assert isinstance(ref, null) or val not in ref, format_test_val_ref(val, ref, pre='Not In Fail', msg=msg) def check_val_type(val, ref, msg=None): - assert isinstance(val, ref), msg or format_test_val_ref(val, repr(ref), pre='Type Fail') + assert isinstance(val, ref), format_test_val_ref(val, repr(ref), pre='Type Fail', msg=msg) def check_response_basic_info(response, expected_code=200, expected_type='application/json', expected_method='GET'): @@ -488,5 +491,38 @@ def delete_TestUser(test_class): resp = test_request(test_class.url, 'DELETE', route, headers=test_class.json_headers, cookies=test_class.cookies) - check_response_basic_info(resp.status_code, 200, expected_method='DELETE') + check_response_basic_info(resp, 200, expected_method='DELETE') TestSetup.check_NonExistingTestUser(test_class) + + @staticmethod + def get_RegisteredGroupsList(test_class): + resp = test_request(test_class.url, 'GET', '/groups', + headers=test_class.json_headers, + cookies=test_class.cookies) + json_body = check_response_basic_info(resp, 200, expected_method='GET') + return json_body['group_names'] + + @staticmethod + def check_NonExistingTestGroup(test_class): + groups = TestSetup.get_RegisteredGroupsList(test_class) + check_val_not_in(test_class.test_group_name, groups) + + @staticmethod + def create_TestGroup(test_class): + data = {"group_name": test_class.test_group_name} + resp = test_request(test_class.url, 'POST', '/groups', + headers=test_class.json_headers, + cookies=test_class.cookies, json=data) + return check_response_basic_info(resp, 201, expected_method='POST') + + @staticmethod + def delete_TestGroup(test_class): + groups = TestSetup.get_RegisteredGroupsList(test_class) + # delete as required, skip if non-existing + if test_class.test_group_name in groups: + route = '/groups/{grp}'.format(grp=test_class.test_group_name) + resp = test_request(test_class.url, 'DELETE', route, + headers=test_class.json_headers, + cookies=test_class.cookies) + check_response_basic_info(resp, 200, expected_method='DELETE') + TestSetup.check_NonExistingTestGroup(test_class) From b19b06d95692d0eeebd1f1207ab9862d84a64b31 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Wed, 24 Oct 2018 19:08:07 -0400 Subject: [PATCH 07/13] fix services ziggurat query --- magpie/services.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/magpie/services.py b/magpie/services.py index 78008910c..42e5fec5e 100644 --- a/magpie/services.py +++ b/magpie/services.py @@ -39,7 +39,7 @@ def expand_acl(self, resource, user): # Custom acl if user: - permissions = resource.perms_for_user(user) + permissions = ResourceService.perms_for_user(resource, user, db_session=self.request.db) for outcome, perm_user, perm_name in permission_to_pyramid_acls(permissions): self.acl.append((outcome, perm_user, perm_name,)) else: @@ -47,7 +47,7 @@ def expand_acl(self, resource, user): if user is None: raise Exception('No Anonymous user in the database') else: - permissions = resource.perms_for_user(user) + permissions = ResourceService.perms_for_user(resource, user, db_session=self.request.db) for outcome, perm_user, perm_name in permission_to_pyramid_acls(permissions): self.acl.append((outcome, EVERYONE, perm_name,)) From 8e87202387a99fc335c22a9a31c20aa894dc90f5 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Wed, 24 Oct 2018 20:01:19 -0400 Subject: [PATCH 08/13] MagpieAdapter using admin requests --- magpie/adapter/__init__.py | 33 +++++++++++++- magpie/adapter/magpieowssecurity.py | 18 +++++--- magpie/adapter/magpieservice.py | 52 +++++++++-------------- magpie/definitions/pyramid_definitions.py | 1 + 4 files changed, 64 insertions(+), 40 deletions(-) diff --git a/magpie/adapter/__init__.py b/magpie/adapter/__init__.py index 4c6d730b6..b96dc7365 100644 --- a/magpie/adapter/__init__.py +++ b/magpie/adapter/__init__.py @@ -4,13 +4,44 @@ from magpie.adapter.magpieowssecurity import * from magpie.adapter.magpieservice import MagpieServiceStore from magpie.models import get_user +from magpie.constants import get_constant from magpie.security import auth_config_from_settings from magpie.db import * from magpie import __meta__ +from six.moves.urllib.parse import urlparse +from typing import Dict import logging logger = logging.getLogger("TWITCHER") +def get_admin_cookies(magpie_url, verify=True): + # type: (str, bool) -> Dict[str,str] + magpie_login_url = '{}/signin'.format(magpie_url) + cred = {'user_name': get_constant('MAGPIE_ADMIN_USER'), 'password': get_constant('MAGPIE_ADMIN_PASSWORD')} + resp = requests.post(magpie_login_url, data=cred, headers='application/json', verify=verify) + if resp.status_code != HTTPOk.code: + raise resp.raise_for_status() + return dict(auth_tkt=resp.cookies.get('auth_tkt')) + + +def get_magpie_url(registry): + # type: (Registry) -> str + try: + # add 'http' scheme to url if omitted from config since further 'requests' calls fail without it + # mostly for testing when only 'localhost' is specified + # otherwise twitcher config should explicitly define it in MAGPIE_URL + url_parsed = urlparse(registry.settings.get('magpie.url').strip('/')) + if url_parsed.scheme in ['http', 'https']: + return url_parsed.geturl() + else: + magpie_url = 'http://{}'.format(url_parsed.geturl()) + LOGGER.warn("Missing scheme from registry url, new value: '{}'".format(magpie_url)) + return magpie_url + except AttributeError: + # If magpie.url does not exist, calling strip fct over None will raise this issue + raise ConfigurationError('magpie.url config cannot be found') + + class MagpieAdapter(AdapterInterface): def describe_adapter(self): return {"name": self.__class__.__name__, "version": __meta__.__version__} @@ -29,7 +60,7 @@ def jobstore_factory(self, registry): return DefaultAdapter().jobstore_factory(registry) def owssecurity_factory(self, registry): - return MagpieOWSSecurity() + return MagpieOWSSecurity(registry=registry) def configurator_factory(self, settings): # Disable rpcinterface which is conflicting with postgres db diff --git a/magpie/adapter/magpieowssecurity.py b/magpie/adapter/magpieowssecurity.py index 1770360b9..3a2e738e0 100644 --- a/magpie/adapter/magpieowssecurity.py +++ b/magpie/adapter/magpieowssecurity.py @@ -3,6 +3,7 @@ from magpie.services import service_factory from magpie.models import Service from magpie.api.api_except import evaluate_call, verify_param +from magpie.adapter import get_magpie_url import requests import logging LOGGER = logging.getLogger("TWITCHER") @@ -10,10 +11,15 @@ class MagpieOWSSecurity(OWSSecurityInterface): + def __init__(self, registry): + super(MagpieOWSSecurity, self).__init__(registry=registry) + self.magpie_url = get_magpie_url(registry) + self.twitcher_ssl_verify = asbool(registry.settings.get('twitcher.ows_proxy_ssl_verify', True)) + self.twitcher_protected_path = registry.settings.get('twitcher.ows_proxy_protected_path', '/ows') + def check_request(self, request): - twitcher_protected_path = request.registry.settings.get('twitcher.ows_proxy_protected_path', '/ows') - if request.path.startswith(twitcher_protected_path): - service_name = parse_service_name(request.path, twitcher_protected_path) + if request.path.startswith(self.twitcher_protected_path): + service_name = parse_service_name(request.path, self.twitcher_protected_path) service = evaluate_call(lambda: Service.by_service_name(service_name, db_session=request.db), fallback=lambda: request.db.rollback(), httpError=HTTPForbidden, msgOnFail="Service query by name refused by db") @@ -34,8 +40,7 @@ def check_request(self, request): if not has_permission: raise OWSAccessForbidden("Not authorized to access this resource.") - @staticmethod - def update_request_cookies(request): + def update_request_cookies(self, request): """ Ensure login of the user and update the request cookies if twitcher is in a special configuration. Only update if Magpie `auth_tkt` is missing and can be retrieved from `access_token` in `Authorization` header. @@ -43,14 +48,13 @@ def update_request_cookies(request): """ not_default = get_twitcher_configuration(request.registry.settings) != TWITCHER_CONFIGURATION_DEFAULT if not_default and 'Authorization' in request.headers and 'auth_tkt' not in request.cookies: - ssl_verify = asbool(request.registry.settings.get('twitcher.ows_proxy_ssl_verify', True)) magpie_url = request.registry.settings.get('magpie.url') magpie_prov = request.params.get('provider', 'WSO2') magpie_auth = '{host}/providers/{provider}/signin'.format(host=magpie_url, provider=magpie_prov) headers = request.headers headers['Homepage-Route'] = '/session' headers['Accept'] = 'application/json' - auth_resp = requests.get(magpie_auth, headers=headers, verify=ssl_verify) + auth_resp = requests.get(magpie_auth, headers=headers, verify=self.twitcher_ssl_verify) if auth_resp.status_code != HTTPOk.code: raise auth_resp.raise_for_status() if not auth_resp.json().get('authenticated') or 'auth_tkt' not in auth_resp.request._cookies: diff --git a/magpie/adapter/magpieservice.py b/magpie/adapter/magpieservice.py index d9acbc87c..8adb00430 100644 --- a/magpie/adapter/magpieservice.py +++ b/magpie/adapter/magpieservice.py @@ -2,34 +2,22 @@ Store adapters to read data from magpie. """ -from six.moves.urllib.parse import urlparse -import logging +from magpie.definitions.twitcher_definitions import * +from magpie.definitions.pyramid_definitions import HTTPOk, asbool +from magpie.adapter import get_admin_cookies, get_magpie_url import requests -import json +import logging LOGGER = logging.getLogger("TWITCHER") -from magpie.definitions.twitcher_definitions import * -from magpie.definitions.pyramid_definitions import ConfigurationError, HTTPOk - class MagpieServiceStore(ServiceStore): """ Registry for OWS services. Uses magpie to fetch service url and attributes. """ def __init__(self, registry): - try: - # add 'http' scheme to url if omitted from config since further 'requests' calls fail without it - # mostly for testing when only 'localhost' is specified - # otherwise twitcher config should explicitly define it in MAGPIE_URL - url_parsed = urlparse(registry.settings.get('magpie.url').strip('/')) - if url_parsed.scheme in ['http', 'https']: - self.magpie_url = url_parsed.geturl() - else: - self.magpie_url = 'http://{}'.format(url_parsed.geturl()) - LOGGER.warn("Missing scheme from MagpieServiceStore url, new value: '{}'".format(self.magpie_url)) - except AttributeError: - #If magpie.url does not exist, calling strip fct over None will raise this issue - raise ConfigurationError('magpie.url config cannot be found') + self.magpie_url = get_magpie_url(registry) + self.twitcher_ssl_verify = asbool(registry.settings.get('twitcher.ows_proxy_ssl_verify', True)) + self.magpie_admin_token = get_admin_cookies(self.magpie_url, self.twitcher_ssl_verify) def save_service(self, service, overwrite=True, request=None): """ @@ -47,19 +35,19 @@ def list_services(self, request=None): """ Lists all services registered in magpie. """ - my_services = [] - path = '/users/current/services?inherit=True&cascade=True' - response = requests.get('{url}{path}'.format(url=self.magpie_url, path=path), - cookies=request.cookies) - if response.status_code != HTTPOk.code: - raise response.raise_for_status() - services = json.loads(response.text) - for service_type in services['services']: - for key, service in services['services'][service_type].items(): - my_services.append(Service(url=service['service_url'], - name=service['service_name'], - type=service['service_type'])) - return my_services + # obtain admin access since 'service_url' is only provided on admin routes + services = [] + path = '{}/services'.format(self.magpie_url) + resp = requests.get(path, cookies=self.magpie_admin_token, headers={'Accept': 'application/json'}) + if resp.status_code != HTTPOk.code: + raise resp.raise_for_status() + json_body = resp.json() + for service_type in json_body['services']: + for key, service in json_body['services'][service_type].items(): + services.append(Service(url=service['service_url'], + name=service['service_name'], + type=service['service_type'])) + return services def fetch_by_name(self, name, request=None): """ diff --git a/magpie/definitions/pyramid_definitions.py b/magpie/definitions/pyramid_definitions.py index 00d37273a..ab58c11e7 100644 --- a/magpie/definitions/pyramid_definitions.py +++ b/magpie/definitions/pyramid_definitions.py @@ -19,6 +19,7 @@ HTTPInternalServerError, ) from pyramid.settings import asbool +from pyramid.registry import Registry from pyramid.interfaces import IAuthenticationPolicy, IAuthorizationPolicy from pyramid.response import Response, FileResponse from pyramid.view import ( From 8f51a1ab2886dfe0ae578cf2fa65bafbdd05ed14 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Wed, 24 Oct 2018 20:09:15 -0400 Subject: [PATCH 09/13] fix imports --- magpie/adapter/__init__.py | 39 +++-------------------------- magpie/adapter/magpieowssecurity.py | 2 +- magpie/adapter/magpieservice.py | 2 +- magpie/adapter/utils.py | 35 ++++++++++++++++++++++++++ 4 files changed, 41 insertions(+), 37 deletions(-) create mode 100644 magpie/adapter/utils.py diff --git a/magpie/adapter/__init__.py b/magpie/adapter/__init__.py index b96dc7365..f4c58ad82 100644 --- a/magpie/adapter/__init__.py +++ b/magpie/adapter/__init__.py @@ -4,42 +4,11 @@ from magpie.adapter.magpieowssecurity import * from magpie.adapter.magpieservice import MagpieServiceStore from magpie.models import get_user -from magpie.constants import get_constant from magpie.security import auth_config_from_settings from magpie.db import * from magpie import __meta__ -from six.moves.urllib.parse import urlparse -from typing import Dict import logging -logger = logging.getLogger("TWITCHER") - - -def get_admin_cookies(magpie_url, verify=True): - # type: (str, bool) -> Dict[str,str] - magpie_login_url = '{}/signin'.format(magpie_url) - cred = {'user_name': get_constant('MAGPIE_ADMIN_USER'), 'password': get_constant('MAGPIE_ADMIN_PASSWORD')} - resp = requests.post(magpie_login_url, data=cred, headers='application/json', verify=verify) - if resp.status_code != HTTPOk.code: - raise resp.raise_for_status() - return dict(auth_tkt=resp.cookies.get('auth_tkt')) - - -def get_magpie_url(registry): - # type: (Registry) -> str - try: - # add 'http' scheme to url if omitted from config since further 'requests' calls fail without it - # mostly for testing when only 'localhost' is specified - # otherwise twitcher config should explicitly define it in MAGPIE_URL - url_parsed = urlparse(registry.settings.get('magpie.url').strip('/')) - if url_parsed.scheme in ['http', 'https']: - return url_parsed.geturl() - else: - magpie_url = 'http://{}'.format(url_parsed.geturl()) - LOGGER.warn("Missing scheme from registry url, new value: '{}'".format(magpie_url)) - return magpie_url - except AttributeError: - # If magpie.url does not exist, calling strip fct over None will raise this issue - raise ConfigurationError('magpie.url config cannot be found') +LOGGER = logging.getLogger("TWITCHER") class MagpieAdapter(AdapterInterface): @@ -66,14 +35,14 @@ def configurator_factory(self, settings): # Disable rpcinterface which is conflicting with postgres db settings['twitcher.rpcinterface'] = False - logger.info('Loading MagpieAdapter config') + LOGGER.info('Loading MagpieAdapter config') config = auth_config_from_settings(settings) config.set_request_property(get_user, 'user', reify=True) self.owsproxy_config(settings, config) return config def owsproxy_config(self, settings, config): - logger.info('Loading MagpieAdapter owsproxy config') + LOGGER.info('Loading MagpieAdapter owsproxy config') # use pyramid_tm to hook the transaction lifecycle to the request config.include('pyramid_tm') @@ -89,7 +58,7 @@ def owsproxy_config(self, settings, config): reify=True ) - logger.info('Adding MagpieAdapter owsproxy routes and views') + LOGGER.info('Adding MagpieAdapter owsproxy routes and views') protected_path = settings.get('twitcher.ows_proxy_protected_path', '/ows') config.add_route('owsproxy', protected_path + '/{service_name}') config.add_route('owsproxy_extra', protected_path + '/{service_name}/{extra_path:.*}') diff --git a/magpie/adapter/magpieowssecurity.py b/magpie/adapter/magpieowssecurity.py index 3a2e738e0..61e5f31fd 100644 --- a/magpie/adapter/magpieowssecurity.py +++ b/magpie/adapter/magpieowssecurity.py @@ -3,7 +3,7 @@ from magpie.services import service_factory from magpie.models import Service from magpie.api.api_except import evaluate_call, verify_param -from magpie.adapter import get_magpie_url +from magpie.adapter.utils import get_magpie_url import requests import logging LOGGER = logging.getLogger("TWITCHER") diff --git a/magpie/adapter/magpieservice.py b/magpie/adapter/magpieservice.py index 8adb00430..3ab8e4ca5 100644 --- a/magpie/adapter/magpieservice.py +++ b/magpie/adapter/magpieservice.py @@ -4,7 +4,7 @@ from magpie.definitions.twitcher_definitions import * from magpie.definitions.pyramid_definitions import HTTPOk, asbool -from magpie.adapter import get_admin_cookies, get_magpie_url +from magpie.adapter.utils import get_admin_cookies, get_magpie_url import requests import logging LOGGER = logging.getLogger("TWITCHER") diff --git a/magpie/adapter/utils.py b/magpie/adapter/utils.py new file mode 100644 index 000000000..ff6c9ec16 --- /dev/null +++ b/magpie/adapter/utils.py @@ -0,0 +1,35 @@ +from magpie.constants import get_constant +from magpie.definitions.pyramid_definitions import HTTPOk, ConfigurationError, Registry +from six.moves.urllib.parse import urlparse +from typing import Dict +import requests +import logging +LOGGER = logging.getLogger("TWITCHER") + + +def get_admin_cookies(magpie_url, verify=True): + # type: (str, bool) -> Dict[str,str] + magpie_login_url = '{}/signin'.format(magpie_url) + cred = {'user_name': get_constant('MAGPIE_ADMIN_USER'), 'password': get_constant('MAGPIE_ADMIN_PASSWORD')} + resp = requests.post(magpie_login_url, data=cred, headers='application/json', verify=verify) + if resp.status_code != HTTPOk.code: + raise resp.raise_for_status() + return dict(auth_tkt=resp.cookies.get('auth_tkt')) + + +def get_magpie_url(registry): + # type: (Registry) -> str + try: + # add 'http' scheme to url if omitted from config since further 'requests' calls fail without it + # mostly for testing when only 'localhost' is specified + # otherwise twitcher config should explicitly define it in MAGPIE_URL + url_parsed = urlparse(registry.settings.get('magpie.url').strip('/')) + if url_parsed.scheme in ['http', 'https']: + return url_parsed.geturl() + else: + magpie_url = 'http://{}'.format(url_parsed.geturl()) + LOGGER.warn("Missing scheme from registry url, new value: '{}'".format(magpie_url)) + return magpie_url + except AttributeError: + # If magpie.url does not exist, calling strip fct over None will raise this issue + raise ConfigurationError('magpie.url config cannot be found') From 07cce824a294554372f528025211297af16ea909 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Wed, 24 Oct 2018 20:11:38 -0400 Subject: [PATCH 10/13] fix init --- magpie/adapter/magpieowssecurity.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/magpie/adapter/magpieowssecurity.py b/magpie/adapter/magpieowssecurity.py index 61e5f31fd..0d7d28fe9 100644 --- a/magpie/adapter/magpieowssecurity.py +++ b/magpie/adapter/magpieowssecurity.py @@ -12,7 +12,7 @@ class MagpieOWSSecurity(OWSSecurityInterface): def __init__(self, registry): - super(MagpieOWSSecurity, self).__init__(registry=registry) + super(MagpieOWSSecurity, self).__init__() self.magpie_url = get_magpie_url(registry) self.twitcher_ssl_verify = asbool(registry.settings.get('twitcher.ows_proxy_ssl_verify', True)) self.twitcher_protected_path = registry.settings.get('twitcher.ows_proxy_protected_path', '/ows') From 8b9568a354de19399634f1836d4855c88befe7e0 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Wed, 24 Oct 2018 20:16:47 -0400 Subject: [PATCH 11/13] fix accept headers --- magpie/adapter/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/magpie/adapter/utils.py b/magpie/adapter/utils.py index ff6c9ec16..3effa952c 100644 --- a/magpie/adapter/utils.py +++ b/magpie/adapter/utils.py @@ -11,7 +11,7 @@ def get_admin_cookies(magpie_url, verify=True): # type: (str, bool) -> Dict[str,str] magpie_login_url = '{}/signin'.format(magpie_url) cred = {'user_name': get_constant('MAGPIE_ADMIN_USER'), 'password': get_constant('MAGPIE_ADMIN_PASSWORD')} - resp = requests.post(magpie_login_url, data=cred, headers='application/json', verify=verify) + resp = requests.post(magpie_login_url, data=cred, headers={'Accept': 'application/json'}, verify=verify) if resp.status_code != HTTPOk.code: raise resp.raise_for_status() return dict(auth_tkt=resp.cookies.get('auth_tkt')) From ffc1adf2406e66ba6d71d9c0d81967ac6fb2aaa1 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Thu, 25 Oct 2018 11:45:19 -0400 Subject: [PATCH 12/13] resolve OWSParser for json requests --- magpie/owsrequest.py | 50 ++++++++++++++++++++++++++++++++++---------- magpie/services.py | 14 +++++++------ 2 files changed, 47 insertions(+), 17 deletions(-) diff --git a/magpie/owsrequest.py b/magpie/owsrequest.py index 75dac4408..b0aad529e 100644 --- a/magpie/owsrequest.py +++ b/magpie/owsrequest.py @@ -5,19 +5,42 @@ * https://github.com/geopython/pywps/blob/master/pywps/app/WPSRequest.py """ +from magpie.api.api_except import raise_http +from pyramid.httpexceptions import HTTPMethodNotAllowed +from requests import Request import lxml.etree -from pyramid.httpexceptions import HTTPBadRequest +import json import logging logger = logging.getLogger(__name__) def ows_parser_factory(request): - if request.method == 'GET': - return Get(request) - elif request.method == 'POST': - return Post(request) + # type: (Request) -> OWSParser + """ + Retrieve the appropriate OWS Request parser using the Content-Type header. + Default to JSON if no Content-Type is specified or if it is 'text/plain' but can be parsed as JSON. + Otherwise, use the GET/POST WPS parsers. + """ + content_type = request.headers.get('Content-Type', 'application/json') + if content_type == 'text/plain': + try: + if request.body: + # raises if parsing fails + json.loads(request.body) + content_type = 'application/json' + except ValueError: + pass + if content_type == 'application/json': + return JSONParser(request) else: - raise HTTPBadRequest() + if request.method == 'GET': + return WPSGet(request) + elif request.method == 'POST': + return WPSPost(request) + + # method not supported, raise using the specified content type header + raise_http(httpError=HTTPMethodNotAllowed, contentType=content_type, + detail="Method not implemented by Magpie OWSParser.") class OWSParser(object): @@ -35,17 +58,16 @@ def _get_param_value(self, param): raise NotImplementedError -class Get(OWSParser): +class WPSGet(OWSParser): def _request_params(self): new_params = {} for param in self.request.params: - # new_params[param.lower()] = self.request.params.getone(param) new_params[param.lower()] = self.request.params[param].lower() return new_params def __init__(self, request): - super(Get, self).__init__(request) + super(WPSGet, self).__init__(request) self.all_params = self._request_params() def _get_param_value(self, param): @@ -65,10 +87,10 @@ def lxml_strip_ns(tree): node.tag = node.tag.split('}', 1)[1] -class Post(OWSParser): +class WPSPost(OWSParser): def __init__(self, request): - super(Post, self).__init__(request) + super(WPSPost, self).__init__(request) try: self.document = lxml.etree.fromstring(self.request.body) lxml_strip_ns(self.document) @@ -82,3 +104,9 @@ def _get_param_value(self, param): return self.document.tag.lower() else: return None + + +class JSONParser(OWSParser): + def _get_param_value(self, param): + param = param or '' # in case None + return self.params.get(param.lower(), '').lower() diff --git a/magpie/services.py b/magpie/services.py index 42e5fec5e..03227fc42 100644 --- a/magpie/services.py +++ b/magpie/services.py @@ -2,14 +2,18 @@ from magpie.definitions.ziggurat_definitions import * from magpie.definitions.pyramid_definitions import EVERYONE, ALLOW from magpie.api.api_except import * +from magpie.owsrequest import * from magpie import models -from owsrequest import * +from typing import List, Dict class ServiceI(object): - permission_names = [] # global permissions allowed for the service (top-level resource) - params_expected = [] # derived services must have 'request' at least for 'permission_requested' method - resource_types_permissions = {} # dict of list for each corresponding allowed resource permissions + # required request parameters for the service + params_expected = [] # type: List[str] + # global permissions allowed for the service (top-level resource) + permission_names = [] # type: List[str] + # dict of list for each corresponding allowed resource permissions + resource_types_permissions = {} # type: Dict[str,List[str]] # make 'property' getter from derived classes class __metaclass__(type): @@ -36,7 +40,6 @@ def expand_acl(self, resource, user): if resource: for ace in resource.__acl__: self.acl.append(ace) - # Custom acl if user: permissions = ResourceService.perms_for_user(resource, user, db_session=self.request.db) @@ -154,7 +157,6 @@ def __acl__(self): # https://colibri.crim.ca/twitcher/ows/proxy/ncWMS2/wms?SERVICE=WMS&REQUEST=GetCapabilities&VERSION=1.3.0&DATASET=outputs/ouranos/subdaily/aet/pcp/aet_pcp_1961.nc if 'dataset' in self.parser.params.keys(): netcdf_file = self.parser.params['dataset'] - # replace output/ with birdhouse/ elif permission_requested == 'getmap': # https://colibri.crim.ca/ncWMS2/wms?SERVICE=WMS&VERSION=1.3.0&REQUEST=GetMap&FORMAT=image%2Fpng&TRANSPARENT=TRUE&ABOVEMAXCOLOR=extend&STYLES=default-scalar%2Fseq-Blues&LAYERS=outputs/ouranos/subdaily/aet/pcp/aet_pcp_1961.nc/PCP&EPSG=4326 From 48c727f2ccef72cb831ef0771dda1e1edc13b2e9 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Thu, 25 Oct 2018 17:59:54 -0400 Subject: [PATCH 13/13] add typing req --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 6cab63a28..d51efe632 100644 --- a/requirements.txt +++ b/requirements.txt @@ -33,3 +33,4 @@ threddsclient==0.3.4 humanize requests_file webtest +typing