From 4ff27a231f3020ef63e82bbec777f465bae800b4 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Thu, 24 Jan 2019 19:49:43 -0500 Subject: [PATCH 01/76] attempt db missing creation + format fixes + setup permissions auto-generate --- HISTORY.rst | 8 ++ env/.gitignore | 3 +- env/postgres.env.example | 2 +- magpie/__meta__.py | 2 +- magpie/alembic/env.py | 44 +++++-- magpie/api/management/user/user_utils.py | 3 + magpie/constants.py | 4 +- magpie/db.py | 21 +-- magpie/definitions/sqlalchemy_definitions.py | 1 + magpie/register.py | 129 +++++++++++++++---- permissions.cfg | 23 ++++ 11 files changed, 186 insertions(+), 54 deletions(-) create mode 100644 permissions.cfg diff --git a/HISTORY.rst b/HISTORY.rst index a05cfc489..d8337b301 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -3,6 +3,14 @@ History ======= +0.8.x +--------------------- + +* update MagpieAdapter to match process store changes +* provide user ID on API routes returning user info +* attempt db creation on first migration if not existing +* add permissions config to auto-generate user/group rules on startup + 0.7.x --------------------- diff --git a/env/.gitignore b/env/.gitignore index 046a93e2f..6b6a58755 100644 --- a/env/.gitignore +++ b/env/.gitignore @@ -1,3 +1,2 @@ -magpie.env -postgres.env +*.env *.pem diff --git a/env/postgres.env.example b/env/postgres.env.example index cc4b8ff65..7fad7b855 100644 --- a/env/postgres.env.example +++ b/env/postgres.env.example @@ -2,4 +2,4 @@ MAGPIE_POSTGRES_USER=postgres MAGPIE_POSTGRES_PASSWORD=qwerty MAGPIE_POSTGRES_HOST=postgres MAGPIE_POSTGRES_PORT=5432 -MAGPIE_POSTGRES_DB=magpiedb +MAGPIE_POSTGRES_DB=magpie diff --git a/magpie/__meta__.py b/magpie/__meta__.py index 1c8e30680..1d9801098 100644 --- a/magpie/__meta__.py +++ b/magpie/__meta__.py @@ -2,7 +2,7 @@ General meta information on the magpie package. """ -__version__ = '0.8.2' +__version__ = '0.8.3' __author__ = "Francois-Xavier Derue, Francis Charette-Migneault" __maintainer__ = "Francis Charette-Migneault" __email__ = 'francis.charette-migneault@crim.ca' diff --git a/magpie/alembic/env.py b/magpie/alembic/env.py index 3deea8ef6..ad4de2e92 100644 --- a/magpie/alembic/env.py +++ b/magpie/alembic/env.py @@ -1,9 +1,12 @@ from __future__ import with_statement from alembic import context -from magpie.definitions.sqlalchemy_definitions import engine_from_config, pool, create_engine -from magpie.db import get_db_url from logging.config import fileConfig from sqlalchemy.schema import MetaData +from sqlalchemy.engine import create_engine +from sqlalchemy.exc import OperationalError +from magpie.db import get_db_url +from magpie.constants import get_constant + # this is the Alembic Config object, which provides # access to the values within the .ini file in use. @@ -14,11 +17,6 @@ fileConfig(config.config_file_name) # add your model's MetaData object here -# for 'autogenerate' support -# from myapp import mymodel -# target_metadata = mymodel.Base.metadata - - target_metadata = MetaData(naming_convention={ "ix": 'ix_%(column_0_label)s', "uq": "uq_%(table_name)s_%(column_0_name)s", @@ -40,13 +38,12 @@ def run_migrations_offline(): This configures the context with just a URL and not an Engine, though an Engine is acceptable here as well. By skipping the Engine creation - we don't even need a DBAPI to be available. + we don't even need a DB-API to be available. Calls to context.execute() here emit the given string to the script output. """ - #url = config.get_main_option("sqlalchemy.url") url = get_db_url() context.configure( url=url, target_metadata=target_metadata, literal_binds=True) @@ -62,12 +59,31 @@ def run_migrations_online(): and associate a connection with the context. """ - #connectable = engine_from_config( - # config.get_section(config.config_ini_section), - # prefix='sqlalchemy.', - # poolclass=pool.NullPool) - connectable = create_engine(get_db_url()) + # test the connection, if database is missing try creating it + url = get_db_url() + try: + connectable = create_engine(url) + with connectable.connect(): + pass + except OperationalError as ex: + # see for details: + # https://stackoverflow.com/questions/6506578 + db_name = get_constant('MAGPIE_POSTGRES_DB') + if 'database "{}" does not exist'.format(db_name) not in ex.message: + raise # any error is OperationalError, so validate only missing db error + db_default_postgres_url = get_db_url( + username='postgres', # only postgres user can connect to default 'postgres' db + password='', + db_host=get_constant('MAGPIE_POSTGRES_HOST'), + db_port=get_constant('MAGPIE_POSTGRES_PORT'), + db_name='postgres' + ) + connectable = create_engine(db_default_postgres_url) + with connectable.connect() as connection: + connection.execute("commit") # end initial transaction + connection.execute("create database {}".format(db_name)) + # retry connection and run migration with connectable.connect() as connection: context.configure( connection=connection, diff --git a/magpie/api/management/user/user_utils.py b/magpie/api/management/user/user_utils.py index b560e3ea7..8de7eda88 100644 --- a/magpie/api/management/user/user_utils.py +++ b/magpie/api/management/user/user_utils.py @@ -26,6 +26,7 @@ def create_user(user_name, password, email, group_name, db_session): msgOnFail=User_Check_ConflictResponseSchema.description) # Create user with specified name and group to assign + # noinspection PyArgumentList new_user = models.User(user_name=user_name, email=email) if password: UserService.set_password(new_user, password) @@ -37,6 +38,7 @@ def create_user(user_name, password, email, group_name, db_session): httpError=HTTPForbidden, msgOnFail=UserNew_POST_ForbiddenResponseSchema.description) # Assign user to group + # noinspection PyArgumentList group_entry = models.UserGroup(group_id=group_check.id, user_id=new_user.id) evaluate_call(lambda: db.add(group_entry), fallback=lambda: db.rollback(), httpError=HTTPForbidden, msgOnFail=UserGroup_GET_ForbiddenResponseSchema.description) @@ -54,6 +56,7 @@ def create_user_resource_permission(permission_name, resource, user_id, db_sessi content={u'resource_id': resource_id, u'user_id': user_id, u'permission_name': permission_name}, msgOnFail=UserResourcePermissions_POST_ConflictResponseSchema.description) + # noinspection PyArgumentList new_perm = models.UserResourcePermission(resource_id=resource_id, user_id=user_id, perm_name=permission_name) verify_param(new_perm, notNone=True, httpError=HTTPNotAcceptable, content={u'resource_id': resource_id, u'user_id': user_id}, diff --git a/magpie/constants.py b/magpie/constants.py index 6365e43d5..cf4b8313f 100644 --- a/magpie/constants.py +++ b/magpie/constants.py @@ -71,13 +71,13 @@ MAGPIE_POSTGRES_PASSWORD = os.getenv('MAGPIE_POSTGRES_PASSWORD', 'qwerty') MAGPIE_POSTGRES_HOST = os.getenv('MAGPIE_POSTGRES_HOST', 'postgres') MAGPIE_POSTGRES_PORT = int(os.getenv('MAGPIE_POSTGRES_PORT', 5432)) -MAGPIE_POSTGRES_DB = os.getenv('MAGPIE_POSTGRES_DB', 'magpiedb') +MAGPIE_POSTGRES_DB = os.getenv('MAGPIE_POSTGRES_DB', 'magpie') # =========================== # other constants # =========================== MAGPIE_ADMIN_PERMISSION = 'admin' -#MAGPIE_ADMIN_PERMISSION = NO_PERMISSION_REQUIRED +# MAGPIE_ADMIN_PERMISSION = NO_PERMISSION_REQUIRED MAGPIE_LOGGED_USER = 'current' MAGPIE_DEFAULT_PROVIDER = 'ziggurat' diff --git a/magpie/db.py b/magpie/db.py index d8bf4ed1e..89ad0995c 100644 --- a/magpie/db.py +++ b/magpie/db.py @@ -9,24 +9,26 @@ import inspect import zope.sqlalchemy import logging -logger = logging.getLogger(__name__) # import or define all models here to ensure they are attached to the # Base.metadata prior to any initialization routines from magpie import models + +logger = logging.getLogger(__name__) + # run configure_mappers after defining all of the models to ensure # all relationships can be setup configure_mappers() -def get_db_url(): +def get_db_url(username=None, password=None, db_host=None, db_port=None, db_name=None): return "postgresql://%s:%s@%s:%s/%s" % ( - constants.MAGPIE_POSTGRES_USER, - constants.MAGPIE_POSTGRES_PASSWORD, - constants.MAGPIE_POSTGRES_HOST, - constants.MAGPIE_POSTGRES_PORT, - constants.MAGPIE_POSTGRES_DB, + username if username is not None else constants.MAGPIE_POSTGRES_USER, + password if password is not None else constants.MAGPIE_POSTGRES_PASSWORD, + db_host if db_host is not None else constants.MAGPIE_POSTGRES_HOST, + db_port if db_port is not None else constants.MAGPIE_POSTGRES_PORT, + db_name if db_name is not None else constants.MAGPIE_POSTGRES_DB, ) @@ -59,7 +61,7 @@ def get_tm_session(session_factory, transaction_manager): engine = get_engine(settings) session_factory = get_session_factory(engine) with transaction.manager: - dbsession = get_tm_session(session_factory, transaction.manager) + db_session = get_tm_session(session_factory, transaction.manager) """ db_session = session_factory() @@ -104,11 +106,12 @@ def is_database_ready(): for name, obj in inspect.getmembers(models): if inspect.isclass(obj): + # noinspection PyBroadException try: curr_table_name = obj.__tablename__ if curr_table_name not in table_names: return False - except: + except Exception: continue return True diff --git a/magpie/definitions/sqlalchemy_definitions.py b/magpie/definitions/sqlalchemy_definitions.py index 2f603774d..2eeb9277a 100644 --- a/magpie/definitions/sqlalchemy_definitions.py +++ b/magpie/definitions/sqlalchemy_definitions.py @@ -4,6 +4,7 @@ from sqlalchemy.engine.reflection import Inspector from sqlalchemy.ext.declarative import declarative_base, declared_attr from sqlalchemy.orm import relationship, sessionmaker, configure_mappers +from sqlalchemy.orm.session import Session from sqlalchemy.sql import select from sqlalchemy import engine_from_config, pool, create_engine from sqlalchemy import exc as sa_exc diff --git a/magpie/register.py b/magpie/register.py index 90faab5ee..3d34d2a67 100644 --- a/magpie/register.py +++ b/magpie/register.py @@ -1,16 +1,19 @@ +from magpie.api.api_rest_schemas import SigninAPI, SignoutAPI, ServicesAPI from magpie.services import service_type_dict from magpie.common import make_dirs, print_log, raise_log, bool2str from magpie.constants import get_constant from magpie import models from magpie.definitions.ziggurat_definitions import UserService, UserResourcePermissionService +from magpie.definitions.sqlalchemy_definitions import Session +from typing import AnyStr, Dict, List, Optional, Tuple, TYPE_CHECKING import os import time import yaml import subprocess import requests import transaction +import warnings import logging - LOGGER = logging.getLogger(__name__) LOGIN_ATTEMPT = 10 # max attempts for login @@ -25,6 +28,9 @@ SERVICES_PHOENIX = 'PHOENIX' SERVICES_PHOENIX_ALLOWED = ['wps'] +if TYPE_CHECKING: + ConfigDict = Dict[AnyStr, Dict[AnyStr, AnyStr]] + def login_loop(login_url, cookies_file, data=None, message='Login response'): make_dirs(cookies_file) @@ -46,9 +52,15 @@ def login_loop(login_url, cookies_file, data=None, message='Login response'): def request_curl(url, cookie_jar=None, cookies=None, form_params=None, msg='Response'): + # type: (AnyStr, Optional[AnyStr], Optional[AnyStr], Optional[AnyStr], Optional[AnyStr]) -> Tuple[int, int] + """Executes a request using cURL. + + :returns: tuple of the returned system command code and the response http code + """ + # arg -k allows to ignore insecure SSL errors, ie: access 'https' page not configured for it - ###curl_cmd = 'curl -k -L -s -o /dev/null -w "{msg_out} : %{{http_code}}\\n" {params} {url}' - ###curl_cmd = curl_cmd.format(msg_out=msg, params=params, url=url) + # curl_cmd = 'curl -k -L -s -o /dev/null -w "{msg_out} : %{{http_code}}\\n" {params} {url}' + # curl_cmd = curl_cmd.format(msg_out=msg, params=params, url=url) msg_sep = msg + ": " params = ['curl', '-k', '-L', '-s', '-o', '/dev/null', '-w', msg_sep + '%{http_code}'] if cookie_jar is not None and cookies is not None: @@ -61,8 +73,8 @@ def request_curl(url, cookie_jar=None, cookies=None, form_params=None, msg='Resp params.extend(['--data', form_params]) params.extend([url]) curl_out = subprocess.Popen(params, stdout=subprocess.PIPE) - curl_msg = curl_out.communicate()[0] - curl_err = curl_out.returncode + curl_msg = curl_out.communicate()[0] # type: AnyStr + curl_err = curl_out.returncode # type: int http_code = int(curl_msg.split(msg_sep)[1]) print_log("[{url}] {response}".format(response=curl_msg, url=url)) return curl_err, http_code @@ -91,8 +103,8 @@ def phoenix_login(cookies): def phoenix_login_check(cookies): """ Since Phoenix always return 200, even on invalid login, 'hack' check unauthorized access. - :param cookies: - :return: + :param cookies: temporary cookies file storage used for login with `phoenix_login`. + :return: status indicating if login access was granted with defined credentials. """ no_access_error = "Unauthorized: Services failed permission check" svc_url = get_phoenix_url() + '/services' @@ -103,6 +115,11 @@ def phoenix_login_check(cookies): def phoenix_remove_services(): + # type: (...) -> bool + """Removes the Phoenix services using temporary cookies retrieved from login with defined `PHOENIX` constants. + + :returns: success status of the procedure. + """ phoenix_cookies = os.path.join(LOGIN_TMP_DIR, 'login_cookie_phoenix') error = 0 try: @@ -139,10 +156,8 @@ def phoenix_register_services(services_dict, allowed_service_types=None): filtered_services_dict[svc]['type'] = filtered_services_dict[svc]['type'].upper() # Register services - phoenix_url = get_phoenix_url() - register_service_url = phoenix_url + '/services/register' - success, statuses = register_services(register_service_url, filtered_services_dict, - phoenix_cookies, 'Phoenix register service', SERVICES_PHOENIX) + success, statuses = register_services(SERVICES_PHOENIX, filtered_services_dict, + phoenix_cookies, 'Phoenix register service') except Exception as e: print_log("Exception during phoenix register services: [" + repr(e) + "]") finally: @@ -179,14 +194,26 @@ def get_twitcher_protected_service_url(magpie_service_name, hostname=None): return "{0}/{1}".format(twitcher_proxy_url, magpie_service_name) -def register_services(register_service_url, services_dict, cookies, - message='Register response', where=SERVICES_MAGPIE): +def register_services(where, # type: Optional[AnyStr] + services_dict, # type: Dict[AnyStr, Dict[AnyStr, AnyStr]] + cookies, # type: AnyStr + message='Register response', # type: AnyStr + ): # type: (...) -> Tuple[bool, Dict[AnyStr, int]] + """ + Registers services on desired location using provided configurations and access cookies. + + :returns: tuple of overall success and individual http response of each service registration. + """ success = True + svc_url = None statuses = dict() + register_service_url = None if where == SERVICES_MAGPIE: svc_url_tag = 'service_url' + get_magpie_url() + ServicesAPI.path elif where == SERVICES_PHOENIX: svc_url_tag = 'url' + register_service_url = get_phoenix_url() + '/services/register' else: raise ValueError("Unknown location for service registration", where) for service_name in services_dict: @@ -219,6 +246,7 @@ def sync_services_phoenix(services_object_dict, services_as_dicts=False): """ Syncs Magpie services by pushing updates to Phoenix. Services must be one of types specified in SERVICES_PHOENIX_ALLOWED. + :param services_object_dict: dictionary of {svc-name: models.Service} objects containing each service's information :param services_as_dicts: alternatively specify `services_object_dict` as dict of {svc-name: {service-info}} where {service-info} = {'public_url': , 'service_name': , 'service_type': } @@ -289,6 +317,8 @@ def magpie_add_register_services_perms(services, statuses, curl_cookies, request def magpie_update_services_conflict(conflict_services, services_dict, request_cookies): + # type: (List[AnyStr], ConfigDict, Dict[AnyStr, AnyStr]) -> Dict[AnyStr, int] + """Resolve conflicting services by name during registration by updating them only if pointing to different URL.""" magpie_url = get_magpie_url() statuses = dict() for svc_name in conflict_services: @@ -306,16 +336,29 @@ def magpie_update_services_conflict(conflict_services, services_dict, request_co return statuses -def magpie_register_services(services_dict, push_to_phoenix, user, password, provider, +def magpie_register_services(services_dict, push_to_phoenix, username, password, provider, force_update=False, disable_getcapabilities=False): + # type: (ConfigDict, bool, AnyStr, AnyStr, AnyStr, Optional[bool], Optional[bool]) -> bool + """ + Registers magpie services using the provided services configuration. + + :param services_dict: services configuration definition. + :param push_to_phoenix: push registered Magpie services to Phoenix for synced configurations. + :param username: login username to use to obtain permissions for services registration. + :param password: login password to use to obtain permissions for services registration. + :param provider: login provider to use to obtain permissions for services registration. + :param force_update: override existing services matched by name + :param disable_getcapabilities: do not execute 'GetCapabilities' validation for applicable services. + :return: successful operation status + """ magpie_url = get_magpie_url() curl_cookies = os.path.join(LOGIN_TMP_DIR, 'login_cookie_magpie') session = requests.Session() success = False try: # Need to login first as admin - login_url = magpie_url + '/signin' - login_data = {'user_name': user, 'password': password, 'provider_name': provider} + login_url = magpie_url + SigninAPI.path + login_data = {'user_name': username, 'password': password, 'provider_name': provider} login_loop(login_url, curl_cookies, login_data, 'Magpie login response') login_resp = session.post(login_url, data=login_data) if login_resp.status_code != 200: @@ -324,9 +367,8 @@ def magpie_register_services(services_dict, push_to_phoenix, user, password, pro # Register services # Magpie will not overwrite existing services by default, 409 Conflict instead of 201 Created - register_service_url = magpie_url + '/services' - success, statuses_register = register_services(register_service_url, services_dict, curl_cookies, - 'Magpie register service', SERVICES_MAGPIE) + success, statuses_register = register_services(SERVICES_MAGPIE, services_dict, + curl_cookies, 'Magpie register service') # Service URL update if conflicting and requested if force_update and not success: conflict_services = [svc_name for svc_name, http_code in statuses_register.items() if http_code == 409] @@ -337,7 +379,7 @@ def magpie_register_services(services_dict, push_to_phoenix, user, password, pro # Phoenix doesn't register the service if it cannot be checked with this request magpie_add_register_services_perms(services_dict, statuses_register, curl_cookies, request_cookies, disable_getcapabilities) - session.get(magpie_url + '/signout') + session.get(magpie_url + SignoutAPI.path) # Push updated services to Phoenix if push_to_phoenix: @@ -376,6 +418,7 @@ def magpie_register_services_with_db_session(services_dict, db_session, push_to_ print_log("Skipping service [{svc}] (conflict)" .format(svc=svc_name)) else: print_log("Adding service [{svc}]".format(svc=svc_name)) + # noinspection PyArgumentList svc = models.Service(resource_name=svc_name, resource_type=models.Service.resource_type_name, url=svc_new_url, @@ -393,6 +436,7 @@ def magpie_register_services_with_db_session(services_dict, db_session, push_to_ ) if svc_perm_getcapabilities is None: print_log("Adding 'getcapabilities' permission to anonymous user") + # noinspection PyArgumentList svc_perm_getcapabilities = models.UserResourcePermission( user_id=anonymous_user.id, perm_name='getcapabilities', resource_id=svc.resource_id ) @@ -406,13 +450,28 @@ def magpie_register_services_with_db_session(services_dict, db_session, push_to_ return True +def _load_config(path, section): + try: + cfg = yaml.load(open(path, 'r')) + return cfg[section] + except KeyError as ex: + raise_log("Config file section [{!s}] not found.".format(section), exception=type(ex)) + except Exception as ex: + raise_log("Invalid config file [{!r}]".format(ex), exception=type(ex)) + + def magpie_register_services_from_config(service_config_file_path, push_to_phoenix=False, force_update=False, disable_getcapabilities=False, db_session=None): - try: - services_cfg = yaml.load(open(service_config_file_path, 'r')) - services = services_cfg['providers'] - except Exception as e: - raise_log("Bad service file + [" + repr(e) + "]", exception=type(e)) + # type: (AnyStr, Optional[bool], Optional[bool], Optional[bool], Optional[Session]) -> None + """ + Registers Magpie services from a `providers.cfg` file. + Uses the provided DB session to directly update service definitions, or uses API request routes as admin. + Optionally pushes updates to Phoenix. + """ + services = _load_config(service_config_file_path, 'providers') + if not services: + warnings.warn("Services configuration are empty.", UserWarning) + return # register services using API POSTs if db_session is None: @@ -426,3 +485,23 @@ def magpie_register_services_from_config(service_config_file_path, push_to_phoen magpie_register_services_with_db_session(services, db_session, push_to_phoenix=push_to_phoenix, force_update=force_update, update_getcapabilities_permissions=not disable_getcapabilities) + + +def magpie_register_permissions_from_config(permissions_config_file_path): + permissions = _load_config(permissions_config_file_path, 'permissions') + if not permissions: + warnings.warn("Permissions configuration are empty.", UserWarning) + return + + admin_usr = get_constant('MAGPIE_ADMIN_USER') + admin_pwd = get_constant('MAGPIE_ADMIN_PASSWORD') + body = {'user_name': admin_usr, 'password': admin_pwd} + resp = requests.get(get_magpie_url() + SigninAPI.path, json=body) + if not resp.status_code == 200: + raise_log("Cannot register Magpie permissions without proper credentials.") + + for perm in permissions: + if not isinstance(perm, dict) or not all(f in perm for f in ['permission', 'service', 'resource']): + warnings.warn("Invalid permission format for [{!s}].".format(perm), UserWarning) + continue + diff --git a/permissions.cfg b/permissions.cfg new file mode 100644 index 000000000..6f0446e86 --- /dev/null +++ b/permissions.cfg @@ -0,0 +1,23 @@ +# For each permission to be updated +# +# service: service name to receive the permission (directly on it if resource not specified) +# resource (optional): tree path of the service's resource (ex: /res1/sub-res2/sub-sub-res3) +# user and/or group: user/group to apply the permission on +# permission: name of the permission to be applied +# action: one of [create, delete] (default: create) +# +# Default behaviour is to create missing resources if supported by the service, then apply permissions. +# If the service is missing, corresponding permissions are ignored and not updated. +# If permission configuration is already satisfied, it is left as is. +# + +permissions: + - service: api + resource: /api + permission: read + user: + action: create + - service: test + resource: + permission: getcapabilities + group: From 50498c6d47ab316934e5c87f666dc29b83fa4f73 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Fri, 25 Jan 2019 19:01:27 -0500 Subject: [PATCH 02/76] pep8 & swagger fixes --- HISTORY.rst | 1 + magpie/api/__init__.py | 2 +- magpie/api/api_generic.py | 17 +- magpie/api/api_rest_schemas.py | 139 ++++++++++----- magpie/api/login/__init__.py | 10 +- magpie/api/management/group/group_views.py | 6 +- .../api/management/resource/resource_utils.py | 7 +- .../api/management/resource/resource_views.py | 2 +- .../api/management/service/service_views.py | 7 +- magpie/definitions/pyramid_definitions.py | 1 + magpie/models.py | 8 +- magpie/permissions.py | 19 +++ magpie/register.py | 159 ++++++++++++++++-- magpie/security.py | 4 +- magpie/services.py | 16 +- permissions.cfg | 18 +- 16 files changed, 319 insertions(+), 97 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index d8337b301..6ef0c1b8e 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -10,6 +10,7 @@ History * provide user ID on API routes returning user info * attempt db creation on first migration if not existing * add permissions config to auto-generate user/group rules on startup +* fix many invalid or erroneous swagger specifications 0.7.x --------------------- diff --git a/magpie/api/__init__.py b/magpie/api/__init__.py index 9a640f666..7da589f11 100644 --- a/magpie/api/__init__.py +++ b/magpie/api/__init__.py @@ -14,4 +14,4 @@ def includeme(config): config.add_notfound_view(not_found) config.add_exception_view(internal_server_error) config.add_forbidden_view(unauthorized_access) - #config.scan() + # config.scan() diff --git a/magpie/api/api_generic.py b/magpie/api/api_generic.py index 94b34f446..b21466b5b 100644 --- a/magpie/api/api_generic.py +++ b/magpie/api/api_generic.py @@ -1,27 +1,28 @@ -from magpie.api.api_except import * -from magpie.api.api_rest_schemas import * +from magpie.definitions.pyramid_definitions import HTTPUnauthorized, HTTPNotFound, HTTPInternalServerError +from magpie.api.api_except import raise_http, HTTPServerError +from magpie.api import api_rest_schemas as s -#@notfound_view_config() +# @notfound_view_config() def not_found(request): - content = get_request_info(request, default_msg=NotFoundResponseSchema.description) + content = get_request_info(request, default_msg=s.NotFoundResponseSchema.description) return raise_http(nothrow=True, httpError=HTTPNotFound, contentType='application/json', detail=content['detail'], content=content) -#@exception_view_config() +# @exception_view_config() def internal_server_error(request): - content = get_request_info(request, default_msg=InternalServerErrorResponseSchema.description) + content = get_request_info(request, default_msg=s.InternalServerErrorResponseSchema.description) return raise_http(nothrow=True, httpError=HTTPInternalServerError, contentType='application/json', detail=content['detail'], content=content) -#@forbidden_view_config() +# @forbidden_view_config() def unauthorized_access(request): # if not overridden, default is HTTPForbidden [403], which is for a slightly different situation # this better reflects the HTTPUnauthorized [401] user access with specified AuthZ headers # [http://www.restapitutorial.com/httpstatuscodes.html] - content = get_request_info(request, default_msg=UnauthorizedResponseSchema.description) + content = get_request_info(request, default_msg=s.UnauthorizedResponseSchema.description) return raise_http(nothrow=True, httpError=HTTPUnauthorized, contentType='application/json', detail=content['detail'], content=content) diff --git a/magpie/api/api_rest_schemas.py b/magpie/api/api_rest_schemas.py index 40837dd63..cde413990 100644 --- a/magpie/api/api_rest_schemas.py +++ b/magpie/api/api_rest_schemas.py @@ -1,6 +1,7 @@ from magpie.definitions.cornice_definitions import * from magpie.definitions.pyramid_definitions import * from magpie.constants import get_constant +# from magpie.security import get_provider_names from magpie import __meta__ import six @@ -247,10 +248,32 @@ def service_api_route_info(service_api): # Common path parameters -GroupNameParameter = colander.SchemaNode(colander.String(), description="Registered user group.") -UserNameParameter = colander.SchemaNode(colander.String(), description="Registered local user.") -ProviderNameParameter = colander.SchemaNode(colander.String(), description="External identity provider.", - validator=colander.OneOf([])) +GroupNameParameter = colander.SchemaNode( + colander.String(), + description="Registered user group.", + example="users",) +UserNameParameter = colander.SchemaNode( + colander.String(), + description="Registered local user.", + example="toto",) +ProviderNameParameter = colander.SchemaNode( + colander.String(), + description="External identity provider.", + example="DKRZ", + # validator=colander.OneOf(get_provider_names()) +) +PermissionNameParameter = colander.SchemaNode( + colander.String(), + description="Permissions applicable to the service/resource.", + example="read",) +ResourceIdParameter = colander.SchemaNode( + colander.String(), + description="Registered resource ID.", + example="123") +ServiceNameParameter = colander.SchemaNode( + colander.String(), + description="Registered service name.", + example="my-wps") class HeaderResponseSchema(colander.MappingSchema): @@ -290,8 +313,8 @@ class BaseResponseBodySchema(colander.MappingSchema): __code = None __desc = None - def __init__(self, code, description): - super(BaseResponseBodySchema, self).__init__() + def __init__(self, code, description, **kw): + super(BaseResponseBodySchema, self).__init__(**kw) assert isinstance(code, int) assert isinstance(description, six.string_types) self.__code = code @@ -414,19 +437,11 @@ class ResourceTypesListSchema(colander.SequenceSchema): class GroupNamesListSchema(colander.SequenceSchema): - group_name = colander.SchemaNode( - colander.String(), - description="List of groups depending on context.", - example="administrators" - ) + group_name = GroupNameParameter class UserNamesListSchema(colander.SequenceSchema): - user_name = colander.SchemaNode( - colander.String(), - description="Users registered in the db", - example="bob" - ) + user_name = UserNameParameter class PermissionListSchema(colander.SequenceSchema): @@ -438,10 +453,7 @@ class PermissionListSchema(colander.SequenceSchema): class UserBodySchema(colander.MappingSchema): - user_name = colander.SchemaNode( - colander.String(), - description="Name of the user.", - example="toto") + user_name = UserNameParameter email = colander.SchemaNode( colander.String(), description="Email of the user.", @@ -645,6 +657,7 @@ class Resource_PUT_RequestBodySchema(colander.MappingSchema): class Resource_PUT_RequestSchema(colander.MappingSchema): header = HeaderRequestSchema() body = Resource_PUT_RequestBodySchema() + resource_id = ResourceIdParameter class Resource_PUT_ResponseBodySchema(BaseResponseBodySchema): @@ -690,6 +703,7 @@ class Resource_DELETE_RequestBodySchema(colander.MappingSchema): class Resource_DELETE_RequestSchema(colander.MappingSchema): header = HeaderRequestSchema() body = Resource_DELETE_RequestBodySchema() + resource_id = ResourceIdParameter class Resource_DELETE_OkResponseSchema(colander.MappingSchema): @@ -710,7 +724,7 @@ class Resources_GET_OkResponseSchema(colander.MappingSchema): body = Resources_ResponseBodySchema(code=HTTPOk.code, description=description) -class Resources_POST_BodySchema(colander.MappingSchema): +class Resources_POST_RequestBodySchema(colander.MappingSchema): resource_name = colander.SchemaNode( colander.String(), description="Name of the resource to create" @@ -731,9 +745,9 @@ class Resources_POST_BodySchema(colander.MappingSchema): ) -class Resources_POST_RequestBodySchema(colander.MappingSchema): +class Resources_POST_RequestSchema(colander.MappingSchema): header = HeaderRequestSchema() - body = Resources_POST_BodySchema() + body = Resources_POST_RequestBodySchema() class Resource_POST_ResponseBodySchema(BaseResponseBodySchema): @@ -1007,8 +1021,10 @@ class Service_PUT_ConflictResponseSchema(colander.MappingSchema): body = Service_FailureBodyResponseSchema(code=HTTPConflict.code, description=description) -# delete service use same method as direct resource delete -Service_DELETE_RequestSchema = Resource_DELETE_RequestSchema +class Service_DELETE_RequestSchema(colander.MappingSchema): + header = HeaderRequestSchema() + body = Resource_DELETE_RequestBodySchema() + service_name = ServiceNameParameter class Service_DELETE_OkResponseSchema(colander.MappingSchema): @@ -1044,8 +1060,10 @@ class ServicePermissions_GET_NotAcceptableResponseSchema(colander.MappingSchema) # create service's resource use same method as direct resource create -ServiceResources_POST_BodySchema = Resources_POST_BodySchema -ServiceResources_POST_RequestBodySchema = Resources_POST_RequestBodySchema +class ServiceResources_POST_RequestSchema(Resources_POST_RequestSchema): + service_name = ServiceNameParameter + + ServiceResources_POST_CreatedResponseSchema = Resources_POST_CreatedResponseSchema ServiceResources_POST_BadRequestResponseSchema = Resources_POST_BadRequestResponseSchema ServiceResources_POST_ForbiddenResponseSchema = Resources_POST_ForbiddenResponseSchema @@ -1054,7 +1072,10 @@ class ServicePermissions_GET_NotAcceptableResponseSchema(colander.MappingSchema) # delete service's resource use same method as direct resource delete -ServiceResource_DELETE_RequestSchema = Resource_DELETE_RequestSchema +class ServiceResource_DELETE_RequestSchema(Resource_DELETE_RequestSchema): + service_name = ServiceNameParameter + + ServiceResource_DELETE_ForbiddenResponseSchema = Resource_DELETE_ForbiddenResponseSchema ServiceResource_DELETE_OkResponseSchema = Resource_DELETE_OkResponseSchema @@ -1363,6 +1384,7 @@ class UserGroups_POST_RequestBodySchema(colander.MappingSchema): class UserGroups_POST_RequestSchema(colander.MappingSchema): header = HeaderRequestSchema() body = UserGroups_POST_RequestBodySchema() + user_name = UserNameParameter class UserGroups_POST_ResponseBodySchema(BaseResponseBodySchema): @@ -1528,12 +1550,6 @@ class UserResourcePermissions_GET_NotFoundResponseSchema(colander.MappingSchema) class UserResourcePermissions_POST_RequestBodySchema(colander.MappingSchema): - resource_id = colander.SchemaNode( - colander.Integer(), - description="resource_id of the created user-resource-permission reference.") - user_id = colander.SchemaNode( - colander.Integer(), - description="user_id of the created user-resource-permission reference.") permission_name = colander.SchemaNode( colander.String(), description="permission_name of the created user-resource-permission reference.") @@ -1542,6 +1558,8 @@ class UserResourcePermissions_POST_RequestBodySchema(colander.MappingSchema): class UserResourcePermissions_POST_RequestSchema(colander.MappingSchema): header = HeaderRequestSchema() body = UserResourcePermissions_POST_RequestBodySchema() + resource_id = ResourceIdParameter + user_name = UserNameParameter class UserResourcePermissions_POST_ResponseBodySchema(BaseResponseBodySchema): @@ -1604,7 +1622,11 @@ class UserResourcePermissions_POST_ConflictResponseSchema(colander.MappingSchema class UserResourcePermission_DELETE_RequestSchema(colander.MappingSchema): + header = HeaderRequestSchema() body = colander.MappingSchema(default={}) + user_name = UserNameParameter + resource_id = ResourceIdParameter + permission_name = PermissionNameParameter class UserResourcePermissions_DELETE_OkResponseSchema(colander.MappingSchema): @@ -1636,6 +1658,8 @@ class UserServiceResources_GET_QuerySchema(colander.MappingSchema): class UserServiceResources_GET_RequestSchema(colander.MappingSchema): header = HeaderRequestSchema() querystring = UserServiceResources_GET_QuerySchema() + user_name = UserNameParameter + service_name = ServiceNameParameter class UserServicePermissions_POST_RequestBodySchema(colander.MappingSchema): @@ -1645,11 +1669,16 @@ class UserServicePermissions_POST_RequestBodySchema(colander.MappingSchema): class UserServicePermissions_POST_RequestSchema(colander.MappingSchema): header = HeaderRequestSchema() body = UserServicePermissions_POST_RequestBodySchema() + user_name = UserNameParameter + service_name = ServiceNameParameter class UserServicePermission_DELETE_RequestSchema(colander.MappingSchema): header = HeaderRequestSchema() body = colander.MappingSchema(default={}) + user_name = UserNameParameter + service_name = ServiceNameParameter + permission_name = PermissionNameParameter class UserServices_GET_QuerySchema(colander.MappingSchema): @@ -1663,6 +1692,7 @@ class UserServices_GET_QuerySchema(colander.MappingSchema): class UserServices_GET_RequestSchema(colander.MappingSchema): header = HeaderRequestSchema() querystring = UserServices_GET_QuerySchema() + user_name = UserNameParameter class UserServices_GET_ResponseBodySchema(BaseResponseBodySchema): @@ -1682,6 +1712,8 @@ class UserServicePermissions_GET_QuerySchema(colander.MappingSchema): class UserServicePermissions_GET_RequestSchema(colander.MappingSchema): header = HeaderRequestSchema() querystring = UserServicePermissions_GET_QuerySchema() + user_name = UserNameParameter + service_name = ServiceNameParameter class UserServicePermissions_GET_ResponseBodySchema(BaseResponseBodySchema): @@ -1793,10 +1825,16 @@ class Group_GET_NotFoundResponseSchema(colander.MappingSchema): body = ErrorResponseBodySchema(code=HTTPNotFound.code, description=description) -class Group_PUT_RequestSchema(colander.MappingSchema): +class Group_PUT_RequestBodySchema(colander.MappingSchema): group_name = colander.SchemaNode(colander.String(), description="New name to apply to the group.") +class Group_PUT_RequestSchema(colander.MappingSchema): + header = HeaderRequestSchema() + body = Group_PUT_RequestBodySchema() + group_name = GroupNameParameter + + class Group_PUT_OkResponseSchema(colander.MappingSchema): description = "Update group successful." header = HeaderResponseSchema() @@ -1831,6 +1869,7 @@ class Group_PUT_ConflictResponseSchema(colander.MappingSchema): class Group_DELETE_RequestSchema(colander.MappingSchema): header = HeaderRequestSchema() body = colander.MappingSchema(default={}) + group_name = GroupNameParameter class Group_DELETE_OkResponseSchema(colander.MappingSchema): @@ -1900,11 +1939,22 @@ class GroupServicePermissions_GET_InternalServerErrorResponseSchema(colander.Map code=HTTPInternalServerError.code, description=description) -class GroupServicePermissions_POST_RequestSchema(colander.MappingSchema): +class GroupServicePermissions_POST_RequestBodySchema(colander.MappingSchema): permission_name = colander.SchemaNode(colander.String(), description="Name of the permission to create.") -GroupResourcePermissions_POST_RequestSchema = GroupServicePermissions_POST_RequestSchema +class GroupServicePermissions_POST_RequestSchema(colander.MappingSchema): + header = HeaderRequestSchema() + body = GroupServicePermissions_POST_RequestBodySchema() + group_name = GroupNameParameter + service_name = ServiceNameParameter + + +class GroupResourcePermissions_POST_RequestSchema(colander.MappingSchema): + header = HeaderRequestSchema() + body = GroupServicePermissions_POST_RequestBodySchema() + group_name = GroupNameParameter + resource_id = ResourceIdParameter class GroupResourcePermissions_POST_ResponseBodySchema(BaseResponseBodySchema): @@ -1946,6 +1996,9 @@ class GroupResourcePermissions_POST_ConflictResponseSchema(colander.MappingSchem class GroupResourcePermission_DELETE_RequestSchema(colander.MappingSchema): header = HeaderRequestSchema() body = colander.MappingSchema(default={}) + group_name = GroupNameParameter + resource_id = ResourceIdParameter + permission_name = PermissionNameParameter class GroupResourcesPermissions_InternalServerErrorResponseBodySchema(InternalServerErrorResponseBodySchema): @@ -2014,10 +2067,18 @@ class GroupServiceResources_GET_OkResponseSchema(colander.MappingSchema): body = GroupServiceResources_GET_ResponseBodySchema(code=HTTPOk.code, description=description) -class GroupServicePermission_DELETE_RequestSchema(colander.MappingSchema): +class GroupServicePermission_DELETE_RequestBodySchema(colander.MappingSchema): permission_name = colander.SchemaNode(colander.String(), description="Name of the permission to delete.") +class GroupServicePermission_DELETE_RequestSchema(colander.MappingSchema): + header = HeaderRequestSchema() + body = GroupServicePermission_DELETE_RequestBodySchema() + group_name = GroupNameParameter + service_name = ServiceNameParameter + permission_name = PermissionNameParameter + + class GroupServicePermission_DELETE_ResponseBodySchema(BaseResponseBodySchema): permission_name = colander.SchemaNode(colander.String(), description="Name of the permission requested.") resource = ResourceBodySchema() @@ -2102,8 +2163,8 @@ class ProviderSignin_GET_HeaderRequestSchema(HeaderRequestSchema): class ProviderSignin_GET_RequestSchema(colander.MappingSchema): - provider_name = ProviderNameParameter header = ProviderSignin_GET_HeaderRequestSchema() + provider_name = ProviderNameParameter class ProviderSignin_GET_FoundResponseBodySchema(BaseResponseBodySchema): diff --git a/magpie/api/login/__init__.py b/magpie/api/login/__init__.py index 901909743..f5063a089 100644 --- a/magpie/api/login/__init__.py +++ b/magpie/api/login/__init__.py @@ -1,4 +1,4 @@ -from magpie.api.api_rest_schemas import * +from magpie.api import api_rest_schemas as s import logging logger = logging.getLogger(__name__) @@ -6,8 +6,8 @@ def includeme(config): logger.info('Adding api login ...') # Add all the rest api routes - config.add_route(**service_api_route_info(SessionAPI)) - config.add_route(**service_api_route_info(SigninAPI)) - config.add_route(**service_api_route_info(ProvidersAPI)) - config.add_route(**service_api_route_info(ProviderSigninAPI)) + config.add_route(**s.service_api_route_info(s.SessionAPI)) + config.add_route(**s.service_api_route_info(s.SigninAPI)) + config.add_route(**s.service_api_route_info(s.ProvidersAPI)) + config.add_route(**s.service_api_route_info(s.ProviderSigninAPI)) config.scan() diff --git a/magpie/api/management/group/group_views.py b/magpie/api/management/group/group_views.py index 20b1c33f3..da26e6543 100644 --- a/magpie/api/management/group/group_views.py +++ b/magpie/api/management/group/group_views.py @@ -127,7 +127,7 @@ def create_group_service_permission(request): response_schemas=GroupServicePermission_DELETE_responses) @view_config(route_name=GroupServicePermissionAPI.name, request_method='DELETE') def delete_group_service_permission(request): - """Delete a permission from a specific resource for a group.""" + """Delete a permission from a specific service for a group.""" group = get_group_matchdict_checked(request) service = get_service_matchdict_checked(request) perm_name = get_permission_matchdict_checked(request, service) @@ -168,8 +168,8 @@ def create_group_resource_permission_view(request): return create_group_resource_permission(perm_name, resource, group, db_session=request.db) -@GroupResourcePermissionAPI.post(schema=GroupResourcePermission_DELETE_RequestSchema(), tags=[GroupsTag], - response_schemas=GroupResourcePermission_DELETE_responses) +@GroupResourcePermissionAPI.delete(schema=GroupResourcePermission_DELETE_RequestSchema(), tags=[GroupsTag], + response_schemas=GroupResourcePermission_DELETE_responses) @view_config(route_name=GroupResourcePermissionAPI.name, request_method='DELETE') def delete_group_resource_permission_view(request): """Delete a permission from a specific resource for a group.""" diff --git a/magpie/api/management/resource/resource_utils.py b/magpie/api/management/resource/resource_utils.py index 05195eecd..efd6856d9 100644 --- a/magpie/api/management/resource/resource_utils.py +++ b/magpie/api/management/resource/resource_utils.py @@ -6,6 +6,10 @@ from magpie.api.api_requests import * from magpie.api.api_except import verify_param, evaluate_call, raise_http, valid_http from magpie.api.management.resource.resource_formats import format_resource +from typing import AnyStr, Union, TYPE_CHECKING +if TYPE_CHECKING: + from magpie.definitions.pyramid_definitions import HTTPException + from magpie.definitions.sqlalchemy_definitions import Session def check_valid_service_resource_permission(permission_name, service_resource, db_session): @@ -125,11 +129,12 @@ def get_resource_root_service(resource, db_session): def create_resource(resource_name, resource_display_name, resource_type, parent_id, db_session): + # type: (AnyStr, Union[AnyStr, None], AnyStr, int, Session) -> HTTPException verify_param(resource_name, paramName=u'resource_name', notNone=True, notEmpty=True, httpError=HTTPBadRequest, msgOnFail="Invalid `resource_name` specified for child resource creation.") verify_param(resource_type, paramName=u'resource_type', notNone=True, notEmpty=True, httpError=HTTPBadRequest, msgOnFail="Invalid `resource_type` specified for child resource creation.") - verify_param(parent_id, paramName=u'parent_id', notNone=True, notEmpty=True, httpError=HTTPBadRequest, + verify_param(parent_id, paramName=u'parent_id', notNone=True, notEmpty=True, ofType=int, httpError=HTTPBadRequest, msgOnFail="Invalid `parent_id` specified for child resource creation.") parent_resource = evaluate_call(lambda: ResourceService.by_resource_id(parent_id, db_session=db_session), fallback=lambda: db_session.rollback(), httpError=HTTPNotFound, diff --git a/magpie/api/management/resource/resource_views.py b/magpie/api/management/resource/resource_views.py index 37ed4ac79..67e5f27ef 100644 --- a/magpie/api/management/resource/resource_views.py +++ b/magpie/api/management/resource/resource_views.py @@ -37,7 +37,7 @@ def get_resource_view(request): content={resource.resource_id: res_json}) -@ResourcesAPI.post(schema=Resources_POST_RequestBodySchema, tags=[ResourcesTag], +@ResourcesAPI.post(schema=Resources_POST_RequestSchema, tags=[ResourcesTag], response_schemas=Resources_POST_responses) @view_config(route_name=ResourcesAPI.name, request_method='POST') def create_resource_view(request): diff --git a/magpie/api/management/service/service_views.py b/magpie/api/management/service/service_views.py index f5a14c0e2..948ad94a4 100644 --- a/magpie/api/management/service/service_views.py +++ b/magpie/api/management/service/service_views.py @@ -60,6 +60,7 @@ def register_service(request): httpError=HTTPConflict, msgOnFail=Services_POST_ConflictResponseSchema.description, content={u'service_name': str(service_name)}, paramName=u'service_name') + # noinspection PyArgumentList service = evaluate_call(lambda: models.Service(resource_name=str(service_name), resource_type=models.Service.resource_type_name, url=str(service_url), type=str(service_type)), @@ -188,7 +189,7 @@ def get_service_resources_view(request): content={str(service.resource_name): svc_res_json}) -@ServiceResourcesAPI.post(schema=ServiceResources_POST_RequestBodySchema, tags=[ServicesTag], +@ServiceResourcesAPI.post(schema=ServiceResources_POST_RequestSchema, tags=[ServicesTag], response_schemas=ServiceResources_POST_responses) @view_config(route_name=ServiceResourcesAPI.name, request_method='POST') def create_service_direct_resource(request): @@ -200,8 +201,8 @@ def create_service_direct_resource(request): parent_id = get_multiformat_post(request, 'parent_id') # no check because None/empty is allowed if not parent_id: parent_id = service.resource_id - return create_resource(resource_name, resource_display_name, resource_type, parent_id=parent_id, - db_session=request.db) + return create_resource(resource_name, resource_display_name, resource_type, + parent_id=parent_id, db_session=request.db) @ServiceResourceTypesAPI.get(tags=[ServicesTag], response_schemas=ServiceResource_GET_responses) diff --git a/magpie/definitions/pyramid_definitions.py b/magpie/definitions/pyramid_definitions.py index f54fab654..dec696242 100644 --- a/magpie/definitions/pyramid_definitions.py +++ b/magpie/definitions/pyramid_definitions.py @@ -18,6 +18,7 @@ HTTPUnprocessableEntity, HTTPInternalServerError, HTTPNotImplemented, + HTTPException, ) from pyramid.registry import Registry from pyramid.settings import asbool diff --git a/magpie/models.py b/magpie/models.py index d5d04d7b8..def34e1ed 100644 --- a/magpie/models.py +++ b/magpie/models.py @@ -1,7 +1,7 @@ from magpie.definitions.pyramid_definitions import * from magpie.definitions.ziggurat_definitions import * from magpie.definitions.sqlalchemy_definitions import * -from magpie.api.api_except import * +from magpie.api.api_except import evaluate_call from magpie.permissions import * Base = declarative_base() @@ -12,10 +12,10 @@ def get_session_callable(request): def get_user(request): - userid = request.unauthenticated_userid + user_id = request.unauthenticated_userid db_session = get_session_callable(request) - if userid is not None: - return UserService.by_id(userid, db_session=db_session) + if user_id is not None: + return UserService.by_id(user_id, db_session=db_session) class Group(GroupMixin, Base): diff --git a/magpie/permissions.py b/magpie/permissions.py index ef42cf8df..a4145eed8 100644 --- a/magpie/permissions.py +++ b/magpie/permissions.py @@ -15,3 +15,22 @@ PERMISSION_EXECUTE = u'execute' PERMISSION_LOCK_FEATURE = u'lockfeature' PERMISSION_TRANSACTION = u'transaction' + +permissions_supported = frozenset([ + PERMISSION_READ, + PERMISSION_READ_MATCH, + PERMISSION_WRITE, + PERMISSION_WRITE_MATCH, + PERMISSION_ACCESS, + PERMISSION_GET_CAPABILITIES, + PERMISSION_GET_MAP, + PERMISSION_GET_FEATURE_INFO, + PERMISSION_GET_LEGEND_GRAPHIC, + PERMISSION_GET_METADATA, + PERMISSION_GET_FEATURE, + PERMISSION_DESCRIBE_FEATURE_TYPE, + PERMISSION_DESCRIBE_PROCESS, + PERMISSION_EXECUTE, + PERMISSION_LOCK_FEATURE, + PERMISSION_TRANSACTION, +]) diff --git a/magpie/register.py b/magpie/register.py index 3d34d2a67..56638ec44 100644 --- a/magpie/register.py +++ b/magpie/register.py @@ -1,12 +1,22 @@ -from magpie.api.api_rest_schemas import SigninAPI, SignoutAPI, ServicesAPI -from magpie.services import service_type_dict +from magpie.api.api_rest_schemas import ( + SigninAPI, + SignoutAPI, + ServicesAPI, + ServiceAPI, + ServiceResourcesAPI, + GroupResourcePermissionAPI, + UserResourcePermissionAPI, +) from magpie.common import make_dirs, print_log, raise_log, bool2str from magpie.constants import get_constant -from magpie import models from magpie.definitions.ziggurat_definitions import UserService, UserResourcePermissionService from magpie.definitions.sqlalchemy_definitions import Session -from typing import AnyStr, Dict, List, Optional, Tuple, TYPE_CHECKING +from magpie.permissions import permissions_supported +from magpie.services import service_type_dict +from magpie import models +from typing import AnyStr, Dict, List, Optional, Tuple, Union, TYPE_CHECKING import os +import six import time import yaml import subprocess @@ -29,7 +39,9 @@ SERVICES_PHOENIX_ALLOWED = ['wps'] if TYPE_CHECKING: - ConfigDict = Dict[AnyStr, Dict[AnyStr, AnyStr]] + ConfigItem = Dict[AnyStr, AnyStr] + ConfigList = List[ConfigItem] + ConfigDict = Dict[AnyStr, Union[ConfigItem, ConfigList]] def login_loop(login_url, cookies_file, data=None, message='Login response'): @@ -450,9 +462,14 @@ def magpie_register_services_with_db_session(services_dict, db_session, push_to_ return True -def _load_config(path, section): +def _load_config(path_or_dict, section): + # type: (Union[AnyStr, ConfigDict], AnyStr) -> ConfigDict + """Loads a file path or dictionary as YAML/JSON configuration.""" try: - cfg = yaml.load(open(path, 'r')) + if isinstance(path_or_dict, six.string_types): + cfg = yaml.load(open(path_or_dict, 'r')) + else: + cfg = path_or_dict return cfg[section] except KeyError as ex: raise_log("Config file section [{!s}] not found.".format(section), exception=type(ex)) @@ -487,21 +504,135 @@ def magpie_register_services_from_config(service_config_file_path, push_to_phoen update_getcapabilities_permissions=not disable_getcapabilities) -def magpie_register_permissions_from_config(permissions_config_file_path): - permissions = _load_config(permissions_config_file_path, 'permissions') +def warn_permission(msg, _i, trail="skipping...", detail=None): + if detail: + trail = "{}\nDetail: [{!r}]".format(trail, detail) + warnings.warn("{!s} [permission #{}], {}".format(msg, _i, trail), UserWarning) + + +def parse_resource_path(permission_config_entry, entry_index, service_info, cookies): + # type: (ConfigItem, int, ConfigItem, Dict[AnyStr, AnyStr]) -> Tuple[Union[int, None], bool] + """ + Parses the `resource` field of a permission config entry and retrieves the final resource id. + Creates missing resources as necessary if they can be automatically resolved. + + :returns: tuple of found id (if any, `None` otherwise), and success status of the parsing operation (error) + """ + resource = None + resource_path = permission_config_entry.get('resource', '') + if resource_path.startswith('/'): + resource_path = resource_path[1:] + if resource_path.endswith('/'): + resource_path = resource_path[:-1] + if resource_path: + try: + svc_name = service_info['service_name'] + svc_type = service_info['service_type'] + res_path = get_magpie_url() + ServiceResourcesAPI.path.format(service_name=svc_name) + res_dict = requests.get(res_path, cookies=cookies).json()[svc_name]['resources'] + parent = None + for res in resource_path.split('/'): + if len(res_dict): + # resource is specified by id/name + res_id = filter(lambda r: res in [r, res_dict[r]["resource_name"]], res_dict) + if res_id: + res_dict = res_dict[res_id[0]]['children'] # update for upcoming sub-resource iteration + parent = int(res[0]) + continue + # missing resource, attempt creation + type_count = len(service_type_dict[svc_type].resource_types) + if type_count != 1: + warn_permission("Cannot automatically generate resources", entry_index, + detail="Service [{}] of type [{}] allows {} sub-resource types" + .format(svc_name, svc_type, type_count)) + raise Exception("Missing resource to apply permission.") # fail fast + res_type = service_type_dict[svc_type].resource_types[0] + body = {'resource_name': res, 'resource_type': res_type, 'parent_id': parent} + resp = requests.post(res_path, json=body, cookies=cookies) + if resp.status_code != 201: + resp.raise_for_status() + parent = resp.json()['resource']['resource_id'] + resource = parent + if not resource: + raise Exception("Could not extract child resource from resource path.") + except Exception as ex: + warn_permission("Failed resources parsing.", entry_index, detail=ex) + return None, False + return resource, True + + +def magpie_register_permissions_from_config(permissions_config): + # type: (Union[AnyStr, ConfigDict]) -> None + """ + Applies permissions specified in configuration. + + .. seealso:: + `magpie/permissions.cfg` for specific parameter or operational details. + """ + + permissions = _load_config(permissions_config, 'permissions') if not permissions: warnings.warn("Permissions configuration are empty.", UserWarning) return + magpie_url = get_magpie_url() admin_usr = get_constant('MAGPIE_ADMIN_USER') admin_pwd = get_constant('MAGPIE_ADMIN_PASSWORD') body = {'user_name': admin_usr, 'password': admin_pwd} - resp = requests.get(get_magpie_url() + SigninAPI.path, json=body) - if not resp.status_code == 200: + resp = requests.get(magpie_url + SigninAPI.path, json=body) + if resp.status_code != 200: raise_log("Cannot register Magpie permissions without proper credentials.") + admin_cookies = resp.cookies - for perm in permissions: - if not isinstance(perm, dict) or not all(f in perm for f in ['permission', 'service', 'resource']): - warnings.warn("Invalid permission format for [{!s}].".format(perm), UserWarning) + for i, perm in enumerate(permissions): + # parameter validation + if not isinstance(perm, dict) or not all(f in perm for f in ['permission', 'service']): + warn_permission("Invalid permission format for [{!s}]".format(perm), i) + continue + if perm['permission'] not in permissions_supported: + warn_permission("Unknown permission [{!s}]".format(perm['permission']), i) + continue + if not any(f in perm for f in ['user', 'group']): + warn_permission("Missing required user and/or group field.", i) continue + if 'action' not in perm: + warn_permission("Unspecified action", i, trail="using default...") + perm['action'] = 'create' + if perm['action'] not in ['create', 'remove']: + warn_permission("Unknown action [{!s}]".format(perm['action']), i) + continue + usr_name = perm['user'] + grp_name = perm['group'] + svc_name = perm['service'] + svc_path = magpie_url + ServiceAPI.path.format(service_name=svc_name) + svc_resp = requests.get(svc_path, cookies=admin_cookies) + if svc_resp.status_code != 200: + warn_permission("Unknown service [{!s}]".format(svc_name), i) + continue + service_info = svc_resp.json()[svc_name] + + # apply permission config + resource, found = parse_resource_path(perm, i, service_info, admin_cookies) + if not found: + continue + if not resource: + resource = service_info['resource_id'] + perm_actions = list() + #if usr_name: + + + + + #for usr_grp_name, usr_grp_name in [(usr_name, UserResourcePermissionAPI.path.format(user_name=usr_name)), + # (grp_name, GroupResourcePermissionAPI.path.format(group_name=grp_name))]: + if perm['action'] == 'create': + action_func = requests.post + action_path = '/permissions' + else: + action_func = requests.delete + action_path = '/permissions/{perm_name}'.format(perm_name=perm['permission']) + #if usr_name: + # perm_path = + #for action, path, perm in perm_actions: + # action(path, json cookies=) diff --git a/magpie/security.py b/magpie/security.py index f3a28b7ff..b5a44fa67 100644 --- a/magpie/security.py +++ b/magpie/security.py @@ -2,7 +2,6 @@ from magpie.definitions.ziggurat_definitions import * from magpie.api.login import esgfopenid, wso2 from magpie.constants import get_constant -from magpie import models from authomatic import Authomatic, provider_id from authomatic.providers import oauth2, openid import logging @@ -30,6 +29,7 @@ def auth_config_from_settings(settings): ) authz_policy = ACLAuthorizationPolicy() + from magpie import models config = Configurator( settings=settings, root_factory=models.RootFactory, @@ -101,7 +101,7 @@ def authomatic_config(request=None): 'consumer_key': get_constant('GITHUB_CLIENT_ID', **_get_const_info), 'consumer_secret': get_constant('GITHUB_CLIENT_SECRET', **_get_const_info), 'redirect_uri': request.application_url if request else None, - #'redirect_uri': '{}/providers/github/signin'.format(request.application_url) if request else None, + # 'redirect_uri': '{}/providers/github/signin'.format(request.application_url) if request else None, 'access_headers': {'User-Agent': 'Magpie'}, 'id': provider_id(), '_apis': { diff --git a/magpie/services.py b/magpie/services.py index ef9cfd766..db76a7368 100644 --- a/magpie/services.py +++ b/magpie/services.py @@ -223,13 +223,13 @@ def __init__(self, service, request): def __acl__(self): self.expand_acl(self.service, self.request.user) - #localhost:8087/geoserver/WATERSHED/wms?layers=WATERSHED:BV_1NS&request=getmap - #localhost:8087/geoserver/wms?layers=WATERERSHED:BV1_NS&request=getmap - #those two request lead to the same thing so, here we need to check the workspace in the layer + # localhost:8087/geoserver/WATERSHED/wms?layers=WATERSHED:BV_1NS&request=getmap + # localhost:8087/geoserver/wms?layers=WATERERSHED:BV1_NS&request=getmap + # those two request lead to the same thing so, here we need to check the workspace in the layer - #localhost:8087/geoserver/wms?request=getcapabilities (dangerous, get all workspace) + # localhost:8087/geoserver/wms?request=getcapabilities (dangerous, get all workspace) # localhost:8087/geoserver/WATERSHED/wms?request=getcapabilities (only for the workspace in the path) - #here we need to check the workspace in the path + # here we need to check the workspace in the path request_type = self.permission_requested() if request_type == PERMISSION_GET_CAPABILITIES: @@ -244,7 +244,7 @@ def __acl__(self): workspace_name = layer_name.split(':')[0] # load workspace resource from the database - workspace = models.find_children_by_name(name=workspace_name, + workspace = models.find_children_by_name(child_name=workspace_name, parent_id=self.service.resource_id, db_session=self.request.db) if workspace: @@ -373,7 +373,7 @@ def __acl__(self): workspace_name = layer_name.split(':')[0] # load workspace resource from the database - workspace = models.find_children_by_name(name=workspace_name, + workspace = models.find_children_by_name(child_name=workspace_name, parent_id=self.service.resource_id, db_session=self.request.db) if workspace: @@ -422,7 +422,7 @@ def __acl__(self): while new_child and elems: elem_name = elems.pop(0) if ".nc" in elem_name: - elem_name = elem_name.split(".nc")[0]+".nc" #in case there is more extension to discard such as .dds + elem_name = elem_name.split(".nc")[0]+".nc" # in case there is more extension to discard such as .dds parent_id = new_child.resource_id new_child = models.find_children_by_name(elem_name, parent_id=parent_id, db_session=self.request.db) self.expand_acl(new_child, self.request.user) diff --git a/permissions.cfg b/permissions.cfg index 6f0446e86..a1db3d22e 100644 --- a/permissions.cfg +++ b/permissions.cfg @@ -1,14 +1,16 @@ -# For each permission to be updated # -# service: service name to receive the permission (directly on it if resource not specified) +# Parameters: +# service: service name to receive the permission (directly on it if no 'resource' mentioned, must exist) # resource (optional): tree path of the service's resource (ex: /res1/sub-res2/sub-sub-res3) -# user and/or group: user/group to apply the permission on -# permission: name of the permission to be applied -# action: one of [create, delete] (default: create) +# user and/or group: user/group to apply the permission on (user/group must exist) +# permission: name of the permission to be applied (see 'magpie/permissions.py' for supported values) +# action: one of [create, remove] (default: create) # -# Default behaviour is to create missing resources if supported by the service, then apply permissions. -# If the service is missing, corresponding permissions are ignored and not updated. -# If permission configuration is already satisfied, it is left as is. +# Default behaviour: +# - create missing resources if supported by the service, then apply permissions. +# - applicable service, user or group is missing, corresponding permissions are ignored and not updated. +# - unknown actions are ignored and corresponding permission are not updated, unspecified action resolves to 'create'. +# - already satisfied permission configurations are left as is. # permissions: From 520af2c80547e7e1cdc937f81e12f1d91c09814b Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Mon, 28 Jan 2019 14:37:13 -0500 Subject: [PATCH 03/76] permission setup from config --- magpie/api/login/__init__.py | 2 +- magpie/magpiectl.py | 7 ++-- magpie/register.py | 80 ++++++++++++++++++++++-------------- permissions.cfg | 7 ++-- 4 files changed, 57 insertions(+), 39 deletions(-) diff --git a/magpie/api/login/__init__.py b/magpie/api/login/__init__.py index f5063a089..3bae7f2c1 100644 --- a/magpie/api/login/__init__.py +++ b/magpie/api/login/__init__.py @@ -1,9 +1,9 @@ -from magpie.api import api_rest_schemas as s import logging logger = logging.getLogger(__name__) def includeme(config): + from magpie.api import api_rest_schemas as s logger.info('Adding api login ...') # Add all the rest api routes config.add_route(**s.service_api_route_info(s.SessionAPI)) diff --git a/magpie/magpiectl.py b/magpie/magpiectl.py index 1e638e633..d6943db07 100644 --- a/magpie/magpiectl.py +++ b/magpie/magpiectl.py @@ -17,8 +17,9 @@ # -- Standard library -------------------------------------------------------- import time import warnings +# noinspection PyUnresolvedReferences import logging -import logging.config +import logging.config # find config in 'logging.ini' LOGGER = logging.getLogger(__name__) @@ -56,7 +57,7 @@ def main(global_config=None, **settings): register_default_users() print_log('Running configurations setup...') - magpie_url_template = 'http://{hostname}:{port}/magpie' + magpie_url_template = 'http://{hostname}:{port}' port = get_constant('MAGPIE_PORT', settings=settings, settings_name='magpie.port') if port: settings['magpie.port'] = port @@ -71,7 +72,7 @@ def main(global_config=None, **settings): config.include('magpie') # Don't use scan otherwise modules like 'magpie.adapter' are # automatically found and cause import errors on missing packages - #config.scan('magpie') + # config.scan('magpie') config.set_default_permission(get_constant('MAGPIE_ADMIN_PERMISSION')) print_log('Starting Magpie app...') diff --git a/magpie/register.py b/magpie/register.py index 56638ec44..03863285c 100644 --- a/magpie/register.py +++ b/magpie/register.py @@ -4,8 +4,8 @@ ServicesAPI, ServiceAPI, ServiceResourcesAPI, - GroupResourcePermissionAPI, - UserResourcePermissionAPI, + GroupResourcePermissionsAPI, + UserResourcePermissionsAPI, ) from magpie.common import make_dirs, print_log, raise_log, bool2str from magpie.constants import get_constant @@ -22,8 +22,9 @@ import subprocess import requests import transaction -import warnings +# noinspection PyUnresolvedReferences import logging +import logging.config # find config in 'logging.ini' LOGGER = logging.getLogger(__name__) LOGIN_ATTEMPT = 10 # max attempts for login @@ -487,7 +488,7 @@ def magpie_register_services_from_config(service_config_file_path, push_to_phoen """ services = _load_config(service_config_file_path, 'providers') if not services: - warnings.warn("Services configuration are empty.", UserWarning) + LOGGER.warning("Services configuration are empty.") return # register services using API POSTs @@ -504,10 +505,10 @@ def magpie_register_services_from_config(service_config_file_path, push_to_phoen update_getcapabilities_permissions=not disable_getcapabilities) -def warn_permission(msg, _i, trail="skipping...", detail=None): +def warn_permission(msg, _i, trail="skipping...", detail=None, level=logging.WARN): if detail: trail = "{}\nDetail: [{!r}]".format(trail, detail) - warnings.warn("{!s} [permission #{}], {}".format(msg, _i, trail), UserWarning) + LOGGER.log(level, "{!s} [permission #{}], {}".format(msg, _i, trail)) def parse_resource_path(permission_config_entry, entry_index, service_info, cookies): @@ -572,18 +573,19 @@ def magpie_register_permissions_from_config(permissions_config): permissions = _load_config(permissions_config, 'permissions') if not permissions: - warnings.warn("Permissions configuration are empty.", UserWarning) + LOGGER.warning("Permissions configuration are empty.") return magpie_url = get_magpie_url() admin_usr = get_constant('MAGPIE_ADMIN_USER') admin_pwd = get_constant('MAGPIE_ADMIN_PASSWORD') body = {'user_name': admin_usr, 'password': admin_pwd} - resp = requests.get(magpie_url + SigninAPI.path, json=body) + resp = requests.post(magpie_url + SigninAPI.path, json=body) if resp.status_code != 200: raise_log("Cannot register Magpie permissions without proper credentials.") admin_cookies = resp.cookies + logging.info("Found {} permissions to update.".format(len(permissions))) for i, perm in enumerate(permissions): # parameter validation if not isinstance(perm, dict) or not all(f in perm for f in ['permission', 'service']): @@ -592,17 +594,17 @@ def magpie_register_permissions_from_config(permissions_config): if perm['permission'] not in permissions_supported: warn_permission("Unknown permission [{!s}]".format(perm['permission']), i) continue - if not any(f in perm for f in ['user', 'group']): + usr_name = perm.get('user') + grp_name = perm.get('group') + if not any([usr_name, grp_name]): warn_permission("Missing required user and/or group field.", i) continue if 'action' not in perm: - warn_permission("Unspecified action", i, trail="using default...") + warn_permission("Unspecified action", i, trail="using default (create)...") perm['action'] = 'create' if perm['action'] not in ['create', 'remove']: warn_permission("Unknown action [{!s}]".format(perm['action']), i) continue - usr_name = perm['user'] - grp_name = perm['group'] svc_name = perm['service'] svc_path = magpie_url + ServiceAPI.path.format(service_name=svc_name) svc_resp = requests.get(svc_path, cookies=admin_cookies) @@ -617,22 +619,38 @@ def magpie_register_permissions_from_config(permissions_config): continue if not resource: resource = service_info['resource_id'] - perm_actions = list() - #if usr_name: - - - - - #for usr_grp_name, usr_grp_name in [(usr_name, UserResourcePermissionAPI.path.format(user_name=usr_name)), - # (grp_name, GroupResourcePermissionAPI.path.format(group_name=grp_name))]: - if perm['action'] == 'create': - action_func = requests.post - action_path = '/permissions' - else: - action_func = requests.delete - action_path = '/permissions/{perm_name}'.format(perm_name=perm['permission']) - #if usr_name: - # perm_path = - - #for action, path, perm in perm_actions: - # action(path, json cookies=) + perm_paths = list() + if usr_name: + perm_paths.append(UserResourcePermissionsAPI.path.replace('{user_name}', usr_name)) + if grp_name: + perm_paths.append(GroupResourcePermissionsAPI.path.replace('{group_name}', grp_name)) + for path in perm_paths: + create_perm = perm['action'] == 'create' + if create_perm: + action_func = requests.post + action_path = '{url}{path}'.format(url=magpie_url, path=path) + action_body = {'permission_name': perm['permission']} + else: + action_func = requests.delete + action_path = '{url}{path}/{perm_name}'.format(url=magpie_url, path=path, perm_name=perm['permission']) + action_body = {} + action_path = action_path.format(resource_id=resource) + action_resp = action_func(action_path, json=action_body, cookies=admin_cookies) + action_code = action_resp.status_code + if create_perm: + if action_code == 201: + warn_permission("Permission successfully created.", i, level=logging.INFO) + elif action_code == 409: + warn_permission("Permission already exists: {!s}".format(perm), i, level=logging.INFO) + else: + warn_permission("Unknown response for 'create' permission [{}]: {!s}" + .format(action_code, perm), i, level=logging.ERROR) + else: + if action_code == 200: + warn_permission("Permission successfully removed.", i, level=logging.INFO) + elif action_code == 404: + warn_permission("Permission successfully removed.", i, level=logging.INFO) + else: + warn_permission("Unknown response for 'remove' permission [{}]: {!s}" + .format(action_code, perm), i, level=logging.ERROR) + logging.info("Done processing permissions.") diff --git a/permissions.cfg b/permissions.cfg index a1db3d22e..4e553fda3 100644 --- a/permissions.cfg +++ b/permissions.cfg @@ -17,9 +17,8 @@ permissions: - service: api resource: /api permission: read - user: + user: anonymous action: create - - service: test - resource: + - service: flyingpigeon permission: getcapabilities - group: + group: administrators From 7510774c21675b57d8f1b646c7b221da8415c7e0 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Mon, 28 Jan 2019 17:29:43 -0500 Subject: [PATCH 04/76] allow permission config with db session --- Makefile | 2 +- magpie/api/management/group/group_utils.py | 15 +- magpie/api/management/user/user_utils.py | 6 + magpie/constants.py | 1 + magpie/magpiectl.py | 18 +- magpie/register.py | 233 +++++++++++++++------ 6 files changed, 198 insertions(+), 77 deletions(-) diff --git a/Makefile b/Makefile index 17bf99d38..2b29dd9f5 100644 --- a/Makefile +++ b/Makefile @@ -141,7 +141,7 @@ sysinstall: clean .PHONY: install install: sysinstall @echo "Installing Magpie..." - python setup.py install + pip install $(CUR_DIR) .PHONY: cron cron: diff --git a/magpie/api/management/group/group_utils.py b/magpie/api/management/group/group_utils.py index 37ac0ed1b..3b3e5b55b 100644 --- a/magpie/api/management/group/group_utils.py +++ b/magpie/api/management/group/group_utils.py @@ -1,4 +1,3 @@ -from magpie.models import resource_tree_service, resource_type_dict from magpie.services import service_type_dict from magpie.api.api_requests import * from magpie.api.api_except import * @@ -6,7 +5,10 @@ from magpie.api.management.resource.resource_formats import format_resource from magpie.api.management.service.service_formats import format_service_resources, format_service from magpie.api.management.group.group_formats import format_group +from magpie.definitions.sqlalchemy_definitions import Session from magpie.definitions.ziggurat_definitions import * +from magpie import models +from typing import AnyStr def get_all_groups(db_session): @@ -36,6 +38,11 @@ def get_group_resources(group, db_session): def create_group_resource_permission(permission_name, resource, group, db_session): + # type: (AnyStr, models.Resource, models.Group, Session) -> HTTPException + """ + Creates a permission on a group/resource combination if it is permitted and not conflicting. + :returns: corresponding HTTP response according to the encountered situation. + """ resource_id = resource.resource_id check_valid_service_resource_permission(permission_name, resource, db_session) perm_content = {u'permission_name': str(permission_name), @@ -48,6 +55,7 @@ def create_group_resource_permission(permission_name, resource, group, db_sessio ) verify_param(create_perm, isNone=True, httpError=HTTPConflict, msgOnFail=GroupResourcePermissions_POST_ConflictResponseSchema.description, content=perm_content) + # noinspection PyArgumentList new_perm = evaluate_call(lambda: models.GroupResourcePermission(resource_id=resource_id, group_id=group.id), fallback=lambda: db_session.rollback(), httpError=HTTPForbidden, msgOnFail=GroupResourcePermissions_POST_ForbiddenCreateResponseSchema.description, @@ -82,7 +90,7 @@ def get_grp_res_perm(grp, db, res_ids, res_types): def get_group_resource_permissions(group, resource, db_session): def get_grp_res_perms(grp, res, db): if res.owner_group_id == grp.id: - perm_names = resource_type_dict[res.type].permission_names + perm_names = models.resource_type_dict[res.type].permission_names else: grp_res_perm = db.query(models.GroupResourcePermission) \ .filter(models.GroupResourcePermission.resource_id == res.resource_id) \ @@ -157,6 +165,7 @@ def get_grp_svc_perms(grp, db_ses, res_ids): def get_group_service_resources_permissions_dict(group, service, db_session): - res_under_svc = resource_tree_service.from_parent_deeper(parent_id=service.resource_id, db_session=db_session) + res_id = service.resource_id + res_under_svc = models.resource_tree_service.from_parent_deeper(parent_id=res_id, db_session=db_session) res_ids = [resource.Resource.resource_id for resource in res_under_svc] return get_group_resources_permissions_dict(group, db_session, resource_types=None, resource_ids=res_ids) diff --git a/magpie/api/management/user/user_utils.py b/magpie/api/management/user/user_utils.py index 8de7eda88..4a6b7e5da 100644 --- a/magpie/api/management/user/user_utils.py +++ b/magpie/api/management/user/user_utils.py @@ -3,6 +3,7 @@ from magpie.api.management.service.service_formats import format_service from magpie.api.management.resource.resource_utils import check_valid_service_resource_permission from magpie.api.management.user.user_formats import * +from magpie.definitions.sqlalchemy_definitions import Session from magpie.definitions.ziggurat_definitions import * from magpie.definitions.pyramid_definitions import Request from magpie.services import service_factory, ResourcePermissionType, ServiceI @@ -48,6 +49,11 @@ def create_user(user_name, password, email, group_name, db_session): def create_user_resource_permission(permission_name, resource, user_id, db_session): + # type: (AnyStr, models.Resource, models.User, Session) -> HTTPException + """ + Creates a permission on a user/resource combination if it is permitted and not conflicting. + :returns: corresponding HTTP response according to the encountered situation. + """ check_valid_service_resource_permission(permission_name, resource, db_session) resource_id = resource.resource_id existing_perm = UserResourcePermissionService.by_resource_user_and_perm( diff --git a/magpie/constants.py b/magpie/constants.py index cf4b8313f..1fcbb37db 100644 --- a/magpie/constants.py +++ b/magpie/constants.py @@ -13,6 +13,7 @@ MAGPIE_MODULE_DIR = os.path.abspath(os.path.dirname(__file__)) MAGPIE_ROOT = os.path.dirname(MAGPIE_MODULE_DIR) MAGPIE_PROVIDERS_CONFIG_PATH = '{}/providers.cfg'.format(MAGPIE_ROOT) +MAGPIE_PERMISSIONS_CONFIG_PATH = '{}/permissions.cfg'.format(MAGPIE_ROOT) MAGPIE_INI_FILE_PATH = '{}/magpie.ini'.format(MAGPIE_MODULE_DIR) MAGPIE_ALEMBIC_INI_FILE_PATH = '{}/alembic/alembic.ini'.format(MAGPIE_MODULE_DIR) # allow custom location of env files directory to avoid diff --git a/magpie/magpiectl.py b/magpie/magpiectl.py index d6943db07..05210531c 100644 --- a/magpie/magpiectl.py +++ b/magpie/magpiectl.py @@ -10,7 +10,10 @@ from magpie.constants import get_constant from magpie.definitions.sqlalchemy_definitions import * from magpie.helpers.register_default_users import register_default_users -from magpie.helpers.register_providers import magpie_register_services_from_config +from magpie.register import ( + magpie_register_services_from_config, + magpie_register_permissions_from_config, +) from magpie.security import auth_config_from_settings from magpie import db, constants @@ -46,15 +49,18 @@ def main(global_config=None, **settings): time.sleep(2) raise_log('Database not ready') - print_log('Register default providers...', LOGGER) - svc_db_session = db.get_db_session_from_config_ini(constants.MAGPIE_INI_FILE_PATH) + print_log('Register default users...') + register_default_users() + + print_log('Register configuration providers...', LOGGER) + db_session = db.get_db_session_from_config_ini(constants.MAGPIE_INI_FILE_PATH) push_phoenix = str2bool(get_constant('PHOENIX_PUSH', settings=settings, settings_name='magpie.phoenix_push', raise_missing=False, raise_not_set=False, print_missing=True)) magpie_register_services_from_config(constants.MAGPIE_PROVIDERS_CONFIG_PATH, push_to_phoenix=push_phoenix, - force_update=True, disable_getcapabilities=False, db_session=svc_db_session) + force_update=True, disable_getcapabilities=False, db_session=db_session) - print_log('Register default users...') - register_default_users() + print_log('Register configuration permissions...', LOGGER) + magpie_register_permissions_from_config(constants.MAGPIE_PERMISSIONS_CONFIG_PATH, db_session=db_session) print_log('Running configurations setup...') magpie_url_template = 'http://{hostname}:{port}' diff --git a/magpie/register.py b/magpie/register.py index 03863285c..0cbaadd25 100644 --- a/magpie/register.py +++ b/magpie/register.py @@ -7,14 +7,24 @@ GroupResourcePermissionsAPI, UserResourcePermissionsAPI, ) +from magpie.api.management.service.service_formats import format_service_resources, format_service +from magpie.api.management.resource.resource_utils import create_resource +from magpie.api.management.user.user_utils import create_user_resource_permission, delete_user_resource_permission +from magpie.api.management.group.group_utils import create_group_resource_permission, delete_group_resource_permission from magpie.common import make_dirs, print_log, raise_log, bool2str from magpie.constants import get_constant -from magpie.definitions.ziggurat_definitions import UserService, UserResourcePermissionService +from magpie.definitions.ziggurat_definitions import ( + ResourceService, + GroupService, + UserService, + UserResourcePermissionService, +) from magpie.definitions.sqlalchemy_definitions import Session from magpie.permissions import permissions_supported from magpie.services import service_type_dict from magpie import models from typing import AnyStr, Dict, List, Optional, Tuple, Union, TYPE_CHECKING +from requests.cookies import RequestsCookieJar import os import six import time @@ -349,8 +359,8 @@ def magpie_update_services_conflict(conflict_services, services_dict, request_co return statuses -def magpie_register_services(services_dict, push_to_phoenix, username, password, provider, - force_update=False, disable_getcapabilities=False): +def magpie_register_services_with_requests(services_dict, push_to_phoenix, username, password, provider, + force_update=False, disable_getcapabilities=False): # type: (ConfigDict, bool, AnyStr, AnyStr, AnyStr, Optional[bool], Optional[bool]) -> bool """ Registers magpie services using the provided services configuration. @@ -495,8 +505,9 @@ def magpie_register_services_from_config(service_config_file_path, push_to_phoen if db_session is None: admin_usr = get_constant('MAGPIE_ADMIN_USER') admin_pwd = get_constant('MAGPIE_ADMIN_PASSWORD') - magpie_register_services(services, push_to_phoenix, admin_usr, admin_pwd, 'ziggurat', - force_update=force_update, disable_getcapabilities=disable_getcapabilities) + magpie_register_services_with_requests(services, push_to_phoenix, admin_usr, admin_pwd, 'ziggurat', + force_update=force_update, + disable_getcapabilities=disable_getcapabilities) # register services directly to db using session else: @@ -505,20 +516,36 @@ def magpie_register_services_from_config(service_config_file_path, push_to_phoen update_getcapabilities_permissions=not disable_getcapabilities) -def warn_permission(msg, _i, trail="skipping...", detail=None, level=logging.WARN): +def warn_permission(msg, _i, trail="skipping...", detail=None, permission=None, level=logging.WARN): if detail: trail = "{}\nDetail: [{!r}]".format(trail, detail) - LOGGER.log(level, "{!s} [permission #{}], {}".format(msg, _i, trail)) + if permission: + permission = ' [{!s}]' + LOGGER.log(level, "{!s} [permission #{}]{}, {}".format(msg, _i, permission or '', trail)) + + +def use_request(cookies_or_session): + return not isinstance(cookies_or_session, Session) -def parse_resource_path(permission_config_entry, entry_index, service_info, cookies): - # type: (ConfigItem, int, ConfigItem, Dict[AnyStr, AnyStr]) -> Tuple[Union[int, None], bool] +def parse_resource_path(permission_config_entry, # type: ConfigItem + entry_index, # type: int + service_info, # type: ConfigItem + cookies_or_session=None, # type: Union[RequestsCookieJar, Session] + magpie_url=None, # type: AnyStr + ): # type: (...) -> Tuple[Union[int, None], bool] """ Parses the `resource` field of a permission config entry and retrieves the final resource id. Creates missing resources as necessary if they can be automatically resolved. + If `cookies` are provided, uses requests to a running `Magpie` instance (with `magpie_url`) to apply permission. + If `session` to db is provided, uses direct db connection instead to apply permission. + :returns: tuple of found id (if any, `None` otherwise), and success status of the parsing operation (error) """ + if not magpie_url and use_request(cookies_or_session): + raise ValueError("cannot use cookies without corresponding request URL") + resource = None resource_path = permission_config_entry.get('resource', '') if resource_path.startswith('/'): @@ -529,8 +556,12 @@ def parse_resource_path(permission_config_entry, entry_index, service_info, cook try: svc_name = service_info['service_name'] svc_type = service_info['service_type'] - res_path = get_magpie_url() + ServiceResourcesAPI.path.format(service_name=svc_name) - res_dict = requests.get(res_path, cookies=cookies).json()[svc_name]['resources'] + if use_request(cookies_or_session): + res_path = get_magpie_url() + ServiceResourcesAPI.path.format(service_name=svc_name) + res_dict = requests.get(res_path, cookies=cookies_or_session).json()[svc_name]['resources'] + else: + svc = models.Service.by_service_name(svc_name, db_session=cookies_or_session) + res_dict = format_service_resources(svc, db_session=cookies_or_session) parent = None for res in resource_path.split('/'): if len(res_dict): @@ -548,8 +579,12 @@ def parse_resource_path(permission_config_entry, entry_index, service_info, cook .format(svc_name, svc_type, type_count)) raise Exception("Missing resource to apply permission.") # fail fast res_type = service_type_dict[svc_type].resource_types[0] - body = {'resource_name': res, 'resource_type': res_type, 'parent_id': parent} - resp = requests.post(res_path, json=body, cookies=cookies) + if use_request(cookies_or_session): + body = {'resource_name': res, 'resource_type': res_type, 'parent_id': parent} + # noinspection PyUnboundLocalVariable + resp = requests.post(res_path, json=body, cookies=cookies_or_session) + else: + resp = create_resource(res, res, res_type, parent, db_session=cookies_or_session) if resp.status_code != 201: resp.raise_for_status() parent = resp.json()['resource']['resource_id'] @@ -562,28 +597,120 @@ def parse_resource_path(permission_config_entry, entry_index, service_info, cook return resource, True -def magpie_register_permissions_from_config(permissions_config): - # type: (Union[AnyStr, ConfigDict]) -> None +def apply_permission_entry(permission_config_entry, # type: ConfigItem + entry_index, # type: int + resource_id, # type: int + cookies_or_session, # type: Union[RequestsCookieJar, Session] + magpie_url, # type: Optional[AnyStr] + ): # type: (...) -> None + """ + Applies the single permission entry retrieved from the permission configuration. + Assumes that permissions fields where pre-validated. + Permission is applied for the user/group/resource using request or db session accordingly to arguments. + """ + + def _apply_request(_usr_name=None, _grp_name=None): + """Apply operation using HTTP request.""" + action_oper = None + if usr_name: + action_oper = UserResourcePermissionsAPI.path.replace('{user_name}', _usr_name) + if grp_name: + action_oper = GroupResourcePermissionsAPI.path.replace('{group_name}', _grp_name) + if not action_oper: + return None + if create_perm: + action_func = requests.post + action_path = '{url}{path}'.format(url=magpie_url, path=action_oper) + action_body = {'permission_name': perm_name} + else: + action_func = requests.delete + action_path = '{url}{path}/{perm_name}'.format(url=magpie_url, path=action_oper, perm_name=perm_name) + action_body = {} + action_path = action_path.format(resource_id=resource_id) + action_resp = action_func(action_path, json=action_body, cookies=cookies_or_session) + return action_resp + + def _apply_session(_usr_name=None, _grp_name=None): + """Apply operation using db session.""" + res = ResourceService.by_resource_id(resource_id, db_session=cookies_or_session) + if _usr_name: + usr = UserService.by_user_name(_usr_name, db_session=cookies_or_session) + if create_perm: + return create_user_resource_permission(perm_name, res, usr, db_session=cookies_or_session) + else: + return delete_user_resource_permission(perm_name, res, usr, db_session=cookies_or_session) + if _grp_name: + grp = GroupService.by_group_name(_grp_name, db_session=cookies_or_session) + if create_perm: + return create_group_resource_permission(perm_name, res, grp, db_session=cookies_or_session) + else: + return delete_group_resource_permission(perm_name, res, grp, db_session=cookies_or_session) + + def _validate_response(_resp): + """Validate action/operation applied.""" + if _resp is None: + return + if create_perm: + if _resp.status_code == 201: + warn_permission("Permission successfully created.", entry_index, level=logging.INFO) + elif _resp.status_code == 409: + warn_permission("Permission already exists.", entry_index, level=logging.INFO) + else: + warn_permission("Unknown response [{}]".format(_resp.status_code), + entry_index, permission=permission_config_entry, level=logging.ERROR) + else: + if _resp.status_code == 200: + warn_permission("Permission successfully removed.", entry_index, level=logging.INFO) + elif _resp.status_code == 404: + warn_permission("Permission already removed.", entry_index, level=logging.INFO) + else: + warn_permission("Unknown response [{}]".format(_resp.status_code), + entry_index, permission=permission_config_entry, level=logging.ERROR) + + create_perm = permission_config_entry['action'] == 'create' + perm_name = permission_config_entry['permission'] + usr_name = permission_config_entry.get('user') + grp_name = permission_config_entry.get('group') + + if use_request(cookies_or_session): + _validate_response(_apply_request(usr_name, None)) + _validate_response(_apply_request(None, grp_name)) + else: + _validate_response(_apply_session(usr_name, None)) + _validate_response(_apply_session(None, grp_name)) + + +def magpie_register_permissions_from_config(permissions_config, magpie_url=None, db_session=None): + # type: (Union[AnyStr, ConfigDict], Optional[AnyStr], Optional[Session]) -> None """ Applies permissions specified in configuration. + :param permissions_config: file path to 'permissions' config or JSON/YAML equivalent pre-loaded. + :param magpie_url: URL to magpie instance (when using requests; default: `magpie.url` from this app's config). + :param db_session: db session to use instead of requests to directly create/remove permissions with config. + .. seealso:: `magpie/permissions.cfg` for specific parameter or operational details. """ - permissions = _load_config(permissions_config, 'permissions') if not permissions: LOGGER.warning("Permissions configuration are empty.") return - magpie_url = get_magpie_url() - admin_usr = get_constant('MAGPIE_ADMIN_USER') - admin_pwd = get_constant('MAGPIE_ADMIN_PASSWORD') - body = {'user_name': admin_usr, 'password': admin_pwd} - resp = requests.post(magpie_url + SigninAPI.path, json=body) - if resp.status_code != 200: - raise_log("Cannot register Magpie permissions without proper credentials.") - admin_cookies = resp.cookies + if use_request(db_session): + magpie_url = magpie_url or get_magpie_url() + logging.debug("Editing permissions using requests to [{}]...".format(magpie_url)) + + admin_usr = get_constant('MAGPIE_ADMIN_USER') + admin_pwd = get_constant('MAGPIE_ADMIN_PASSWORD') + body = {'user_name': admin_usr, 'password': admin_pwd} + resp = requests.post(magpie_url + SigninAPI.path, json=body) + if resp.status_code != 200: + raise_log("Cannot register Magpie permissions without proper credentials.") + cookies_or_session = resp.cookies + else: + logging.debug("Editing permissions using db session...") + cookies_or_session = db_session logging.info("Found {} permissions to update.".format(len(permissions))) for i, perm in enumerate(permissions): @@ -605,52 +732,24 @@ def magpie_register_permissions_from_config(permissions_config): if perm['action'] not in ['create', 'remove']: warn_permission("Unknown action [{!s}]".format(perm['action']), i) continue + + # retrieve service for permissions validation svc_name = perm['service'] - svc_path = magpie_url + ServiceAPI.path.format(service_name=svc_name) - svc_resp = requests.get(svc_path, cookies=admin_cookies) - if svc_resp.status_code != 200: - warn_permission("Unknown service [{!s}]".format(svc_name), i) - continue - service_info = svc_resp.json()[svc_name] + if use_request(cookies_or_session): + svc_path = magpie_url + ServiceAPI.path.format(service_name=svc_name) + svc_resp = requests.get(svc_path, cookies=cookies_or_session) + if svc_resp.status_code != 200: + warn_permission("Unknown service [{!s}]".format(svc_name), i) + continue + service_info = svc_resp.json()[svc_name] + else: + service_info = format_service(models.Service.by_service_name(svc_name, db_session)) # apply permission config - resource, found = parse_resource_path(perm, i, service_info, admin_cookies) + resource_id, found = parse_resource_path(perm, i, service_info, cookies_or_session, magpie_url) if not found: continue - if not resource: - resource = service_info['resource_id'] - perm_paths = list() - if usr_name: - perm_paths.append(UserResourcePermissionsAPI.path.replace('{user_name}', usr_name)) - if grp_name: - perm_paths.append(GroupResourcePermissionsAPI.path.replace('{group_name}', grp_name)) - for path in perm_paths: - create_perm = perm['action'] == 'create' - if create_perm: - action_func = requests.post - action_path = '{url}{path}'.format(url=magpie_url, path=path) - action_body = {'permission_name': perm['permission']} - else: - action_func = requests.delete - action_path = '{url}{path}/{perm_name}'.format(url=magpie_url, path=path, perm_name=perm['permission']) - action_body = {} - action_path = action_path.format(resource_id=resource) - action_resp = action_func(action_path, json=action_body, cookies=admin_cookies) - action_code = action_resp.status_code - if create_perm: - if action_code == 201: - warn_permission("Permission successfully created.", i, level=logging.INFO) - elif action_code == 409: - warn_permission("Permission already exists: {!s}".format(perm), i, level=logging.INFO) - else: - warn_permission("Unknown response for 'create' permission [{}]: {!s}" - .format(action_code, perm), i, level=logging.ERROR) - else: - if action_code == 200: - warn_permission("Permission successfully removed.", i, level=logging.INFO) - elif action_code == 404: - warn_permission("Permission successfully removed.", i, level=logging.INFO) - else: - warn_permission("Unknown response for 'remove' permission [{}]: {!s}" - .format(action_code, perm), i, level=logging.ERROR) + if not resource_id: + resource_id = service_info['resource_id'] + apply_permission_entry(perm, i, resource_id, cookies_or_session, magpie_url) logging.info("Done processing permissions.") From b46232eff07f9258f735e6a5150aa0642049d6c2 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Mon, 28 Jan 2019 18:52:29 -0500 Subject: [PATCH 05/76] adjust common methods --- magpie/adapter/magpieowssecurity.py | 10 +- magpie/adapter/magpieprocess.py | 2 +- magpie/adapter/magpieservice.py | 2 +- magpie/adapter/utils.py | 36 -------- magpie/api/api_except.py | 19 +--- .../api/management/service/service_formats.py | 2 +- magpie/common.py | 16 ++++ magpie/register.py | 91 ++++++++----------- magpie/utils.py | 66 ++++++++++++++ tests/interfaces.py | 8 +- tests/utils.py | 36 ++++---- 11 files changed, 154 insertions(+), 134 deletions(-) delete mode 100644 magpie/adapter/utils.py create mode 100644 magpie/utils.py diff --git a/magpie/adapter/magpieowssecurity.py b/magpie/adapter/magpieowssecurity.py index 6ee61b17c..9701de516 100644 --- a/magpie/adapter/magpieowssecurity.py +++ b/magpie/adapter/magpieowssecurity.py @@ -1,10 +1,10 @@ -from magpie.definitions.twitcher_definitions import * -from magpie.definitions.pyramid_definitions import * -from magpie.services import service_factory -from magpie.models import Service from magpie.api.api_except import evaluate_call, verify_param -from magpie.adapter.utils import get_magpie_url from magpie.constants import get_constant +from magpie.definitions.pyramid_definitions import * +from magpie.definitions.twitcher_definitions import * +from magpie.models import Service +from magpie.services import service_factory +from magpie.utils import get_magpie_url from requests.cookies import RequestsCookieJar from six.moves.urllib.parse import urlparse import requests diff --git a/magpie/adapter/magpieprocess.py b/magpie/adapter/magpieprocess.py index 89f89965a..67bc6dd51 100644 --- a/magpie/adapter/magpieprocess.py +++ b/magpie/adapter/magpieprocess.py @@ -1,7 +1,7 @@ """ Store adapters to read data from magpie. """ -from magpie.adapter.utils import get_magpie_url, get_admin_cookies +from magpie.utils import get_magpie_url, get_admin_cookies from magpie.api.api_except import raise_http from magpie.constants import get_constant from magpie.definitions.pyramid_definitions import ( diff --git a/magpie/adapter/magpieservice.py b/magpie/adapter/magpieservice.py index 1304cda82..a14e4c3e8 100644 --- a/magpie/adapter/magpieservice.py +++ b/magpie/adapter/magpieservice.py @@ -4,7 +4,7 @@ from magpie.definitions.twitcher_definitions import * from magpie.definitions.pyramid_definitions import HTTPOk, asbool -from magpie.adapter.utils import get_admin_cookies, get_magpie_url +from magpie.utils import get_admin_cookies, get_magpie_url import requests import logging LOGGER = logging.getLogger("TWITCHER") diff --git a/magpie/adapter/utils.py b/magpie/adapter/utils.py deleted file mode 100644 index b42b7f3ae..000000000 --- a/magpie/adapter/utils.py +++ /dev/null @@ -1,36 +0,0 @@ -from magpie.constants import get_constant -from magpie.definitions.pyramid_definitions import HTTPOk, ConfigurationError, Registry -from six.moves.urllib.parse import urlparse -from typing import Dict -import requests -import logging -LOGGER = logging.getLogger("TWITCHER") - - -def get_admin_cookies(magpie_url, verify=True): - # type: (str, bool) -> Dict[str,str] - magpie_login_url = '{}/signin'.format(magpie_url) - cred = {'user_name': get_constant('MAGPIE_ADMIN_USER'), 'password': get_constant('MAGPIE_ADMIN_PASSWORD')} - resp = requests.post(magpie_login_url, data=cred, headers={'Accept': 'application/json'}, verify=verify) - if resp.status_code != HTTPOk.code: - raise resp.raise_for_status() - token_name = get_constant('MAGPIE_COOKIE_NAME') - return {token_name: resp.cookies.get(token_name)} - - -def get_magpie_url(registry): - # type: (Registry) -> str - try: - # add 'http' scheme to url if omitted from config since further 'requests' calls fail without it - # mostly for testing when only 'localhost' is specified - # otherwise twitcher config should explicitly define it in MAGPIE_URL - url_parsed = urlparse(registry.settings.get('magpie.url').strip('/')) - if url_parsed.scheme in ['http', 'https']: - return url_parsed.geturl() - else: - magpie_url = 'http://{}'.format(url_parsed.geturl()) - LOGGER.warn("Missing scheme from registry url, new value: '{}'".format(magpie_url)) - return magpie_url - except AttributeError: - # If magpie.url does not exist, calling strip fct over None will raise this issue - raise ConfigurationError('magpie.url config cannot be found') diff --git a/magpie/api/api_except.py b/magpie/api/api_except.py index ecb0659a0..3836e490f 100644 --- a/magpie/api/api_except.py +++ b/magpie/api/api_except.py @@ -1,7 +1,6 @@ +from magpie.common import islambda, isclass from pyramid.httpexceptions import * from sys import exc_info -import types -import six # control variables to avoid infinite recursion in case of # major programming error to avoid application hanging @@ -270,7 +269,7 @@ def validate_params(httpClass, httpBase, detail, content, contentType): contentType='application/json', content={u'caller': {u'content': content, u'detail': detail, - u'code': 520, #'unknown' error + u'code': 520, # 'unknown' error u'type': contentType}}) # if `httpClass` derives from `httpBase` (ex: `HTTPSuccessful` or `HTTPError`) it is of proper requested type # if it derives from `HTTPException`, it *could* be different than base (ex: 2xx instead of 4xx codes) @@ -365,17 +364,3 @@ def generate_response_http_format(httpClass, httpKWArgs, jsonContent, outputType u'caller': {u'httpKWArgs': repr(httpKWArgs), u'httpClass': repr(httpClass), u'outputType': str(outputType)}}) - - -def islambda(func): - return isinstance(func, types.LambdaType) and func.__name__ == (lambda: None).__name__ - - -def isclass(obj): - """ - Evaluate an object for class type (ie: class definition, not an instance nor any other type). - - :param obj: object to evaluate for class type - :return: (bool) indicating if `object` is a class - """ - return isinstance(obj, (type, six.class_types)) diff --git a/magpie/api/management/service/service_formats.py b/magpie/api/management/service/service_formats.py index 4b5e1f4a7..10688ca79 100644 --- a/magpie/api/management/service/service_formats.py +++ b/magpie/api/management/service/service_formats.py @@ -1,4 +1,4 @@ -from magpie.register import get_twitcher_protected_service_url +from magpie.utils import get_twitcher_protected_service_url from magpie.services import service_type_dict from magpie.definitions.pyramid_definitions import * from magpie.api.api_except import evaluate_call diff --git a/magpie/common.py b/magpie/common.py index e53bbd429..f05e89baa 100644 --- a/magpie/common.py +++ b/magpie/common.py @@ -3,6 +3,8 @@ from __future__ import unicode_literals from distutils.dir_util import mkpath import logging +import types +import six import os LOGGER = logging.getLogger(__name__) @@ -26,6 +28,20 @@ def str2bool(value): return True if value in ['on', 'true', 'True', True] else False +def islambda(func): + return isinstance(func, types.LambdaType) and func.__name__ == (lambda: None).__name__ + + +def isclass(obj): + """ + Evaluate an object for class type (ie: class definition, not an instance nor any other type). + + :param obj: object to evaluate for class type + :return: (bool) indicating if `object` is a class + """ + return isinstance(obj, (type, six.class_types)) + + # alternative to 'makedirs' with 'exists_ok' parameter only available for python>3.5 def make_dirs(path): dir_path = os.path.dirname(path) diff --git a/magpie/register.py b/magpie/register.py index 0cbaadd25..264c1f67d 100644 --- a/magpie/register.py +++ b/magpie/register.py @@ -7,11 +7,7 @@ GroupResourcePermissionsAPI, UserResourcePermissionsAPI, ) -from magpie.api.management.service.service_formats import format_service_resources, format_service -from magpie.api.management.resource.resource_utils import create_resource -from magpie.api.management.user.user_utils import create_user_resource_permission, delete_user_resource_permission -from magpie.api.management.group.group_utils import create_group_resource_permission, delete_group_resource_permission -from magpie.common import make_dirs, print_log, raise_log, bool2str +from magpie.common import make_dirs, print_log, raise_log, bool2str, islambda from magpie.constants import get_constant from magpie.definitions.ziggurat_definitions import ( ResourceService, @@ -20,9 +16,11 @@ UserResourcePermissionService, ) from magpie.definitions.sqlalchemy_definitions import Session +from magpie.definitions.pyramid_definitions import HTTPException from magpie.permissions import permissions_supported from magpie.services import service_type_dict from magpie import models +from magpie.utils import get_twitcher_protected_service_url, get_phoenix_url, get_magpie_url, get_admin_cookies from typing import AnyStr, Dict, List, Optional, Tuple, Union, TYPE_CHECKING from requests.cookies import RequestsCookieJar import os @@ -189,34 +187,6 @@ def phoenix_register_services(services_dict, allowed_service_types=None): return success, statuses -def get_phoenix_url(): - hostname = get_constant('HOSTNAME') - phoenix_port = get_constant('PHOENIX_PORT', raise_not_set=False) - return 'https://{0}{1}'.format(hostname, ':{}'.format(phoenix_port) if phoenix_port else '') - - -def get_magpie_url(): - hostname = get_constant('HOSTNAME') - magpie_port = get_constant('MAGPIE_PORT', raise_not_set=False) - return 'http://{0}{1}'.format(hostname, ':{}'.format(magpie_port) if magpie_port else '') - - -def get_twitcher_protected_service_url(magpie_service_name, hostname=None): - twitcher_proxy_url = get_constant('TWITCHER_PROTECTED_URL', raise_not_set=False) - if not twitcher_proxy_url: - twitcher_proxy = get_constant('TWITCHER_PROTECTED_PATH', raise_not_set=False) - if not twitcher_proxy.endswith('/'): - twitcher_proxy = twitcher_proxy + '/' - if not twitcher_proxy.startswith('/'): - twitcher_proxy = '/' + twitcher_proxy - if not twitcher_proxy.startswith('/twitcher'): - twitcher_proxy = '/twitcher' + twitcher_proxy - hostname = hostname or get_constant('HOSTNAME') - twitcher_proxy_url = "https://{0}{1}".format(hostname, twitcher_proxy) - twitcher_proxy_url = twitcher_proxy_url.rstrip('/') - return "{0}/{1}".format(twitcher_proxy_url, magpie_service_name) - - def register_services(where, # type: Optional[AnyStr] services_dict, # type: Dict[AnyStr, Dict[AnyStr, AnyStr]] cookies, # type: AnyStr @@ -560,6 +530,7 @@ def parse_resource_path(permission_config_entry, # type: ConfigItem res_path = get_magpie_url() + ServiceResourcesAPI.path.format(service_name=svc_name) res_dict = requests.get(res_path, cookies=cookies_or_session).json()[svc_name]['resources'] else: + from magpie.api.management.service.service_formats import format_service_resources svc = models.Service.by_service_name(svc_name, db_session=cookies_or_session) res_dict = format_service_resources(svc, db_session=cookies_or_session) parent = None @@ -584,6 +555,7 @@ def parse_resource_path(permission_config_entry, # type: ConfigItem # noinspection PyUnboundLocalVariable resp = requests.post(res_path, json=body, cookies=cookies_or_session) else: + from magpie.api.management.resource.resource_utils import create_resource resp = create_resource(res, res, res_type, parent, db_session=cookies_or_session) if resp.status_code != 201: resp.raise_for_status() @@ -632,24 +604,38 @@ def _apply_request(_usr_name=None, _grp_name=None): def _apply_session(_usr_name=None, _grp_name=None): """Apply operation using db session.""" + from magpie.api.management.user import user_utils as ut + from magpie.api.management.group import group_utils as gt + res = ResourceService.by_resource_id(resource_id, db_session=cookies_or_session) if _usr_name: usr = UserService.by_user_name(_usr_name, db_session=cookies_or_session) if create_perm: - return create_user_resource_permission(perm_name, res, usr, db_session=cookies_or_session) + return ut.create_user_resource_permission(perm_name, res, usr, db_session=cookies_or_session) else: - return delete_user_resource_permission(perm_name, res, usr, db_session=cookies_or_session) + return ut.delete_user_resource_permission(perm_name, res, usr, db_session=cookies_or_session) if _grp_name: grp = GroupService.by_group_name(_grp_name, db_session=cookies_or_session) if create_perm: - return create_group_resource_permission(perm_name, res, grp, db_session=cookies_or_session) + return gt.create_group_resource_permission(perm_name, res, grp, db_session=cookies_or_session) else: - return delete_group_resource_permission(perm_name, res, grp, db_session=cookies_or_session) + return gt.delete_group_resource_permission(perm_name, res, grp, db_session=cookies_or_session) - def _validate_response(_resp): + def _validate_response(operation): """Validate action/operation applied.""" - if _resp is None: - return + # handle HTTPException raised + if not islambda(operation): + raise Exception("invalid use of method") + try: + _resp = operation() + if _resp is None: + return + except HTTPException as exc: + _resp = exc + except Exception: + raise + + # validation according to status code returned if create_perm: if _resp.status_code == 201: warn_permission("Permission successfully created.", entry_index, level=logging.INFO) @@ -673,11 +659,11 @@ def _validate_response(_resp): grp_name = permission_config_entry.get('group') if use_request(cookies_or_session): - _validate_response(_apply_request(usr_name, None)) - _validate_response(_apply_request(None, grp_name)) + _validate_response(lambda: _apply_request(usr_name, None)) + _validate_response(lambda: _apply_request(None, grp_name)) else: - _validate_response(_apply_session(usr_name, None)) - _validate_response(_apply_session(None, grp_name)) + _validate_response(lambda: _apply_session(usr_name, None)) + _validate_response(lambda: _apply_session(None, grp_name)) def magpie_register_permissions_from_config(permissions_config, magpie_url=None, db_session=None): @@ -700,14 +686,8 @@ def magpie_register_permissions_from_config(permissions_config, magpie_url=None, if use_request(db_session): magpie_url = magpie_url or get_magpie_url() logging.debug("Editing permissions using requests to [{}]...".format(magpie_url)) - - admin_usr = get_constant('MAGPIE_ADMIN_USER') - admin_pwd = get_constant('MAGPIE_ADMIN_PASSWORD') - body = {'user_name': admin_usr, 'password': admin_pwd} - resp = requests.post(magpie_url + SigninAPI.path, json=body) - if resp.status_code != 200: - raise_log("Cannot register Magpie permissions without proper credentials.") - cookies_or_session = resp.cookies + err_msg = "Invalid credentials to register Magpie permissions." + cookies_or_session = get_admin_cookies(magpie_url, raise_message=err_msg) else: logging.debug("Editing permissions using db session...") cookies_or_session = db_session @@ -743,7 +723,12 @@ def magpie_register_permissions_from_config(permissions_config, magpie_url=None, continue service_info = svc_resp.json()[svc_name] else: - service_info = format_service(models.Service.by_service_name(svc_name, db_session)) + svc = models.Service.by_service_name(svc_name, db_session=db_session) + if not svc: + warn_permission("Unknown service [{!s}]".format(svc_name), i) + continue + from magpie.api.management.service.service_formats import format_service + service_info = format_service(svc) # apply permission config resource_id, found = parse_resource_path(perm, i, service_info, cookies_or_session, magpie_url) diff --git a/magpie/utils.py b/magpie/utils.py new file mode 100644 index 000000000..523144e8a --- /dev/null +++ b/magpie/utils.py @@ -0,0 +1,66 @@ +from magpie.common import raise_log +from magpie.constants import get_constant +from magpie.definitions.pyramid_definitions import HTTPOk, ConfigurationError, Registry +from six.moves.urllib.parse import urlparse +from typing import AnyStr, Dict, Optional +import requests +import logging +LOGGER = logging.getLogger(__name__) + + +def get_admin_cookies(magpie_url, verify=True, raise_message=None): + # type: (str, Optional[bool], Optional[AnyStr]) -> Dict[str,str] + magpie_login_url = '{}/signin'.format(magpie_url) + cred = {'user_name': get_constant('MAGPIE_ADMIN_USER'), 'password': get_constant('MAGPIE_ADMIN_PASSWORD')} + resp = requests.post(magpie_login_url, data=cred, headers={'Accept': 'application/json'}, verify=verify) + if resp.status_code != HTTPOk.code: + if raise_message: + raise_log(raise_message, logger=LOGGER) + raise resp.raise_for_status() + token_name = get_constant('MAGPIE_COOKIE_NAME') + return {token_name: resp.cookies.get(token_name)} + + +def get_magpie_url(registry=None): + # type: (Optional[Registry]) -> str + if registry is None: + LOGGER.warning("Registry not specified, trying to find Magpie URL from environment") + hostname = get_constant('HOSTNAME') + magpie_port = get_constant('MAGPIE_PORT', raise_not_set=False) + return 'http://{0}{1}'.format(hostname, ':{}'.format(magpie_port) if magpie_port else '') + try: + # add 'http' scheme to url if omitted from config since further 'requests' calls fail without it + # mostly for testing when only 'localhost' is specified + # otherwise twitcher config should explicitly define it in MAGPIE_URL + url_parsed = urlparse(registry.settings.get('magpie.url').strip('/')) + if url_parsed.scheme in ['http', 'https']: + return url_parsed.geturl() + else: + magpie_url = 'http://{}'.format(url_parsed.geturl()) + LOGGER.warning("Missing scheme from registry url, new value: '{}'".format(magpie_url)) + return magpie_url + except AttributeError: + # If magpie.url does not exist, calling strip fct over None will raise this issue + raise ConfigurationError('magpie.url config cannot be found') + + +def get_phoenix_url(): + hostname = get_constant('HOSTNAME') + phoenix_port = get_constant('PHOENIX_PORT', raise_not_set=False) + return 'https://{0}{1}'.format(hostname, ':{}'.format(phoenix_port) if phoenix_port else '') + + +def get_twitcher_protected_service_url(magpie_service_name, hostname=None): + twitcher_proxy_url = get_constant('TWITCHER_PROTECTED_URL', raise_not_set=False) + if not twitcher_proxy_url: + twitcher_proxy = get_constant('TWITCHER_PROTECTED_PATH', raise_not_set=False) + if not twitcher_proxy.endswith('/'): + twitcher_proxy = twitcher_proxy + '/' + if not twitcher_proxy.startswith('/'): + twitcher_proxy = '/' + twitcher_proxy + if not twitcher_proxy.startswith('/twitcher'): + twitcher_proxy = '/twitcher' + twitcher_proxy + hostname = hostname or get_constant('HOSTNAME') + twitcher_proxy_url = "https://{0}{1}".format(hostname, twitcher_proxy) + twitcher_proxy_url = twitcher_proxy_url.rstrip('/') + return "{0}/{1}".format(twitcher_proxy_url, magpie_service_name) diff --git a/tests/interfaces.py b/tests/interfaces.py index e8b2f780a..f00944be2 100644 --- a/tests/interfaces.py +++ b/tests/interfaces.py @@ -7,7 +7,7 @@ from magpie.api.api_rest_schemas import SwaggerGenerator from magpie.constants import get_constant from magpie.services import service_type_dict -from magpie.register import get_twitcher_protected_service_url +from magpie.utils import get_twitcher_protected_service_url from tests import utils, runner @@ -889,8 +889,8 @@ def test_PostServiceResources_DirectResource_Conflict(self): cookies=self.cookies, json=data, expect_errors=True) json_body = utils.check_response_basic_info(resp, 409, expected_method='POST') utils.check_error_param_structure(json_body, version=self.version, - isParamValueLiteralUnicode=True, paramCompareExists=True, - paramValue=self.test_resource_name, paramName=u'resource_name') + is_param_value_literal_unicode=True, param_compare_exists=True, + param_value=self.test_resource_name, param_name=u'resource_name') @pytest.mark.services @pytest.mark.defaults @@ -998,7 +998,7 @@ def test_PostResources_MissingParentID(self): resp = utils.test_request(self.url, 'POST', '/resources', headers=self.json_headers, cookies=self.cookies, data=data, expect_errors=True) json_body = utils.check_response_basic_info(resp, 422, expected_method='POST') - utils.check_error_param_structure(json_body, paramName='parent_id', paramValue=repr(None), version=self.version) + utils.check_error_param_structure(json_body, param_name='parent_id', param_value=repr(None), version=self.version) @pytest.mark.resources @unittest.skipUnless(runner.MAGPIE_TEST_RESOURCES, reason=runner.MAGPIE_TEST_DISABLED_MESSAGE('resources')) diff --git a/tests/utils.py b/tests/utils.py index 1c29f9644..962ade9f5 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -63,6 +63,8 @@ def test_request(app_or_url, method, path, timeout=5, allow_redirects=True, **kw :param app_or_url: `webtest.TestApp` instance of the test application or remote server URL to call with `requests` :param method: request method (GET, POST, PUT, DELETE) :param path: test path starting at base path + :param timeout: `timeout` to pass down to `request` + :param allow_redirects: `allow_redirects` to pass down to `request` :return: response of the request """ method = method.upper() @@ -91,6 +93,7 @@ def test_request(app_or_url, method, path, timeout=5, allow_redirects=True, **kw kwargs.update({'params': json.dumps(json_body, cls=json.JSONEncoder)}) if status and status >= 300: kwargs.update({'expect_errors': True}) + # noinspection PyProtectedMember resp = app_or_url._gen_request(method, path, **kwargs) # automatically follow the redirect if any and evaluate its response max_redirect = kwargs.get('max_redirects', 5) @@ -263,16 +266,16 @@ def __repr__(self): Null = null() -def check_error_param_structure(json_body, paramValue=Null, paramName=Null, paramCompare=Null, - isParamValueLiteralUnicode=False, paramCompareExists=False, version=None): +def check_error_param_structure(json_body, param_value=Null, param_name=Null, param_compare=Null, + is_param_value_literal_unicode=False, param_compare_exists=False, version=None): """ Validates error response 'param' information based on different Magpie version formats. :param json_body: json body of the response to validate. - :param paramValue: expected 'value' of param, not verified if - :param paramName: expected 'name' of param, not verified if or non existing for Magpie version - :param paramCompare: expected 'compare'/'paramCompare' value, not verified if - :param isParamValueLiteralUnicode: param value is represented as `u'{paramValue}'` for older Magpie version - :param paramCompareExists: verify that 'compare'/'paramCompare' is in the body, not necessarily validating the value + :param param_value: expected 'value' of param, not verified if + :param param_name: expected 'name' of param, not verified if or non existing for Magpie version + :param param_compare: expected 'compare'/'paramCompare' value, not verified if + :param is_param_value_literal_unicode: param value is represented as `u'{paramValue}'` for older Magpie version + :param param_compare_exists: verify that 'compare'/'paramCompare' is in the body, not validating its actual value :param version: version of application/remote server to use for format validation, use local Magpie version if None :raise failing condition """ @@ -283,19 +286,19 @@ def check_error_param_structure(json_body, paramValue=Null, paramName=Null, para check_val_type(json_body['param'], dict) check_val_is_in('value', json_body['param']) check_val_is_in('name', json_body['param']) - check_val_equal(json_body['param']['name'], paramName) - check_val_equal(json_body['param']['value'], paramValue) - if paramCompareExists: + check_val_equal(json_body['param']['name'], param_name) + check_val_equal(json_body['param']['value'], param_value) + if param_compare_exists: check_val_is_in('compare', json_body['param']) - check_val_equal(json_body['param']['compare'], paramCompare) + check_val_equal(json_body['param']['compare'], param_compare) else: # unicode representation was explicitly returned in value only when of string type - if isParamValueLiteralUnicode and isinstance(paramValue, six.string_types): - paramValue = u'u\'{}\''.format(paramValue) - check_val_equal(json_body['param'], paramValue) - if paramCompareExists: + if is_param_value_literal_unicode and isinstance(param_value, six.string_types): + param_value = u'u\'{}\''.format(param_value) + check_val_equal(json_body['param'], param_value) + if param_compare_exists: check_val_is_in('paramCompare', json_body) - check_val_equal(json_body['paramCompare'], paramCompare) + check_val_equal(json_body['paramCompare'], param_compare) def check_post_resource_structure(json_body, resource_name, resource_type, resource_display_name, version=None): @@ -362,6 +365,7 @@ def check_resource_children(resource_dict, parent_resource_id, root_service_id): # Generic setup and validation methods across unittests +# noinspection PyPep8Naming class TestSetup(object): @staticmethod def get_Version(test_class): From 08f15c5084338cf900aaf0c63655fcd78ed42b2b Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Mon, 28 Jan 2019 19:35:45 -0500 Subject: [PATCH 06/76] adjust tests (dont rely on existing service) + version 0.9.0 --- HISTORY.rst | 10 +++++++--- magpie/__meta__.py | 2 +- tests/interfaces.py | 22 ++++++++++++++++------ tests/utils.py | 39 ++++++++++++++++++++++++++++++++------- 4 files changed, 56 insertions(+), 17 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 6ef0c1b8e..fc4a2f0f0 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -3,14 +3,18 @@ History ======= +0.9.x +--------------------- + +* add permissions config to auto-generate user/group rules on startup +* fix many invalid or erroneous swagger specifications +* attempt db creation on first migration if not existing + 0.8.x --------------------- * update MagpieAdapter to match process store changes * provide user ID on API routes returning user info -* attempt db creation on first migration if not existing -* add permissions config to auto-generate user/group rules on startup -* fix many invalid or erroneous swagger specifications 0.7.x --------------------- diff --git a/magpie/__meta__.py b/magpie/__meta__.py index 1d9801098..d83a49765 100644 --- a/magpie/__meta__.py +++ b/magpie/__meta__.py @@ -2,7 +2,7 @@ General meta information on the magpie package. """ -__version__ = '0.8.3' +__version__ = '0.9.0' __author__ = "Francois-Xavier Derue, Francis Charette-Migneault" __maintainer__ = "Francis Charette-Migneault" __email__ = 'francis.charette-migneault@crim.ca' diff --git a/tests/interfaces.py b/tests/interfaces.py index f00944be2..2dc2d31d9 100644 --- a/tests/interfaces.py +++ b/tests/interfaces.py @@ -106,6 +106,7 @@ def tearDownClass(cls): def tearDown(self): utils.TestSetup.delete_TestServiceResource(self) + utils.TestSetup.delete_TestService(self) utils.TestSetup.delete_TestUser(self) @classmethod @@ -125,8 +126,16 @@ def setup_test_values(cls): if provider_services_info[svc_name]['type'] in possible_service_types: cls.test_services_info[svc_name] = provider_services_info[svc_name] - cls.test_service_name = u'project-api' - cls.test_service_type = cls.test_services_info[cls.test_service_name]['type'] + cls.test_service_name = u'magpie-unittest-service-api' + cls.test_service_type = u'api' + data = { + u'service_name': cls.test_service_name, + u'service_type': cls.test_service_type, + u'service_url': u'http://localhost:9000/dummy-api' + } + resp = utils.test_request(cls.url, 'POST', '/services', json=data, + headers=cls.json_headers, cookies=cls.cookies) + utils.check_response_basic_info(resp, 201, expected_method='POST') resp = utils.test_request(cls.url, 'GET', '/services/{}'.format(cls.test_service_name), headers=cls.json_headers, cookies=cls.cookies) @@ -149,6 +158,7 @@ def setup_test_values(cls): def setUp(self): self.check_requirements() utils.TestSetup.delete_TestServiceResource(self) + utils.TestSetup.delete_TestService(self) utils.TestSetup.delete_TestUser(self) def test_GetAPI(self): @@ -799,7 +809,7 @@ def test_GetServicePermissions(self): @pytest.mark.services @unittest.skipUnless(runner.MAGPIE_TEST_SERVICES, reason=runner.MAGPIE_TEST_DISABLED_MESSAGE('services')) def test_PostServiceResources_DirectResource_NoParentID(self): - resources_prior = utils.TestSetup.get_ExistingTestServiceDirectResources(self) + resources_prior = utils.TestSetup.get_TestServiceDirectResources(self) resources_prior_ids = [res['resource_id'] for res in resources_prior] json_body = utils.TestSetup.create_TestServiceResource(self) if LooseVersion(self.version) >= LooseVersion('0.6.3'): @@ -815,7 +825,7 @@ def test_PostServiceResources_DirectResource_NoParentID(self): @pytest.mark.services @unittest.skipUnless(runner.MAGPIE_TEST_SERVICES, reason=runner.MAGPIE_TEST_DISABLED_MESSAGE('services')) def test_PostServiceResources_DirectResource_WithParentID(self): - resources_prior = utils.TestSetup.get_ExistingTestServiceDirectResources(self) + resources_prior = utils.TestSetup.get_TestServiceDirectResources(self) resources_prior_ids = [res['resource_id'] for res in resources_prior] service_id = utils.TestSetup.get_ExistingTestServiceInfo(self)['resource_id'] extra_data = {"parent_id": service_id} @@ -835,7 +845,7 @@ def test_PostServiceResources_DirectResource_WithParentID(self): def test_PostServiceResources_ChildrenResource_ParentID(self): # create the direct resource json_body = utils.TestSetup.create_TestServiceResource(self) - resources = utils.TestSetup.get_ExistingTestServiceDirectResources(self) + resources = utils.TestSetup.get_TestServiceDirectResources(self) resources_ids = [res['resource_id'] for res in resources] if LooseVersion(self.version) >= LooseVersion('0.6.3'): test_resource_id = json_body['resource']['resource_id'] @@ -1012,7 +1022,7 @@ def test_DeleteResource(self): route = '/resources/{res_id}'.format(res_id=resource_id) resp = utils.test_request(self.url, 'DELETE', route, headers=self.json_headers, cookies=self.cookies) utils.check_response_basic_info(resp, 200, expected_method='DELETE') - utils.TestSetup.check_NonExistingTestResource(self) + utils.TestSetup.check_NonExistingTestServiceResource(self) @pytest.mark.ui diff --git a/tests/utils.py b/tests/utils.py index 962ade9f5..010e9ec20 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -424,28 +424,33 @@ def create_TestServiceResource(test_class, data_override=None): @staticmethod def get_ExistingTestServiceInfo(test_class): route = '/services/{svc}'.format(svc=test_class.test_service_name) - resp = test_request(test_class.url, 'GET', route, headers=test_class.json_headers, cookies=test_class.cookies) + resp = test_request(test_class.url, 'GET', route, + headers=test_class.json_headers, cookies=test_class.cookies) json_body = get_json_body(resp) return json_body[test_class.test_service_name] @staticmethod - def get_ExistingTestServiceDirectResources(test_class): + def get_TestServiceDirectResources(test_class, ignore_missing_service=False): route = '/services/{svc}/resources'.format(svc=test_class.test_service_name) - resp = test_request(test_class.url, 'GET', route, headers=test_class.json_headers, cookies=test_class.cookies) + resp = test_request(test_class.url, 'GET', route, + headers=test_class.json_headers, cookies=test_class.cookies, + expect_errors=ignore_missing_service) + if ignore_missing_service and resp.status_code == 404: + return [] json_body = get_json_body(resp) resources = json_body[test_class.test_service_name]['resources'] return [resources[res] for res in resources] @staticmethod - def check_NonExistingTestResource(test_class): - resources = TestSetup.get_ExistingTestServiceDirectResources(test_class) + def check_NonExistingTestServiceResource(test_class): + resources = TestSetup.get_TestServiceDirectResources(test_class, ignore_missing_service=True) resources_names = [res['resource_name'] for res in resources] check_val_not_in(test_class.test_resource_name, resources_names) @staticmethod def delete_TestServiceResource(test_class, override_resource_name=None): resource_name = override_resource_name or test_class.test_resource_name - resources = TestSetup.get_ExistingTestServiceDirectResources(test_class) + resources = TestSetup.get_TestServiceDirectResources(test_class, ignore_missing_service=True) test_resource = filter(lambda r: r['resource_name'] == resource_name, resources) # delete as required, skip if non-existing if len(test_resource) > 0: @@ -455,7 +460,27 @@ def delete_TestServiceResource(test_class, override_resource_name=None): headers=test_class.json_headers, cookies=test_class.cookies) check_val_equal(resp.status_code, 200) - TestSetup.check_NonExistingTestResource(test_class) + TestSetup.check_NonExistingTestServiceResource(test_class) + + @staticmethod + def check_NonExistingTestService(test_class): + services_info = TestSetup.get_RegisteredServicesList(test_class) + services_names = [svc['service_name'] for svc in services_info] + check_val_not_in(test_class.test_service_name, services_names) + + @staticmethod + def delete_TestService(test_class, override_service_name=None): + service_name = override_service_name or test_class.test_service_name + services_info = TestSetup.get_RegisteredServicesList(test_class) + test_service = filter(lambda r: r['service_name'] == service_name, services_info) + # delete as required, skip if non-existing + if len(test_service) > 0: + route = '/services/{svc_name}'.format(svc_name=test_class.test_service_name) + resp = test_request(test_class.url, 'DELETE', route, + headers=test_class.json_headers, + cookies=test_class.cookies) + check_val_equal(resp.status_code, 200) + TestSetup.check_NonExistingTestService(test_class) @staticmethod def get_RegisteredServicesList(test_class): From cd7ba2d221fbccffdb1ed996ab58345d2596b104 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Tue, 29 Jan 2019 15:00:56 -0500 Subject: [PATCH 07/76] add user/group creation for permissions config --- magpie/api/management/group/group_utils.py | 21 ++++++++++ magpie/api/management/group/group_views.py | 13 +----- magpie/api/management/user/user_utils.py | 16 +++++--- magpie/register.py | 46 +++++++++++++++++----- permissions.cfg | 3 +- 5 files changed, 70 insertions(+), 29 deletions(-) diff --git a/magpie/api/management/group/group_utils.py b/magpie/api/management/group/group_utils.py index 3b3e5b55b..2ec91a40d 100644 --- a/magpie/api/management/group/group_utils.py +++ b/magpie/api/management/group/group_utils.py @@ -37,6 +37,27 @@ def get_group_resources(group, db_session): return json_response +def create_group(group_name, db_session): + # type: (AnyStr, Session) -> models.Group + """ + Creates a group if it is permitted and not conflicting. + :returns: corresponding HTTP response according to the encountered situation. + """ + group = GroupService.by_group_name(group_name, db_session=db_session) + group_content_error = {u'group_name': str(group_name)} + verify_param(group, isNone=True, httpError=HTTPConflict, withParam=False, + msgOnFail=Groups_POST_ConflictResponseSchema.description, content=group_content_error) + # noinspection PyArgumentList + new_group = evaluate_call(lambda: models.Group(group_name=group_name), fallback=lambda: db_session.rollback(), + httpError=HTTPForbidden, msgOnFail=Groups_POST_ForbiddenCreateResponseSchema.description, + content=group_content_error) + evaluate_call(lambda: db_session.add(new_group), fallback=lambda: db_session.rollback(), + httpError=HTTPForbidden, msgOnFail=Groups_POST_ForbiddenAddResponseSchema.description, + content=group_content_error) + return valid_http(httpSuccess=HTTPCreated, detail=Groups_POST_CreatedResponseSchema.description, + content={u'group': format_group(new_group, basic_info=True)}) + + def create_group_resource_permission(permission_name, resource, group, db_session): # type: (AnyStr, models.Resource, models.Group, Session) -> HTTPException """ diff --git a/magpie/api/management/group/group_views.py b/magpie/api/management/group/group_views.py index da26e6543..b37f032b9 100644 --- a/magpie/api/management/group/group_views.py +++ b/magpie/api/management/group/group_views.py @@ -19,18 +19,7 @@ def get_groups(request): def create_group(request): """Create a group.""" group_name = get_value_multiformat_post_checked(request, 'group_name') - group = GroupService.by_group_name(group_name, db_session=request.db) - group_content_error = {u'group_name': str(group_name)} - verify_param(group, isNone=True, httpError=HTTPConflict, withParam=False, - msgOnFail=Groups_POST_ConflictResponseSchema.description, content=group_content_error) - new_group = evaluate_call(lambda: models.Group(group_name=group_name), fallback=lambda: request.db.rollback(), - httpError=HTTPForbidden, msgOnFail=Groups_POST_ForbiddenCreateResponseSchema.description, - content=group_content_error) - evaluate_call(lambda: request.db.add(new_group), fallback=lambda: request.db.rollback(), - httpError=HTTPForbidden, msgOnFail=Groups_POST_ForbiddenAddResponseSchema.description, - content=group_content_error) - return valid_http(httpSuccess=HTTPCreated, detail=Groups_POST_CreatedResponseSchema.description, - content={u'group': format_group(new_group, basic_info=True)}) + return create_group(group_name, request.db) @GroupAPI.get(tags=[GroupsTag], response_schemas=Group_GET_responses) diff --git a/magpie/api/management/user/user_utils.py b/magpie/api/management/user/user_utils.py index 4a6b7e5da..ced4f4bbe 100644 --- a/magpie/api/management/user/user_utils.py +++ b/magpie/api/management/user/user_utils.py @@ -12,16 +12,20 @@ def create_user(user_name, password, email, group_name, db_session): - db = db_session + # type: (AnyStr, AnyStr, AnyStr, AnyStr, Session) -> HTTPException + """ + Creates a user if it is permitted and not conflicting. + :returns: corresponding HTTP response according to the encountered situation. + """ # Check that group already exists - group_check = evaluate_call(lambda: GroupService.by_group_name(group_name, db_session=db), + group_check = evaluate_call(lambda: GroupService.by_group_name(group_name, db_session=db_session), httpError=HTTPForbidden, msgOnFail=UserGroup_GET_ForbiddenResponseSchema.description) verify_param(group_check, notNone=True, httpError=HTTPNotAcceptable, msgOnFail=UserGroup_Check_ForbiddenResponseSchema.description) # Check if user already exists - user_check = evaluate_call(lambda: UserService.by_user_name(user_name=user_name, db_session=db), + user_check = evaluate_call(lambda: UserService.by_user_name(user_name=user_name, db_session=db_session), httpError=HTTPForbidden, msgOnFail=User_Check_ForbiddenResponseSchema.description) verify_param(user_check, isNone=True, httpError=HTTPConflict, msgOnFail=User_Check_ConflictResponseSchema.description) @@ -32,16 +36,16 @@ def create_user(user_name, password, email, group_name, db_session): if password: UserService.set_password(new_user, password) UserService.regenerate_security_code(new_user) - evaluate_call(lambda: db.add(new_user), fallback=lambda: db.rollback(), + evaluate_call(lambda: db_session.add(new_user), fallback=lambda: db_session.rollback(), httpError=HTTPForbidden, msgOnFail=Users_POST_ForbiddenResponseSchema.description) # Fetch user to update fields - new_user = evaluate_call(lambda: UserService.by_user_name(user_name, db_session=db), + new_user = evaluate_call(lambda: UserService.by_user_name(user_name, db_session=db_session), httpError=HTTPForbidden, msgOnFail=UserNew_POST_ForbiddenResponseSchema.description) # Assign user to group # noinspection PyArgumentList group_entry = models.UserGroup(group_id=group_check.id, user_id=new_user.id) - evaluate_call(lambda: db.add(group_entry), fallback=lambda: db.rollback(), + evaluate_call(lambda: db_session.add(group_entry), fallback=lambda: db_session.rollback(), httpError=HTTPForbidden, msgOnFail=UserGroup_GET_ForbiddenResponseSchema.description) return valid_http(httpSuccess=HTTPCreated, detail=Users_POST_CreatedResponseSchema.description, diff --git a/magpie/register.py b/magpie/register.py index 264c1f67d..0f37fd783 100644 --- a/magpie/register.py +++ b/magpie/register.py @@ -4,6 +4,8 @@ ServicesAPI, ServiceAPI, ServiceResourcesAPI, + GroupsAPI, + UsersAPI, GroupResourcePermissionsAPI, UserResourcePermissionsAPI, ) @@ -621,7 +623,28 @@ def _apply_session(_usr_name=None, _grp_name=None): else: return gt.delete_group_resource_permission(perm_name, res, grp, db_session=cookies_or_session) - def _validate_response(operation): + def _apply_profile(_usr_name=None, _grp_name=None): + """Creates the user/group profile as required.""" + usr_data = {'user_name': _usr_name, 'password': '12345', 'email': '{}@mail.com'.format(_usr_name), + 'group_name': get_constant('MAGPIE_ANONYMOUS_GROUP')} + if use_request(cookies_or_session): + if _usr_name: + path = '{url}{path}'.format(url=magpie_url, path=UsersAPI.path) + return requests.post(path, json=usr_data) + if _grp_name: + path = '{url}{path}'.format(url=magpie_url, path=GroupsAPI.path) + data = {'group_name': _grp_name} + return requests.post(path, json=data) + else: + if _usr_name: + from magpie.api.management.user.user_utils import create_user + usr_data['db_session'] = cookies_or_session + return create_user(**usr_data) + if _grp_name: + from magpie.api.management.group.group_utils import create_group + return create_group(_grp_name, cookies_or_session) + + def _validate_response(operation, is_create, item_type='Permission'): """Validate action/operation applied.""" # handle HTTPException raised if not islambda(operation): @@ -636,19 +659,19 @@ def _validate_response(operation): raise # validation according to status code returned - if create_perm: + if is_create: if _resp.status_code == 201: - warn_permission("Permission successfully created.", entry_index, level=logging.INFO) + warn_permission("{} successfully created.".format(item_type), entry_index, level=logging.INFO) elif _resp.status_code == 409: - warn_permission("Permission already exists.", entry_index, level=logging.INFO) + warn_permission("{} already exists.".format(item_type), entry_index, level=logging.INFO) else: warn_permission("Unknown response [{}]".format(_resp.status_code), entry_index, permission=permission_config_entry, level=logging.ERROR) else: if _resp.status_code == 200: - warn_permission("Permission successfully removed.", entry_index, level=logging.INFO) + warn_permission("{} successfully removed.".format(item_type), entry_index, level=logging.INFO) elif _resp.status_code == 404: - warn_permission("Permission already removed.", entry_index, level=logging.INFO) + warn_permission("{} already removed.".format(item_type), entry_index, level=logging.INFO) else: warn_permission("Unknown response [{}]".format(_resp.status_code), entry_index, permission=permission_config_entry, level=logging.ERROR) @@ -658,12 +681,15 @@ def _validate_response(operation): usr_name = permission_config_entry.get('user') grp_name = permission_config_entry.get('group') + _validate_response(lambda: _apply_profile(usr_name, None), is_create=True) + _validate_response(lambda: _apply_profile(None, grp_name), is_create=True) + if use_request(cookies_or_session): - _validate_response(lambda: _apply_request(usr_name, None)) - _validate_response(lambda: _apply_request(None, grp_name)) + _validate_response(lambda: _apply_request(usr_name, None), is_create=create_perm) + _validate_response(lambda: _apply_request(None, grp_name), is_create=create_perm) else: - _validate_response(lambda: _apply_session(usr_name, None)) - _validate_response(lambda: _apply_session(None, grp_name)) + _validate_response(lambda: _apply_session(usr_name, None), is_create=create_perm) + _validate_response(lambda: _apply_session(None, grp_name), is_create=create_perm) def magpie_register_permissions_from_config(permissions_config, magpie_url=None, db_session=None): diff --git a/permissions.cfg b/permissions.cfg index 4e553fda3..42df0bc1c 100644 --- a/permissions.cfg +++ b/permissions.cfg @@ -7,7 +7,8 @@ # action: one of [create, remove] (default: create) # # Default behaviour: -# - create missing resources if supported by the service, then apply permissions. +# - create missing resources if supported by the service (and tree automatically resolvable), then apply permissions. +# - create missing user/group if required (default user created: (group: anonymous, password: 12345). # - applicable service, user or group is missing, corresponding permissions are ignored and not updated. # - unknown actions are ignored and corresponding permission are not updated, unspecified action resolves to 'create'. # - already satisfied permission configurations are left as is. From 81fbdcab6c1a5fe0e5dc311266399909dd9d23a2 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Tue, 29 Jan 2019 18:06:10 -0500 Subject: [PATCH 08/76] more make targets + travis setup --- .gitignore | 1 + .travis.yml | 35 ++++++++ Makefile | 209 ++++++++++++++++++++++++++++++++----------- README.rst | 6 +- requirements-dev.txt | 6 ++ requirements.txt | 6 -- setup.cfg | 2 +- tox.ini | 2 +- 8 files changed, 207 insertions(+), 60 deletions(-) create mode 100644 .travis.yml create mode 100644 requirements-dev.txt diff --git a/.gitignore b/.gitignore index 1bd755c37..3cf16b840 100644 --- a/.gitignore +++ b/.gitignore @@ -29,4 +29,5 @@ magpie.egg-info/ src/ share/ /gunicorn.app.wsgiapp +downloads/ /sys diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 000000000..c799747b8 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,35 @@ +language: python +python: + - "2.7" + - "3.5" + - "3.6" +sudo: false +cache: pip +env: + matrix: + - env TARGET=test-local + - env TARGET=test-remote + - env TARGET=coverage +before_install: + - python --version + - uname -a + - lsb_release -a +install: + - make install + - make version +script: + - make $TARGET +notifications: + email: + on_success: never + on_failure: always +jobs: + include: + - stage: deploy + script: echo "Deploying..." + deploy: + provider: script + script: make conda-env docker-push + skip_existing: true + on: + tags: true diff --git a/Makefile b/Makefile index 2b29dd9f5..2eea8d263 100644 --- a/Makefile +++ b/Makefile @@ -14,6 +14,28 @@ BROWSER := python -c "$$BROWSER_PYSCRIPT" CUR_DIR := $(abspath $(lastword $(MAKEFILE_LIST))/..) APP_ROOT := $(CUR_DIR) APP_NAME := $(shell basename $(APP_ROOT)) +DOCKER_REPO := pavics/magpie + +# conda +CONDA_ENV ?= $(APP_NAME) +CONDA_HOME ?= $(HOME)/conda +CONDA_ENVS_DIR ?= $(CONDA_HOME)/envs +CONDA_ENV_PATH := $(CONDA_ENVS_DIR)/$(CONDA_ENV) +DOWNLOAD_CACHE := $(APP_ROOT)/downloads + +# choose conda installer depending on your OS +CONDA_URL = https://repo.continuum.io/miniconda +OS_NAME := $(shell uname -s || echo "unknown") +ifeq "$(OS_NAME)" "Linux" +FN := Miniconda3-latest-Linux-x86_64.sh +else ifeq "$(OS_NAME)" "Darwin" +FN := Miniconda3-latest-MacOSX-x86_64.sh +else +FN := unknown +endif + + +.DEFAULT_GOAL := help .PHONY: all all: help @@ -22,25 +44,33 @@ all: help help: @echo "Please use \`make ' where is one of:" @echo " Cleaning:" - @echo " clean: remove all build, test, coverage and Python artifacts" - @echo " clean-build: remove build artifacts" - @echo " clean-pyc: remove Python file artifacts" - @echo " clean-test: remove test and coverage artifacts" + @echo " clean: remove all build, test, coverage and Python artifacts" + @echo " clean-build: remove build artifacts" + @echo " clean-pyc: remove Python file artifacts" + @echo " clean-test: remove test and coverage artifacts" + @echo " Build and deploy:" + @echo " bump bump version using version specified as user input" + @echo " bump-dry bump version using version specified as user input (dry-run)" + @echo " bump-tag bump version using version specified as user input, tags it and commits change in git" + @echo " dist: package" + @echo " release: package and upload a release" + @echo " docker-info: tag version of docker image for build/push" + @echo " docker-build: build docker image" + @echo " docker-push: push built docker image" + @echo " version: current version" @echo " Install and run" - @echo " dist: package" - @echo " docs: generate Sphinx HTML documentation, including API docs" - @echo " install: install the package to the active Python's site-packages" - @echo " sysinstall: install system dependencies and required installers/runners" - @echo " migrate: run postgres database migration with alembic" - @echo " release: package and upload a release" - @echo " start: start magpie instance with gunicorn" + @echo " docs: generate Sphinx HTML documentation, including API docs" + @echo " install: install the package to the active Python's site-packages" + @echo " sysinstall: install system dependencies and required installers/runners" + @echo " migrate: run postgres database migration with alembic" + @echo " start: start magpie instance with gunicorn" @echo " Test and coverage" - @echo " coverage: check code coverage quickly with the default Python" - @echo " lint: check style with flake8" - @echo " test: run tests quickly with the default Python" - @echo " test-local: run only local tests with the default Python" - @echo " test-remote: run only remote tests with the default Python" - @echo " test-tox: run tests on every Python version with tox" + @echo " coverage: check code coverage quickly with the default Python" + @echo " lint: check style with flake8" + @echo " test: run tests quickly with the default Python" + @echo " test-local: run only local tests with the default Python" + @echo " test-remote: run only remote tests with the default Python" + @echo " test-tox: run tests on every Python version with tox" .PHONY: clean clean-build clean-pyc clean-test clean: clean-build clean-pyc clean-test @@ -64,84 +94,116 @@ clean-test: @echo "Cleaning tests artifacts..." rm -fr .tox/ rm -f .coverage - rm -fr coverage/ + rm -fr "$(CUR_DIR)/coverage/" .PHONY: lint -lint: +lint: install-dev @echo "Checking code style with flake8..." - flake8 magpie tests --ignore=E501 + @bash -c 'source "$(CONDA_HOME)/bin/activate" "$(CONDA_ENV)"; flake8 magpie tests --ignore=E501 || true' .PHONY: test -test: install +test: install-dev install @echo "Running tests..." - python setup.py test + @bash -c 'source "$(CONDA_HOME)/bin/activate" "$(CONDA_ENV)"; python setup.py test' .PHONY: test-local -test-local: install +test-local: install-dev install @echo "Running local tests..." - MAGPIE_TEST_REMOTE=false python setup.py test + @bash -c 'source "$(CONDA_HOME)/bin/activate" "$(CONDA_ENV)"; MAGPIE_TEST_REMOTE=false python setup.py test' .PHONY: test-remote -test-remote: install +test-remote: install-dev install @echo "Running remote tests..." - MAGPIE_TEST_LOCAL=false python setup.py test + @bash -c 'source "$(CONDA_HOME)/bin/activate" "$(CONDA_ENV)"; MAGPIE_TEST_LOCAL=false python setup.py test' .PHONY: test-tox -test-tox: +test-tox: install-dev install @echo "Running tests with tox..." - tox + @bash -c 'source "$(CONDA_HOME)/bin/activate" "$(CONDA_ENV)"; tox' .PHONY: coverage -coverage: +coverage: install-dev install @echo "Running coverage analysis..." - coverage run --source magpie setup.py test - coverage report -m - coverage html -d coverage - $(BROWSER) coverage/index.html + @bash -c 'source "$(CONDA_HOME)/bin/activate" "$(CONDA_ENV)"; coverage run --source magpie setup.py test || true' + @bash -c 'source "$(CONDA_HOME)/bin/activate" "$(CONDA_ENV)"; coverage report -m' + @bash -c 'source "$(CONDA_HOME)/bin/activate" "$(CONDA_ENV)"; coverage html -d coverage' + "$(BROWSER)" "$(CUR_DIR)/coverage/index.html" .PHONY: migrate -migrate: install +migrate: install conda-env @echo "Running database migration..." - alembic -c $(CUR_DIR)/magpie/alembic/alembic.ini upgrade head + @bash -c 'source "$(CONDA_HOME)/bin/activate" "$(CONDA_ENV)"; \ + alembic -c "$(CUR_DIR)/magpie/alembic/alembic.ini" upgrade head' .PHONY: docs -docs: +docs: install-dev @echo "Building docs..." rm -f $(CUR_DIR)/docs/magpie.rst rm -f $(CUR_DIR)/docs/modules.rst - sphinx-apidoc -o $(CUR_DIR)/docs/ $(CUR_DIR)/magpie - $(MAKE) -C $(CUR_DIR)/docs clean - $(MAKE) -C $(CUR_DIR)/docs html - $(BROWSER) $(CUR_DIR)/docs/_build/html/index.html + @bash -c 'source "$(CONDA_HOME)/bin/activate" "$(CONDA_ENV)"; \ + sphinx-apidoc -o "$(CUR_DIR)/docs/" "$(CUR_DIR)/magpie"; \ + "$(MAKE)" -C "$(CUR_DIR)/docs" clean; \ + "$(MAKE)" -C "$(CUR_DIR)/docs" html;' + "$(BROWSER)" "$(CUR_DIR)/docs/_build/html/index.html"' .PHONY: serve-docs -serve-docs: docs +serve-docs: docs install-dev @echo "Serving docs..." - watchmedo shell-command -p '*.rst' -c '$(MAKE) -C docs html' -R -D . + @bash -c 'source "$(CONDA_HOME)/bin/activate" "$(CONDA_ENV)"; \ + watchmedo shell-command -p '*.rst' -c '$(MAKE) -C docs html' -R -D .' .PHONY: release -release: clean +release: clean install @echo "Creating release..." python setup.py sdist upload python setup.py bdist_wheel upload +.PHONY: bump +bump: + $(shell bash -c 'read -p "Version: " VERSION_PART; \ + source "$(CONDA_HOME)/bin/activate" "$(CONDA_ENV)"; \ + test -f "$(CONDA_ENV_PATH)/bin/bumpversion" || pip install bumpversion; \ + "$(CONDA_ENV_PATH)/bin/bumpversion" --config-file "$(CUR_DIR)/.bumpversion.cfg" \ + --verbose --allow-dirty --no-tag --new-version $$VERSION_PART patch;') + +.PHONY: bump-dry +bump-dry: + $(shell bash -c 'read -p "Version: " VERSION_PART; \ + source "$(CONDA_HOME)/bin/activate" "$(CONDA_ENV)"; \ + test -f "$(CONDA_ENV_PATH)/bin/bumpversion" || pip install bumpversion; \ + "$(CONDA_ENV_PATH)/bin/bumpversion" --config-file "$(CUR_DIR)/.bumpversion.cfg" \ + --verbose --allow-dirty --dry-run --tag --tag-name "{new_version}" --new-version $$VERSION_PART patch;') + +.PHONY: bump-tag +bump-tag: + $(shell bash -c 'read -p "Version: " VERSION_PART; \ + source "$(CONDA_HOME)/bin/activate" "$(CONDA_ENV)"; \ + test -f $(CONDA_ENV_PATH)/bin/bumpversion || pip install bumpversion; \ + "$(CONDA_ENV_PATH)/bin/bumpversion" --config-file "$(CUR_DIR)/.bumpversion.cfg" \ + --verbose --allow-dirty --tag --tag-name "{new_version}" --new-version $$VERSION_PART patch;') + .PHONY: dist -dist: clean +dist: clean conda-env @echo "Creating distribution..." - python setup.py sdist - python setup.py bdist_wheel + @bash -c 'source "$(CONDA_HOME)/bin/activate" "$(CONDA_ENV)"; python setup.py sdist' + @bash -c 'source "$(CONDA_HOME)/bin/activate" "$(CONDA_ENV)"; python setup.py bdist_wheel' ls -l dist .PHONY: sysinstall -sysinstall: clean +sysinstall: clean conda-env @echo "Installing system dependencies..." - pip install --upgrade pip setuptools - pip install gunicorn + @bash -c 'source "$(CONDA_HOME)/bin/activate" "$(CONDA_ENV)"; pip install --upgrade pip setuptools' + @bash -c 'source "$(CONDA_HOME)/bin/activate" "$(CONDA_ENV)"; pip install gunicorn' .PHONY: install install: sysinstall @echo "Installing Magpie..." - pip install $(CUR_DIR) + @bash -c 'source "$(CONDA_HOME)/bin/activate" "$(CONDA_ENV)"; pip install "$(CUR_DIR)"' + +.PHONY: install-dev +install-dev: conda-env + @bash -c 'source "$(CONDA_HOME)/bin/activate" "$(CONDA_ENV)"; pip install -r "$(CUR_DIR)/requirements-dev.txt"' + @echo "Successfully installed dev requirements." .PHONY: cron cron: @@ -152,3 +214,50 @@ cron: start: cron install @echo "Starting Magpie..." exec gunicorn -b 0.0.0.0:2001 --paste "$(CUR_DIR)/magpie/magpie.ini" --workers 10 --preload + +.PHONY: version +version: + @echo "Mapie version:" + @python -c 'from magpie.__meta__ import __version__; print(__version__)' + +## Docker targets + +.PHONY: docker-info +docker-info: + @echo "Will be built, tagged and pushed as:" + @echo "$(DOCKER_REPO):`python -c 'from magpie.__meta__ import __version__; print(__version__)'`" + +.PHONY: docker-build +docker-build: + @bash -c "docker build $(CUR_DIR) \ + -t $(DOCKER_REPO):`python -c 'from magpie.__meta__ import __version__; print(__version__)'`" + +.PHONY: docker-push +docker-push: docker-build + @bash -c "docker push $(DOCKER_REPO):`python -c 'from magpie.__meta__ import __version__; print(__version__)'`" + +## Conda targets + +.PHONY: conda-base +conda-base: + @test -d "$(CONDA_HOME)" || test -d "$(DOWNLOAD_CACHE)" || mkdir "$(DOWNLOAD_CACHE)" + @test -d "$(CONDA_HOME)" || test -f "$(DOWNLOAD_CACHE)/$(FN)" || \ + curl "$(CONDA_URL)/$(FN)" --insecure --output "$(DOWNLOAD_CACHE)/$(FN)" + @test -d "$(CONDA_HOME)" || (bash "$(DOWNLOAD_CACHE)/$(FN)" -b -p "$(CONDA_HOME)" && \ + echo "Make sure to add '$(CONDA_HOME)/bin' to your PATH variable in '~/.bashrc'.") + +.PHONY: conda-cfg +conda_config: conda-base + @echo "Updating conda configuration..." + @"$(CONDA_HOME)/bin/conda" config --set ssl_verify true + @"$(CONDA_HOME)/bin/conda" config --set use_pip true + @"$(CONDA_HOME)/bin/conda" config --set channel_priority true + @"$(CONDA_HOME)/bin/conda" config --set auto_update_conda false + @"$(CONDA_HOME)/bin/conda" config --add channels defaults + +# the conda-env target's dependency on conda-cfg above was removed, will add back later if needed + +.PHONY: conda-env +conda-env: conda-base + @test -d "$(CONDA_ENV_PATH)" || (echo "Creating conda environment at '$(CONDA_ENV_PATH)'..." && \ + "$(CONDA_HOME)/bin/conda" env create --file "$(CUR_DIR)/conda-env.yml" -n "$(APP_NAME)") diff --git a/README.rst b/README.rst index ca7e27502..dc2982e4a 100644 --- a/README.rst +++ b/README.rst @@ -4,12 +4,14 @@ Magpie: A RestFul AuthN/AuthZ service Magpie (the smart-bird) *a very smart bird who knows everything about you.* -Magpie is service for AuthN/AuthZ accessible via a `RestAPI`_ implemented with the Pyramid web framework. It allows you to manage User/Group/Resource/permission with a postgres database. Behind the scene, it uses `Ziggurat-Foundations`_ and `Authomatic`_. +Magpie is service for AuthN/AuthZ accessible via a `RestAPI`_ implemented with the Pyramid web framework. +It allows you to manage User/Group/Resource/permission with a postgres database. +Behind the scene, it uses `Ziggurat-Foundations`_ and `Authomatic`_. REST API Documentation ====================== -The documentation is auto-generated and served under `{HOSTNAME}/magpie/api/` using Swagger-UI with tag `latest`. +The documentation is auto-generated and served under `{HOSTNAME}/api/` using Swagger-UI with tag `latest`. For convenience, older API versions are also provided. diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 000000000..88e52712f --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,6 @@ +tox>=3.0 +bumpversion==0.5.3 +flake8==3.5.0 +coverage==4.0 +Sphinx==1.3.1 +webtest diff --git a/requirements.txt b/requirements.txt index a5efe4a0c..ed785755d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,11 +1,6 @@ -tox>=3.0 -bumpversion==0.5.3 wheel==0.23.0 watchdog==0.8.3 pluggy -flake8==3.5.0 -coverage==4.0 -Sphinx==1.3.1 #cryptography==1.9 PyYAML>=3.11 pyramid==1.8.3 @@ -31,7 +26,6 @@ colander threddsclient==0.3.4 humanize requests_file -webtest typing #authomatic==0.1.0.post1 # until fix merged and deployed diff --git a/setup.cfg b/setup.cfg index dd2e5f6d5..7a9839380 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.6.3 +current_version = 0.9.0 commit = True tag = True diff --git a/tox.ini b/tox.ini index 02cfbf131..c23f581b9 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py26, py27, py33, py34 +envlist = py27, py35, py36 [testenv] setenv = From 14e7d282acca6ebb7b1d93c8770e4d5d3cede37c Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Tue, 29 Jan 2019 19:12:13 -0500 Subject: [PATCH 09/76] updates setup/config --- .bumpversion.cfg | 12 ++++++++++++ CONTRIBUTING.rst | 4 ++-- LICENSE | 2 +- Makefile | 7 ++++++- README.rst | 30 ++++++++++++++++++++++++++++++ docs/Makefile | 1 + docs/conf.py | 10 +++++----- docs/installation.rst | 6 +++--- magpie/__meta__.py | 3 +++ requirements-dev.txt | 2 ++ requirements.txt | 2 +- setup.py | 29 ++++++++++++++++------------- travis_pypi_setup.py | 2 +- 13 files changed, 83 insertions(+), 27 deletions(-) create mode 100644 .bumpversion.cfg diff --git a/.bumpversion.cfg b/.bumpversion.cfg new file mode 100644 index 000000000..52dce8768 --- /dev/null +++ b/.bumpversion.cfg @@ -0,0 +1,12 @@ +[bumpversion] +current_version = 0.9.0 +commit = True +tag = True + +[bumpversion:file:README.rst] +search = {current_version} +replace = {new_version} + +[bumpversion:file:magpie/__meta__.py] +search = __version__ = '{current_version}' +replace = __version__ = '{new_version}' diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index daf862e03..1629a64eb 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -13,7 +13,7 @@ Types of Contributions Report Bugs ~~~~~~~~~~~ -Report bugs at francois-xavier.derue@crim.ca. +Report bugs at francis.charette-migneault@crim.ca. If you are reporting a bug, please include: @@ -54,7 +54,7 @@ Ready to contribute? Here's how to set up `magpie` for local development. 2. Install your local copy and use a virtualenv. Assuming you have virtualenv installed, this is how you set up your fork for local development:: - + $ cd magpie/ $ virtualenv -p python 3.5 env $ source env/bin/activate.csh diff --git a/LICENSE b/LICENSE index 00d92795c..d641dbf44 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2017, Francois-Xavier +Copyright (c) 2017, Francois-Xavier, Francis Charette Migneault All rights reserved. Permission to use, copy, modify, and/or distribute this software for any diff --git a/Makefile b/Makefile index 2eea8d263..1f02d612d 100644 --- a/Makefile +++ b/Makefile @@ -46,6 +46,7 @@ help: @echo " Cleaning:" @echo " clean: remove all build, test, coverage and Python artifacts" @echo " clean-build: remove build artifacts" + @echo " clean-docs: remove doc artifacts" @echo " clean-pyc: remove Python file artifacts" @echo " clean-test: remove test and coverage artifacts" @echo " Build and deploy:" @@ -73,7 +74,7 @@ help: @echo " test-tox: run tests on every Python version with tox" .PHONY: clean clean-build clean-pyc clean-test -clean: clean-build clean-pyc clean-test +clean: clean-build clean-pyc clean-test clean-docs clean-build: @echo "Cleaning build artifacts..." @@ -83,6 +84,10 @@ clean-build: find . -type f -name '*.egg-info' -exec rm -fr {} + find . -type f -name '*.egg' -exec rm -f {} + +clean-docs: + @echo "Cleaning doc artifacts..." + "$(MAKE)" -C "$(CUR_DIR)/docs" clean + clean-pyc: @echo "Cleaning Python artifacts..." find . -type f -name '*.pyc' -exec rm -f {} + diff --git a/README.rst b/README.rst index dc2982e4a..227870640 100644 --- a/README.rst +++ b/README.rst @@ -8,6 +8,36 @@ Magpie is service for AuthN/AuthZ accessible via a `RestAPI`_ implemented with t It allows you to manage User/Group/Resource/permission with a postgres database. Behind the scene, it uses `Ziggurat-Foundations`_ and `Authomatic`_. + +.. start-badges + +.. list-table:: + :stub-columns: 1 + + * - dependencies + - | |py_ver| |requires| + * - releases + - | |version| |commits-since| + +.. |py_ver| image:: https://img.shields.io/badge/python-2.7%2C%203.5%2B-blue.svg + :alt: Requires Python 2.7, 3.5+ + :target: https://www.python.org/getit + +.. |commits-since| image:: https://img.shields.io/github/commits-since/Ouranosinc/Magpie/0.9.0.svg + :alt: Commits since latest release + :target: https://github.com/Ouranosinc/Magpie/compare/v0.9.0...master + +.. |version| image:: https://img.shields.io/github/tag/ouranosinc/magpie.svg?style=flat + :alt: Latest Tag + :target: https://github.com/Ouranosinc/Magpie/tree/0.9.0 + +.. |requires| image:: https://requires.io/github/Ouranosinc/Magpie/requirements.svg?branch=master + :alt: Requirements Status + :target: https://requires.io/github/Ouranosinc/Magpie/requirements/?branch=master + +.. end-badges + + REST API Documentation ====================== diff --git a/docs/Makefile b/docs/Makefile index f7c5c10df..e9cf88807 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -48,6 +48,7 @@ help: clean: rm -rf $(BUILDDIR)/* + rm -f magpie*.rst html: $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html diff --git a/docs/conf.py b/docs/conf.py index 9d3b76b83..57b0c906c 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -30,7 +30,7 @@ # version is used. sys.path.insert(0, PROJECT_ROOT) -from magpie.__meta__ import __version__ +from magpie import __meta__ # -- General configuration --------------------------------------------- @@ -63,16 +63,16 @@ # General information about the project. project = u'Magpie' -copyright = u'2017, Francois-Xavier' +copyright = u'2017, {}'.format(__meta__.__author__) # The version info for the project you're documenting, acts as replacement # for |version| and |release|, also used in various other places throughout # the built documents. # # The short X.Y version. -version = __version__ +version = __meta__.__version__ # The full version, including alpha/beta/rc tags. -release = __version__ +release = __meta__.__version__ # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -218,7 +218,7 @@ latex_documents = [ ('index', 'magpie.tex', u'Magpie Documentation', - u'Francois-Xavier', 'manual'), + __meta__.__author__, 'manual'), ] # The name of an image file (relative to this directory) to place at diff --git a/docs/installation.rst b/docs/installation.rst index 33a1efaed..ae29feadc 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -4,9 +4,9 @@ Installation At the command line:: - $ easy_install magpie + $ pip install magpie -Or, if you have virtualenvwrapper installed:: +Or, if you have conda installed:: - $ mkvirtualenv magpie + $ conda create -n magpie $ pip install magpie diff --git a/magpie/__meta__.py b/magpie/__meta__.py index d83a49765..0bba03bf6 100644 --- a/magpie/__meta__.py +++ b/magpie/__meta__.py @@ -3,8 +3,11 @@ """ __version__ = '0.9.0' +__title__ = 'Magpie' +__package__ = 'magpie' __author__ = "Francois-Xavier Derue, Francis Charette-Migneault" __maintainer__ = "Francis Charette-Migneault" __email__ = 'francis.charette-migneault@crim.ca' __url__ = 'https://github.com/Ouranosinc/Magpie' +__docker__ = 'https://hub.docker.com/r/pavics/magpie' __description__ = "Magpie is a service for AuthN and AuthZ based on Ziggurat-Foundations" diff --git a/requirements-dev.txt b/requirements-dev.txt index 88e52712f..52efde844 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -4,3 +4,5 @@ flake8==3.5.0 coverage==4.0 Sphinx==1.3.1 webtest +nose==1.3.7 +pytest diff --git a/requirements.txt b/requirements.txt index ed785755d..b10e1887f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ wheel==0.23.0 watchdog==0.8.3 pluggy #cryptography==1.9 -PyYAML>=3.11 +PyYAML>=3.13 pyramid==1.8.3 ziggurat-foundations==0.8.1 pyramid_tm==2.2.1 diff --git a/setup.py b/setup.py index c580c7121..e4f40659f 100644 --- a/setup.py +++ b/setup.py @@ -34,15 +34,21 @@ else: REQUIREMENTS.add(line.strip()) +TEST_REQUIREMENTS = set() +with open('requirements-dev.txt', 'r') as requirements_file: + for line in requirements_file: + if 'git+https' in line: + pkg = line.split('#')[-1] + LINKS.add(line.strip()) + REQUIREMENTS.add(pkg.replace('egg=', '').rstrip()) + elif line.startswith('http'): + LINKS.add(line.strip()) + else: + REQUIREMENTS.add(line.strip()) + LINKS = list(LINKS) REQUIREMENTS = list(REQUIREMENTS) - -# put package test requirements here -TEST_REQUIREMENTS = [ - 'nose==1.3.7', - 'webtest', - 'pytest', -] +TEST_REQUIREMENTS = list(TEST_REQUIREMENTS) raw_requirements = set() for req in REQUIREMENTS: @@ -57,7 +63,7 @@ setup( # -- meta information -------------------------------------------------- - name='magpie', + name=__meta__.__package__, version=__meta__.__version__, description=__meta__.__description__, long_description=README + '\n\n' + HISTORY, @@ -69,7 +75,7 @@ url=__meta__.__url__, platforms=['linux_x86_64'], license="ISCL", - keywords='magpie', + keywords=__meta__.__title__ + ", Authentication, AuthN", classifiers=[ 'Development Status :: 2 - Pre-Alpha', 'Intended Audience :: Developers', @@ -80,11 +86,8 @@ ], # -- Package structure ------------------------------------------------- - #packages=[ - # 'magpie', - #], packages=find_packages(), - package_dir={'magpie': 'magpie'}, + package_dir={__meta__.__package__: 'magpie'}, include_package_data=True, install_requires=REQUIREMENTS, dependency_links=LINKS, diff --git a/travis_pypi_setup.py b/travis_pypi_setup.py index 092de8c60..b9bb6d8b3 100644 --- a/travis_pypi_setup.py +++ b/travis_pypi_setup.py @@ -16,7 +16,7 @@ from six.moves.urllib.request import urlopen -GITHUB_REPO = 'fderue/magpie' +GITHUB_REPO = 'pavics/magpie' TRAVIS_CONFIG_FILE = os.path.join( os.path.dirname(os.path.abspath(__file__)), '.travis.yml') From 483db1ebe04885c73730e5b59b1dec4afdc64b0d Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Tue, 29 Jan 2019 21:23:22 -0500 Subject: [PATCH 10/76] fix some tests --- .../travis_pypi_setup.py | 0 magpie/api/login/login.py | 4 +- magpie/api/management/group/group_views.py | 186 ++++++------ magpie/api/management/user/user_utils.py | 3 +- magpie/api/management/user/user_views.py | 273 +++++++++--------- setup.py | 2 +- tests/interfaces.py | 37 ++- tests/test_magpie_adapter.py | 9 + tests/test_register.py | 9 + tests/utils.py | 17 ++ tox.ini | 3 +- 11 files changed, 297 insertions(+), 246 deletions(-) rename travis_pypi_setup.py => ci/travis_pypi_setup.py (100%) create mode 100644 tests/test_register.py diff --git a/travis_pypi_setup.py b/ci/travis_pypi_setup.py similarity index 100% rename from travis_pypi_setup.py rename to ci/travis_pypi_setup.py diff --git a/magpie/api/login/login.py b/magpie/api/login/login.py index f0fc113ce..b4b371628 100644 --- a/magpie/api/login/login.py +++ b/magpie/api/login/login.py @@ -28,7 +28,7 @@ def process_sign_in_external(request, username, provider): query_field = dict(id=username) elif provider_name == 'github': query_field = None - #query_field = dict(login_field=username) + # query_field = dict(login_field=username) elif provider_name == 'wso2': query_field = {} else: @@ -113,6 +113,7 @@ def new_user_external(external_user_name, external_id, email, provider_name, db_ create_user(internal_user_name, password=None, email=email, group_name=group_name, db_session=db_session) user = UserService.by_user_name(internal_user_name, db_session=db_session) + # noinspection PyArgumentList ex_identity = models.ExternalIdentity(external_user_name=external_user_name, external_id=external_id, local_user_id=user.id, provider_name=provider_name) evaluate_call(lambda: db_session.add(ex_identity), fallback=lambda: db_session.rollback(), @@ -251,6 +252,7 @@ def _get_session(req): return valid_http(httpSuccess=HTTPOk, detail=Session_GET_OkResponseSchema.description, content=session_json) +# noinspection PyUnusedLocal @ProvidersAPI.get(tags=[LoginTag], response_schemas=Providers_GET_responses) @view_config(route_name=ProvidersAPI.name, request_method='GET', permission=NO_PERMISSION_REQUIRED) def get_providers(request): diff --git a/magpie/api/management/group/group_views.py b/magpie/api/management/group/group_views.py index b37f032b9..300eaebff 100644 --- a/magpie/api/management/group/group_views.py +++ b/magpie/api/management/group/group_views.py @@ -1,149 +1,157 @@ -from magpie.api.management.group.group_utils import * +from magpie.api import api_requests as ar, api_except as ax +from magpie.api.management.group import group_utils as gu, group_formats as gf +from magpie.api.management.service.service_formats import format_service, format_service_resources from magpie.api.api_rest_schemas import * from magpie.constants import get_constant from magpie.definitions.ziggurat_definitions import * from magpie.definitions.pyramid_definitions import view_config +from magpie import models @GroupsAPI.get(tags=[GroupsTag], response_schemas=Groups_GET_responses) @view_config(route_name=GroupsAPI.name, request_method='GET') -def get_groups(request): +def get_groups_view(request): """Get list of group names.""" - group_names = get_all_groups(request.db) - return valid_http(httpSuccess=HTTPOk, detail=Groups_GET_OkResponseSchema.description, - content={u'group_names': group_names}) + group_names = gu.get_all_groups(request.db) + return ax.valid_http(httpSuccess=HTTPOk, detail=Groups_GET_OkResponseSchema.description, + content={u'group_names': group_names}) @GroupsAPI.post(schema=Groups_POST_RequestSchema(), tags=[GroupsTag], response_schemas=Groups_POST_responses) @view_config(route_name=GroupsAPI.name, request_method='POST') -def create_group(request): +def create_group_view(request): """Create a group.""" - group_name = get_value_multiformat_post_checked(request, 'group_name') - return create_group(group_name, request.db) + group_name = ar.get_value_multiformat_post_checked(request, 'group_name') + return gu.create_group(group_name, request.db) @GroupAPI.get(tags=[GroupsTag], response_schemas=Group_GET_responses) @view_config(route_name=GroupAPI.name, request_method='GET') -def get_group(request): +def get_group_view(request): """Get group information.""" - group = get_group_matchdict_checked(request, group_name_key='group_name') - return valid_http(httpSuccess=HTTPOk, detail=Group_GET_OkResponseSchema.description, - content={u'group': format_group(group)}) + group = ar.get_group_matchdict_checked(request, group_name_key='group_name') + return ax.valid_http(httpSuccess=HTTPOk, detail=Group_GET_OkResponseSchema.description, + content={u'group': gf.format_group(group)}) @GroupAPI.put(schema=Group_PUT_RequestSchema(), tags=[GroupsTag], response_schemas=Group_PUT_responses) @view_config(route_name=GroupAPI.name, request_method='PUT') -def edit_group(request): +def edit_group_view(request): """Update a group by name.""" - group = get_group_matchdict_checked(request, group_name_key='group_name') - new_group_name = get_multiformat_post(request, 'group_name') - verify_param(new_group_name, notNone=True, notEmpty=True, httpError=HTTPNotAcceptable, - msgOnFail=Group_PUT_Name_NotAcceptableResponseSchema.description) - verify_param(len(new_group_name), isIn=True, httpError=HTTPNotAcceptable, - paramCompare=range(1, 1 + get_constant('MAGPIE_USER_NAME_MAX_LENGTH')), - msgOnFail=Group_PUT_Size_NotAcceptableResponseSchema.description) - verify_param(new_group_name, notEqual=True, httpError=HTTPNotAcceptable, - paramCompare=group.group_name, msgOnFail=Group_PUT_Same_NotAcceptableResponseSchema.description) - verify_param(GroupService.by_group_name(new_group_name, db_session=request.db), isNone=True, httpError=HTTPConflict, - msgOnFail=Group_PUT_ConflictResponseSchema.description) + group = ar.get_group_matchdict_checked(request, group_name_key='group_name') + new_group_name = ar.get_multiformat_post(request, 'group_name') + ax.verify_param(new_group_name, notNone=True, notEmpty=True, httpError=HTTPNotAcceptable, + msgOnFail=Group_PUT_Name_NotAcceptableResponseSchema.description) + ax.verify_param(len(new_group_name), isIn=True, httpError=HTTPNotAcceptable, + paramCompare=range(1, 1 + get_constant('MAGPIE_USER_NAME_MAX_LENGTH')), + msgOnFail=Group_PUT_Size_NotAcceptableResponseSchema.description) + ax.verify_param(new_group_name, notEqual=True, httpError=HTTPNotAcceptable, + paramCompare=group.group_name, msgOnFail=Group_PUT_Same_NotAcceptableResponseSchema.description) + ax.verify_param(GroupService.by_group_name(new_group_name, db_session=request.db), + isNone=True, httpError=HTTPConflict, + msgOnFail=Group_PUT_ConflictResponseSchema.description) group.group_name = new_group_name - return valid_http(httpSuccess=HTTPOk, detail=Group_PUT_OkResponseSchema.description) + return ax.valid_http(httpSuccess=HTTPOk, detail=Group_PUT_OkResponseSchema.description) @GroupAPI.delete(schema=Group_DELETE_RequestSchema(), tags=[GroupsTag], response_schemas=Group_DELETE_responses) @view_config(route_name=GroupAPI.name, request_method='DELETE') -def delete_group(request): +def delete_group_view(request): """Delete a group by name.""" - group = get_group_matchdict_checked(request) - evaluate_call(lambda: request.db.delete(group), fallback=lambda: request.db.rollback(), httpError=HTTPForbidden, - msgOnFail=Group_DELETE_ForbiddenResponseSchema.description) - return valid_http(httpSuccess=HTTPOk, detail=Group_DELETE_OkResponseSchema.description) + group = ar.get_group_matchdict_checked(request) + ax.evaluate_call(lambda: request.db.delete(group), + fallback=lambda: request.db.rollback(), httpError=HTTPForbidden, + msgOnFail=Group_DELETE_ForbiddenResponseSchema.description) + return ax.valid_http(httpSuccess=HTTPOk, detail=Group_DELETE_OkResponseSchema.description) @GroupUsersAPI.get(tags=[GroupsTag], response_schemas=GroupUsers_GET_responses) @view_config(route_name=GroupUsersAPI.name, request_method='GET') -def get_group_users(request): +def get_group_users_view(request): """List all user from a group.""" - group = get_group_matchdict_checked(request) - user_names = evaluate_call(lambda: [user.user_name for user in group.users], - httpError=HTTPForbidden, msgOnFail=GroupUsers_GET_ForbiddenResponseSchema.description) - return valid_http(httpSuccess=HTTPOk, detail=GroupUsers_GET_OkResponseSchema.description, - content={u'user_names': sorted(user_names)}) + group = ar.get_group_matchdict_checked(request) + user_names = ax.evaluate_call(lambda: [user.user_name for user in group.users], + httpError=HTTPForbidden, + msgOnFail=GroupUsers_GET_ForbiddenResponseSchema.description) + return ax.valid_http(httpSuccess=HTTPOk, detail=GroupUsers_GET_OkResponseSchema.description, + content={u'user_names': sorted(user_names)}) @GroupServicesAPI.get(tags=[GroupsTag], response_schemas=GroupServices_GET_responses) @view_config(route_name=GroupServicesAPI.name, request_method='GET') def get_group_services_view(request): """List all services a group has permission on.""" - group = get_group_matchdict_checked(request) - res_perm_dict = get_group_resources_permissions_dict(group, - resource_types=[models.Service.resource_type_name], - db_session=request.db) - grp_svc_json = evaluate_call(lambda: get_group_services(res_perm_dict, request.db), - httpError=HTTPInternalServerError, - msgOnFail=GroupServices_InternalServerErrorResponseSchema.description, - content={u'group': format_group(group)}) - return valid_http(httpSuccess=HTTPOk, detail=GroupServices_GET_OkResponseSchema.description, - content={u'services': grp_svc_json}) + group = ar.get_group_matchdict_checked(request) + res_perm_dict = gu.get_group_resources_permissions_dict(group, + resource_types=[models.Service.resource_type_name], + db_session=request.db) + grp_svc_json = ax.evaluate_call(lambda: gu.get_group_services(res_perm_dict, request.db), + httpError=HTTPInternalServerError, + msgOnFail=GroupServices_InternalServerErrorResponseSchema.description, + content={u'group': gf.format_group(group)}) + return ax.valid_http(httpSuccess=HTTPOk, detail=GroupServices_GET_OkResponseSchema.description, + content={u'services': grp_svc_json}) @GroupServicePermissionsAPI.get(tags=[GroupsTag], response_schemas=GroupServicePermissions_GET_responses) @view_config(route_name=GroupServicePermissionsAPI.name, request_method='GET') def get_group_service_permissions_view(request): """List all permissions a group has on a specific service.""" - group = get_group_matchdict_checked(request) - service = get_service_matchdict_checked(request) - svc_perms_found = evaluate_call(lambda: get_group_service_permissions(group, service, request.db), - httpError=HTTPInternalServerError, - msgOnFail=GroupServicePermissions_GET_InternalServerErrorResponseSchema.description, - content={u'group': format_group(group), u'service': format_service(service)}) - return valid_http(httpSuccess=HTTPOk, detail=GroupServicePermissions_GET_OkResponseSchema.description, - content={u'permission_names': svc_perms_found}) + group = ar.get_group_matchdict_checked(request) + service = ar.get_service_matchdict_checked(request) + svc_perms_found = ax.evaluate_call( + lambda: gu.get_group_service_permissions(group, service, request.db), + httpError=HTTPInternalServerError, + msgOnFail=GroupServicePermissions_GET_InternalServerErrorResponseSchema.description, + content={u'group': gf.format_group(group), u'service': format_service(service)}) + return ax.valid_http(httpSuccess=HTTPOk, detail=GroupServicePermissions_GET_OkResponseSchema.description, + content={u'permission_names': svc_perms_found}) @GroupServicePermissionsAPI.post(schema=GroupServicePermissions_POST_RequestSchema(), tags=[GroupsTag], response_schemas=GroupServicePermissions_POST_responses) @view_config(route_name=GroupServicePermissionsAPI.name, request_method='POST') -def create_group_service_permission(request): +def create_group_service_permission_view(request): """Create a permission on a specific resource for a group.""" - group = get_group_matchdict_checked(request) - service = get_service_matchdict_checked(request) - perm_name = get_permission_multiformat_post_checked(request, service) - return create_group_resource_permission(perm_name, service, group, db_session=request.db) + group = ar.get_group_matchdict_checked(request) + service = ar.get_service_matchdict_checked(request) + perm_name = ar.get_permission_multiformat_post_checked(request, service) + return gu.create_group_resource_permission(perm_name, service, group, db_session=request.db) @GroupServicePermissionAPI.delete(schema=GroupServicePermission_DELETE_RequestSchema(), tags=[GroupsTag], response_schemas=GroupServicePermission_DELETE_responses) @view_config(route_name=GroupServicePermissionAPI.name, request_method='DELETE') -def delete_group_service_permission(request): +def delete_group_service_permission_view(request): """Delete a permission from a specific service for a group.""" - group = get_group_matchdict_checked(request) - service = get_service_matchdict_checked(request) - perm_name = get_permission_matchdict_checked(request, service) - return delete_group_resource_permission(perm_name, service, group, db_session=request.db) + group = ar.get_group_matchdict_checked(request) + service = ar.get_service_matchdict_checked(request) + perm_name = ar.get_permission_matchdict_checked(request, service) + return gu.delete_group_resource_permission(perm_name, service, group, db_session=request.db) @GroupResourcesAPI.get(tags=[GroupsTag], response_schemas=GroupResources_GET_responses) @view_config(route_name=GroupResourcesAPI.name, request_method='GET') def get_group_resources_view(request): """List all resources a group has permission on.""" - group = get_group_matchdict_checked(request) - grp_res_json = evaluate_call(lambda: get_group_resources(group, request.db), fallback=lambda: request.db.rollback(), - httpError=HTTPInternalServerError, content={u'group': repr(group)}, - msgOnFail=GroupResources_GET_InternalServerErrorResponseSchema.description) - return valid_http(httpSuccess=HTTPOk, detail=GroupResources_GET_OkResponseSchema.description, - content={u'resources': grp_res_json}) + group = ar.get_group_matchdict_checked(request) + grp_res_json = ax.evaluate_call(lambda: gu.get_group_resources(group, request.db), + fallback=lambda: request.db.rollback(), + httpError=HTTPInternalServerError, content={u'group': repr(group)}, + msgOnFail=GroupResources_GET_InternalServerErrorResponseSchema.description) + return ax.valid_http(httpSuccess=HTTPOk, detail=GroupResources_GET_OkResponseSchema.description, + content={u'resources': grp_res_json}) @GroupResourcePermissionsAPI.get(tags=[GroupsTag], response_schemas=GroupResourcePermissions_GET_responses) @view_config(route_name=GroupResourcePermissionsAPI.name, request_method='GET') def get_group_resource_permissions_view(request): """List all permissions a group has on a specific resource.""" - group = get_group_matchdict_checked(request) - resource = get_resource_matchdict_checked(request) - perm_names = get_group_resource_permissions(group, resource, db_session=request.db) - return valid_http(httpSuccess=HTTPOk, detail=GroupResourcePermissions_GET_OkResponseSchema.description, - content={u'permission_names': perm_names}) + group = ar.get_group_matchdict_checked(request) + resource = ar.get_resource_matchdict_checked(request) + perm_names = gu.get_group_resource_permissions(group, resource, db_session=request.db) + return ax.valid_http(httpSuccess=HTTPOk, detail=GroupResourcePermissions_GET_OkResponseSchema.description, + content={u'permission_names': perm_names}) @GroupResourcePermissionsAPI.post(schema=GroupResourcePermissions_POST_RequestSchema(), tags=[GroupsTag], @@ -151,10 +159,10 @@ def get_group_resource_permissions_view(request): @view_config(route_name=GroupResourcePermissionsAPI.name, request_method='POST') def create_group_resource_permission_view(request): """Create a permission on a specific resource for a group.""" - group = get_group_matchdict_checked(request) - resource = get_resource_matchdict_checked(request) - perm_name = get_permission_multiformat_post_checked(request, resource) - return create_group_resource_permission(perm_name, resource, group, db_session=request.db) + group = ar.get_group_matchdict_checked(request) + resource = ar.get_resource_matchdict_checked(request) + perm_name = ar.get_permission_multiformat_post_checked(request, resource) + return gu.create_group_resource_permission(perm_name, resource, group, db_session=request.db) @GroupResourcePermissionAPI.delete(schema=GroupResourcePermission_DELETE_RequestSchema(), tags=[GroupsTag], @@ -162,20 +170,20 @@ def create_group_resource_permission_view(request): @view_config(route_name=GroupResourcePermissionAPI.name, request_method='DELETE') def delete_group_resource_permission_view(request): """Delete a permission from a specific resource for a group.""" - group = get_group_matchdict_checked(request) - resource = get_resource_matchdict_checked(request) - perm_name = get_permission_matchdict_checked(request, resource) - return delete_group_resource_permission(perm_name, resource, group, db_session=request.db) + group = ar.get_group_matchdict_checked(request) + resource = ar.get_resource_matchdict_checked(request) + perm_name = ar.get_permission_matchdict_checked(request, resource) + return gu.delete_group_resource_permission(perm_name, resource, group, db_session=request.db) @GroupServiceResourcesAPI.get(tags=[GroupsTag], response_schemas=GroupServiceResources_GET_responses) @view_config(route_name=GroupServiceResourcesAPI.name, request_method='GET') def get_group_service_resources_view(request): """List all resources under a service a group has permission on.""" - group = get_group_matchdict_checked(request) - service = get_service_matchdict_checked(request) - svc_perms = get_group_service_permissions(group=group, service=service, db_session=request.db) - res_perms = get_group_service_resources_permissions_dict(group=group, service=service, db_session=request.db) + group = ar.get_group_matchdict_checked(request) + service = ar.get_service_matchdict_checked(request) + svc_perms = gu.get_group_service_permissions(group=group, service=service, db_session=request.db) + res_perms = gu.get_group_service_resources_permissions_dict(group=group, service=service, db_session=request.db) svc_res_json = format_service_resources( service=service, db_session=request.db, @@ -184,5 +192,5 @@ def get_group_service_resources_view(request): display_all=False, show_private_url=False, ) - return valid_http(httpSuccess=HTTPOk, detail=GroupServiceResources_GET_OkResponseSchema.description, - content={u'service': svc_res_json}) + return ax.valid_http(httpSuccess=HTTPOk, detail=GroupServiceResources_GET_OkResponseSchema.description, + content={u'service': svc_res_json}) diff --git a/magpie/api/management/user/user_utils.py b/magpie/api/management/user/user_utils.py index ced4f4bbe..a4636f786 100644 --- a/magpie/api/management/user/user_utils.py +++ b/magpie/api/management/user/user_utils.py @@ -12,9 +12,10 @@ def create_user(user_name, password, email, group_name, db_session): - # type: (AnyStr, AnyStr, AnyStr, AnyStr, Session) -> HTTPException + # type: (AnyStr, Union[AnyStr, None], AnyStr, AnyStr, Session) -> HTTPException """ Creates a user if it is permitted and not conflicting. + Password must be set to `None` if using external identity. :returns: corresponding HTTP response according to the encountered situation. """ diff --git a/magpie/api/management/user/user_views.py b/magpie/api/management/user/user_views.py index bc4ab800d..53457e421 100644 --- a/magpie/api/management/user/user_views.py +++ b/magpie/api/management/user/user_views.py @@ -1,38 +1,35 @@ -from magpie.definitions.pyramid_definitions import * -from magpie.definitions.ziggurat_definitions import * -from magpie.api.api_requests import * from magpie.api.api_rest_schemas import * -from magpie.api.management.user.user_formats import * -from magpie.api.management.user.user_utils import * -from magpie.api.management.group.group_utils import * -from magpie.api.management.service.service_utils import get_services_by_type -from magpie.api.management.service.service_formats import format_service, format_service_resources +from magpie.api import api_except as ax, api_requests as ar +from magpie.api.management.user import user_utils as uu, user_formats as uf +from magpie.api.management.service.service_formats import format_service_resources +from magpie.definitions.ziggurat_definitions import UserService, GroupService from magpie.common import str2bool +from magpie import models import logging LOGGER = logging.getLogger(__name__) @UsersAPI.get(tags=[UsersTag], response_schemas=Users_GET_responses) @view_config(route_name=UsersAPI.name, request_method='GET') -def get_users(request): +def get_users_view(request): """List all registered user names.""" - user_name_list = evaluate_call(lambda: [user.user_name for user in models.User.all(db_session=request.db)], - fallback=lambda: request.db.rollback(), - httpError=HTTPForbidden, msgOnFail=Users_GET_ForbiddenResponseSchema.description) - return valid_http(httpSuccess=HTTPOk, content={u'user_names': sorted(user_name_list)}, - detail=Users_GET_OkResponseSchema.description) + user_name_list = ax.evaluate_call(lambda: [user.user_name for user in models.User.all(db_session=request.db)], + fallback=lambda: request.db.rollback(), + httpError=HTTPForbidden, msgOnFail=Users_GET_ForbiddenResponseSchema.description) + return ax.valid_http(httpSuccess=HTTPOk, content={u'user_names': sorted(user_name_list)}, + detail=Users_GET_OkResponseSchema.description) @UsersAPI.post(schema=Users_POST_RequestSchema(), tags=[UsersTag], response_schemas=Users_POST_responses) @view_config(route_name=UsersAPI.name, request_method='POST') def create_user_view(request): """Create a new user.""" - user_name = get_multiformat_post(request, 'user_name') - email = get_multiformat_post(request, 'email') - password = get_multiformat_post(request, 'password') - group_name = get_multiformat_post(request, 'group_name') - check_user_info(user_name, email, password, group_name) - return create_user(user_name, password, email, group_name, db_session=request.db) + user_name = ar.get_multiformat_post(request, 'user_name') + email = ar.get_multiformat_post(request, 'email') + password = ar.get_multiformat_post(request, 'password') + group_name = ar.get_multiformat_post(request, 'group_name') + uu.check_user_info(user_name, email, password, group_name) + return uu.create_user(user_name, password, email, group_name, db_session=request.db) @UserAPI.put(schema=User_PUT_RequestSchema(), tags=[UsersTag], response_schemas=User_PUT_responses) @@ -40,25 +37,25 @@ def create_user_view(request): @view_config(route_name=UserAPI.name, request_method='PUT') def update_user_view(request): """Update user information by user name.""" - user = get_user_matchdict_checked(request, user_name_key='user_name') - new_user_name = get_multiformat_post(request, 'user_name', default=user.user_name) - new_email = get_multiformat_post(request, 'email', default=user.email) - new_password = get_multiformat_post(request, 'password', default=user.user_password) - check_user_info(new_user_name, new_email, new_password, group_name=new_user_name) + user = ar.get_user_matchdict_checked(request, user_name_key='user_name') + new_user_name = ar.get_multiformat_post(request, 'user_name', default=user.user_name) + new_email = ar.get_multiformat_post(request, 'email', default=user.email) + new_password = ar.get_multiformat_post(request, 'password', default=user.user_password) + uu.check_user_info(new_user_name, new_email, new_password, group_name=new_user_name) update_username = user.user_name != new_user_name update_password = user.user_password != new_password update_email = user.email != new_email - verify_param(any([update_username, update_password, update_email]), isTrue=True, httpError=HTTPBadRequest, - content={u'user_name': user.user_name}, - msgOnFail=User_PUT_BadRequestResponseSchema.description) + ax.verify_param(any([update_username, update_password, update_email]), isTrue=True, httpError=HTTPBadRequest, + content={u'user_name': user.user_name}, + msgOnFail=User_PUT_BadRequestResponseSchema.description) if user.user_name != new_user_name: - existing_user = evaluate_call(lambda: UserService.by_user_name(new_user_name, db_session=request.db), - fallback=lambda: request.db.rollback(), - httpError=HTTPForbidden, msgOnFail=User_PUT_ForbiddenResponseSchema.description) - verify_param(existing_user, isNone=True, httpError=HTTPConflict, - msgOnFail=User_PUT_ConflictResponseSchema.description) + existing_user = ax.evaluate_call(lambda: UserService.by_user_name(new_user_name, db_session=request.db), + fallback=lambda: request.db.rollback(), httpError=HTTPForbidden, + msgOnFail=User_PUT_ForbiddenResponseSchema.description) + ax.verify_param(existing_user, isNone=True, httpError=HTTPConflict, + msgOnFail=User_PUT_ConflictResponseSchema.description) user.user_name = new_user_name if user.email != new_email: user.email = new_email @@ -66,7 +63,7 @@ def update_user_view(request): UserService.set_password(user, new_password) UserService.regenerate_security_code(user) - return valid_http(httpSuccess=HTTPOk, detail=Users_PUT_OkResponseSchema.description) + return ax.valid_http(httpSuccess=HTTPOk, detail=Users_PUT_OkResponseSchema.description) @UserAPI.get(tags=[UsersTag], api_security=SecurityEveryoneAPI, response_schemas=User_GET_responses) @@ -74,59 +71,59 @@ def update_user_view(request): @view_config(route_name=UserAPI.name, request_method='GET', permission=NO_PERMISSION_REQUIRED) def get_user_view(request): """Get user information by name.""" - user = get_user_matchdict_checked_or_logged(request) - return valid_http(httpSuccess=HTTPOk, detail=User_GET_OkResponseSchema.description, - content={u'user': format_user(user)}) + user = ar.get_user_matchdict_checked_or_logged(request) + return ax.valid_http(httpSuccess=HTTPOk, content={u'user': uf.format_user(user)}, + detail=User_GET_OkResponseSchema.description) @UserAPI.delete(schema=User_DELETE_RequestSchema(), tags=[UsersTag], response_schemas=User_DELETE_responses) @LoggedUserAPI.delete(schema=User_DELETE_RequestSchema(), tags=[LoggedUserTag], response_schemas=LoggedUser_DELETE_responses) @view_config(route_name=UserAPI.name, request_method='DELETE') -def delete_user(request): +def delete_user_view(request): """Delete a user by name.""" - user = get_user_matchdict_checked_or_logged(request) - evaluate_call(lambda: request.db.delete(user), fallback=lambda: request.db.rollback(), - httpError=HTTPForbidden, msgOnFail=User_DELETE_ForbiddenResponseSchema.description) - return valid_http(httpSuccess=HTTPOk, detail=User_DELETE_OkResponseSchema.description) + user = ar.get_user_matchdict_checked_or_logged(request) + ax.evaluate_call(lambda: request.db.delete(user), fallback=lambda: request.db.rollback(), + httpError=HTTPForbidden, msgOnFail=User_DELETE_ForbiddenResponseSchema.description) + return ax.valid_http(httpSuccess=HTTPOk, detail=User_DELETE_OkResponseSchema.description) @UserGroupsAPI.get(tags=[UsersTag], api_security=SecurityEveryoneAPI, response_schemas=UserGroups_GET_responses) @LoggedUserGroupsAPI.get(tags=[LoggedUserTag], api_security=SecurityEveryoneAPI, response_schemas=LoggedUserGroups_GET_responses) @view_config(route_name=UserGroupsAPI.name, request_method='GET', permission=NO_PERMISSION_REQUIRED) -def get_user_groups(request): +def get_user_groups_view(request): """List all groups a user belongs to.""" - user = get_user_matchdict_checked_or_logged(request) - group_names = get_user_groups_checked(request, user) - return valid_http(httpSuccess=HTTPOk, detail=UserGroups_GET_OkResponseSchema.description, - content={u'group_names': group_names}) + user = ar.get_user_matchdict_checked_or_logged(request) + group_names = uu.get_user_groups_checked(request, user) + return ax.valid_http(httpSuccess=HTTPOk, content={u'group_names': group_names}, + detail=UserGroups_GET_OkResponseSchema.description) @UserGroupsAPI.post(schema=UserGroups_POST_RequestSchema(), tags=[UsersTag], response_schemas=UserGroups_POST_responses) @LoggedUserGroupsAPI.post(schema=UserGroups_POST_RequestSchema(), tags=[LoggedUserTag], response_schemas=LoggedUserGroups_POST_responses) @view_config(route_name=UserGroupsAPI.name, request_method='POST') -def assign_user_group(request): +def assign_user_group_view(request): """Assign a user to a group.""" - user = get_user_matchdict_checked_or_logged(request) - - group_name = get_value_multiformat_post_checked(request, 'group_name') - group = evaluate_call(lambda: zig.GroupService.by_group_name(group_name, db_session=request.db), - fallback=lambda: request.db.rollback(), - httpError=HTTPForbidden, msgOnFail=UserGroups_POST_ForbiddenResponseSchema.description) - verify_param(group, notNone=True, httpError=HTTPNotFound, - msgOnFail=UserGroups_POST_GroupNotFoundResponseSchema.description) - verify_param(user.id, paramCompare=[usr.id for usr in group.users], notIn=True, httpError=HTTPConflict, - content={u'user_name': user.user_name, u'group_name': group.group_name}, - msgOnFail=UserGroups_POST_ConflictResponseSchema.description) - - evaluate_call(lambda: request.db.add(models.UserGroup(group_id=group.id, user_id=user.id)), - fallback=lambda: request.db.rollback(), - httpError=HTTPForbidden, msgOnFail=UserGroups_POST_RelationshipForbiddenResponseSchema.description, - content={u'user_name': user.user_name, u'group_name': group.group_name}) - return valid_http(httpSuccess=HTTPCreated, detail=UserGroups_POST_CreatedResponseSchema.description, - content={u'user_name': user.user_name, u'group_name': group.group_name}) + user = ar.get_user_matchdict_checked_or_logged(request) + + group_name = ar.get_value_multiformat_post_checked(request, 'group_name') + group = ax.evaluate_call(lambda: GroupService.by_group_name(group_name, db_session=request.db), + fallback=lambda: request.db.rollback(), + httpError=HTTPForbidden, msgOnFail=UserGroups_POST_ForbiddenResponseSchema.description) + ax.verify_param(group, notNone=True, httpError=HTTPNotFound, + msgOnFail=UserGroups_POST_GroupNotFoundResponseSchema.description) + ax.verify_param(user.id, paramCompare=[usr.id for usr in group.users], notIn=True, httpError=HTTPConflict, + content={u'user_name': user.user_name, u'group_name': group.group_name}, + msgOnFail=UserGroups_POST_ConflictResponseSchema.description) + # noinspection PyArgumentList + ax.evaluate_call(lambda: request.db.add(models.UserGroup(group_id=group.id, user_id=user.id)), + fallback=lambda: request.db.rollback(), + httpError=HTTPForbidden, msgOnFail=UserGroups_POST_RelationshipForbiddenResponseSchema.description, + content={u'user_name': user.user_name, u'group_name': group.group_name}) + return ax.valid_http(httpSuccess=HTTPCreated, detail=UserGroups_POST_CreatedResponseSchema.description, + content={u'user_name': user.user_name, u'group_name': group.group_name}) @UserGroupAPI.delete(schema=UserGroup_DELETE_RequestSchema(), tags=[UsersTag], @@ -134,11 +131,11 @@ def assign_user_group(request): @LoggedUserGroupAPI.delete(schema=UserGroup_DELETE_RequestSchema(), tags=[LoggedUserTag], response_schemas=LoggedUserGroup_DELETE_responses) @view_config(route_name=UserGroupAPI.name, request_method='DELETE') -def delete_user_group(request): +def delete_user_group_view(request): """Remove a user from a group.""" db = request.db - user = get_user_matchdict_checked_or_logged(request) - group = get_group_matchdict_checked(request) + user = ar.get_user_matchdict_checked_or_logged(request) + group = ar.get_group_matchdict_checked(request) def del_usr_grp(usr, grp): db.query(models.UserGroup) \ @@ -146,10 +143,10 @@ def del_usr_grp(usr, grp): .filter(models.UserGroup.group_id == grp.id) \ .delete() - evaluate_call(lambda: del_usr_grp(user, group), fallback=lambda: db.rollback(), - httpError=HTTPNotFound, msgOnFail=UserGroup_DELETE_NotFoundResponseSchema.description, - content={u'user_name': user.user_name, u'group_name': group.group_name}) - return valid_http(httpSuccess=HTTPOk, detail=UserGroup_DELETE_OkResponseSchema.description) + ax.evaluate_call(lambda: del_usr_grp(user, group), fallback=lambda: db.rollback(), + httpError=HTTPNotFound, msgOnFail=UserGroup_DELETE_NotFoundResponseSchema.description, + content={u'user_name': user.user_name, u'group_name': group.group_name}) + return ax.valid_http(httpSuccess=HTTPOk, detail=UserGroup_DELETE_OkResponseSchema.description) @UserResourcesAPI.get(schema=UserResources_GET_RequestSchema(), @@ -161,18 +158,18 @@ def del_usr_grp(usr, grp): @view_config(route_name=UserResourcesAPI.name, request_method='GET', permission=NO_PERMISSION_REQUIRED) def get_user_resources_view(request): """List all resources a user has permissions on.""" - inherit_groups_perms = str2bool(get_query_param(request, 'inherit')) - user = get_user_matchdict_checked_or_logged(request) + inherit_groups_perms = str2bool(ar.get_query_param(request, 'inherit')) + user = ar.get_user_matchdict_checked_or_logged(request) db = request.db def build_json_user_resource_tree(usr): json_res = {} for svc in models.Service.all(db_session=db): - svc_perms = get_user_service_permissions( + svc_perms = uu.get_user_service_permissions( user=usr, service=svc, request=request, inherit_groups_permissions=inherit_groups_perms) if svc.type not in json_res: json_res[svc.type] = {} - res_perms_dict = get_user_service_resources_permissions_dict( + res_perms_dict = uu.get_user_service_resources_permissions_dict( user=usr, service=svc, request=request, inherit_groups_permissions=inherit_groups_perms) json_res[svc.type][svc.resource_name] = format_service_resources( svc, @@ -184,13 +181,13 @@ def build_json_user_resource_tree(usr): ) return json_res - usr_res_dict = evaluate_call(lambda: build_json_user_resource_tree(user), - fallback=lambda: db.rollback(), httpError=HTTPNotFound, - msgOnFail=UserResources_GET_NotFoundResponseSchema.description, - content={u'user_name': user.user_name, - u'resource_types': [models.Service.resource_type_name]}) - return valid_http(httpSuccess=HTTPOk, detail=UserResources_GET_OkResponseSchema.description, - content={u'resources': usr_res_dict}) + usr_res_dict = ax.evaluate_call(lambda: build_json_user_resource_tree(user), + fallback=lambda: db.rollback(), httpError=HTTPNotFound, + msgOnFail=UserResources_GET_NotFoundResponseSchema.description, + content={u'user_name': user.user_name, + u'resource_types': [models.Service.resource_type_name]}) + return ax.valid_http(httpSuccess=HTTPOk, content={u'resources': usr_res_dict}, + detail=UserResources_GET_OkResponseSchema.description) @UserInheritedResourcesAPI.get(tags=[UsersTag], api_security=SecurityEveryoneAPI, @@ -215,15 +212,15 @@ def get_user_inherited_resources_view(request): @view_config(route_name=UserResourcePermissionsAPI.name, request_method='GET', permission=NO_PERMISSION_REQUIRED) def get_user_resource_permissions_view(request): """List all permissions a user has on a specific resource.""" - user = get_user_matchdict_checked_or_logged(request) - resource = get_resource_matchdict_checked(request, 'resource_id') - inherit_groups_perms = str2bool(get_query_param(request, 'inherit')) - effective_perms = str2bool(get_query_param(request, 'effective')) - perm_names = get_user_resource_permissions(resource=resource, user=user, request=request, - inherit_groups_permissions=inherit_groups_perms, - effective_permissions=effective_perms) - return valid_http(httpSuccess=HTTPOk, detail=UserResourcePermissions_GET_OkResponseSchema.description, - content={u'permission_names': sorted(perm_names)}) + user = ar.get_user_matchdict_checked_or_logged(request) + resource = ar.get_resource_matchdict_checked(request, 'resource_id') + inherit_groups_perms = str2bool(ar.get_query_param(request, 'inherit')) + effective_perms = str2bool(ar.get_query_param(request, 'effective')) + perm_names = uu.get_user_resource_permissions(resource=resource, user=user, request=request, + inherit_groups_permissions=inherit_groups_perms, + effective_permissions=effective_perms) + return ax.valid_http(httpSuccess=HTTPOk, content={u'permission_names': sorted(perm_names)}, + detail=UserResourcePermissions_GET_OkResponseSchema.description) @UserResourceInheritedPermissionsAPI.get(tags=[UsersTag], api_security=SecurityEveryoneAPI, @@ -247,10 +244,10 @@ def get_user_resource_inherit_groups_permissions_view(request): @view_config(route_name=UserResourcePermissionsAPI.name, request_method='POST') def create_user_resource_permission_view(request): """Create a permission on specific resource for a user.""" - user = get_user_matchdict_checked_or_logged(request) - resource = get_resource_matchdict_checked(request) - perm_name = get_permission_multiformat_post_checked(request, resource) - return create_user_resource_permission(perm_name, resource, user.id, request.db) + user = ar.get_user_matchdict_checked_or_logged(request) + resource = ar.get_resource_matchdict_checked(request) + perm_name = ar.get_permission_multiformat_post_checked(request, resource) + return uu.create_user_resource_permission(perm_name, resource, user.id, request.db) @UserResourcePermissionAPI.delete(schema=UserResourcePermission_DELETE_RequestSchema(), tags=[UsersTag], @@ -260,10 +257,10 @@ def create_user_resource_permission_view(request): @view_config(route_name=UserResourcePermissionAPI.name, request_method='DELETE') def delete_user_resource_permission_view(request): """Delete a direct permission on a resource for a user (not including his groups permissions).""" - user = get_user_matchdict_checked_or_logged(request) - resource = get_resource_matchdict_checked(request) - perm_name = get_permission_matchdict_checked(request, resource) - return delete_user_resource_permission(perm_name, resource, user.id, request.db) + user = ar.get_user_matchdict_checked_or_logged(request) + resource = ar.get_resource_matchdict_checked(request) + perm_name = ar.get_permission_matchdict_checked(request, resource) + return uu.delete_user_resource_permission(perm_name, resource, user.id, request.db) @UserServicesAPI.get(tags=[UsersTag], schema=UserServices_GET_RequestSchema, @@ -273,17 +270,17 @@ def delete_user_resource_permission_view(request): @view_config(route_name=UserServicesAPI.name, request_method='GET', permission=NO_PERMISSION_REQUIRED) def get_user_services_view(request): """List all services a user has permissions on.""" - user = get_user_matchdict_checked_or_logged(request) - cascade_resources = str2bool(get_query_param(request, 'cascade')) - inherit_groups_perms = str2bool(get_query_param(request, 'inherit')) - format_as_list = str2bool(get_query_param(request, 'list')) + user = ar.get_user_matchdict_checked_or_logged(request) + cascade_resources = str2bool(ar.get_query_param(request, 'cascade')) + inherit_groups_perms = str2bool(ar.get_query_param(request, 'inherit')) + format_as_list = str2bool(ar.get_query_param(request, 'list')) - svc_json = get_user_services(user, request=request, - cascade_resources=cascade_resources, - inherit_groups_permissions=inherit_groups_perms, - format_as_list=format_as_list) - return valid_http(httpSuccess=HTTPOk, detail=UserServices_GET_OkResponseSchema.description, - content={u'services': svc_json}) + svc_json = uu.get_user_services(user, request=request, + cascade_resources=cascade_resources, + inherit_groups_permissions=inherit_groups_perms, + format_as_list=format_as_list) + return ax.valid_http(httpSuccess=HTTPOk, content={u'services': svc_json}, + detail=UserServices_GET_OkResponseSchema.description) @UserInheritedServicesAPI.get(tags=[UsersTag], api_security=SecurityEveryoneAPI, @@ -324,16 +321,16 @@ def get_user_service_inherited_permissions_view(request): @view_config(route_name=UserServicePermissionsAPI.name, request_method='GET', permission=NO_PERMISSION_REQUIRED) def get_user_service_permissions_view(request): """List all permissions a user has on a service.""" - user = get_user_matchdict_checked_or_logged(request) - service = get_service_matchdict_checked(request) - inherit_groups_perms = str2bool(get_query_param(request, 'inherit')) - perms = evaluate_call(lambda: get_user_service_permissions(service=service, user=user, request=request, - inherit_groups_permissions=inherit_groups_perms), - fallback=lambda: request.db.rollback(), httpError=HTTPNotFound, - msgOnFail=UserServicePermissions_GET_NotFoundResponseSchema.description, - content={u'service_name': str(service.resource_name), u'user_name': str(user.user_name)}) - return valid_http(httpSuccess=HTTPOk, detail=UserServicePermissions_GET_OkResponseSchema.description, - content={u'permission_names': sorted(perms)}) + user = ar.get_user_matchdict_checked_or_logged(request) + service = ar.get_service_matchdict_checked(request) + inherit_groups_perms = str2bool(ar.get_query_param(request, 'inherit')) + perms = ax.evaluate_call(lambda: uu.get_user_service_permissions(service=service, user=user, request=request, + inherit_groups_permissions=inherit_groups_perms), + fallback=lambda: request.db.rollback(), httpError=HTTPNotFound, + msgOnFail=UserServicePermissions_GET_NotFoundResponseSchema.description, + content={u'service_name': str(service.resource_name), u'user_name': str(user.user_name)}) + return ax.valid_http(httpSuccess=HTTPOk, detail=UserServicePermissions_GET_OkResponseSchema.description, + content={u'permission_names': sorted(perms)}) @UserServicePermissionsAPI.post(schema=UserServicePermissions_POST_RequestSchema, tags=[UsersTag], @@ -341,12 +338,12 @@ def get_user_service_permissions_view(request): @LoggedUserServicePermissionsAPI.post(schema=UserServicePermissions_POST_RequestSchema, tags=[LoggedUserTag], response_schemas=LoggedUserServicePermissions_POST_responses) @view_config(route_name=UserServicePermissionsAPI.name, request_method='POST') -def create_user_service_permission(request): +def create_user_service_permission_view(request): """Create a permission on a service for a user.""" - user = get_user_matchdict_checked_or_logged(request) - service = get_service_matchdict_checked(request) - perm_name = get_permission_multiformat_post_checked(request, service) - return create_user_resource_permission(perm_name, service, user.id, request.db) + user = ar.get_user_matchdict_checked_or_logged(request) + service = ar.get_service_matchdict_checked(request) + perm_name = ar.get_permission_multiformat_post_checked(request, service) + return uu.create_user_resource_permission(perm_name, service, user.id, request.db) @UserServicePermissionAPI.delete(schema=UserServicePermission_DELETE_RequestSchema, tags=[UsersTag], @@ -354,12 +351,12 @@ def create_user_service_permission(request): @LoggedUserServicePermissionAPI.delete(schema=UserServicePermission_DELETE_RequestSchema, tags=[LoggedUserTag], response_schemas=LoggedUserServicePermission_DELETE_responses) @view_config(route_name=UserServicePermissionAPI.name, request_method='DELETE') -def delete_user_service_permission(request): +def delete_user_service_permission_view(request): """Delete a direct permission on a service for a user (not including his groups permissions).""" - user = get_user_matchdict_checked_or_logged(request) - service = get_service_matchdict_checked(request) - perm_name = get_permission_multiformat_post_checked(request, service) - return delete_user_resource_permission(perm_name, service, user.id, request.db) + user = ar.get_user_matchdict_checked_or_logged(request) + service = ar.get_service_matchdict_checked(request) + perm_name = ar.get_permission_multiformat_post_checked(request, service) + return uu.delete_user_resource_permission(perm_name, service, user.id, request.db) @UserServiceResourcesAPI.get(schema=UserServiceResources_GET_RequestSchema, @@ -371,12 +368,12 @@ def delete_user_service_permission(request): @view_config(route_name=UserServiceResourcesAPI.name, request_method='GET', permission=NO_PERMISSION_REQUIRED) def get_user_service_resources_view(request): """List all resources under a service a user has permission on.""" - inherit_groups_perms = str2bool(get_query_param(request, 'inherit')) - user = get_user_matchdict_checked_or_logged(request) - service = get_service_matchdict_checked(request) - service_perms = get_user_service_permissions( + inherit_groups_perms = str2bool(ar.get_query_param(request, 'inherit')) + user = ar.get_user_matchdict_checked_or_logged(request) + service = ar.get_service_matchdict_checked(request) + service_perms = uu.get_user_service_permissions( user, service, request=request, inherit_groups_permissions=inherit_groups_perms) - resources_perms_dict = get_user_service_resources_permissions_dict( + resources_perms_dict = uu.get_user_service_resources_permissions_dict( user, service, request=request, inherit_groups_permissions=inherit_groups_perms) user_svc_res_json = format_service_resources( service=service, @@ -386,8 +383,8 @@ def get_user_service_resources_view(request): display_all=False, show_private_url=False, ) - return valid_http(httpSuccess=HTTPOk, detail=UserServiceResources_GET_OkResponseSchema.description, - content={u'service': user_svc_res_json}) + return ax.valid_http(httpSuccess=HTTPOk, detail=UserServiceResources_GET_OkResponseSchema.description, + content={u'service': user_svc_res_json}) @UserServiceInheritedResourcesAPI.get(tags=[UsersTag], api_security=SecurityEveryoneAPI, diff --git a/setup.py b/setup.py index e4f40659f..49892ef4c 100644 --- a/setup.py +++ b/setup.py @@ -75,7 +75,7 @@ url=__meta__.__url__, platforms=['linux_x86_64'], license="ISCL", - keywords=__meta__.__title__ + ", Authentication, AuthN", + keywords=__meta__.__title__ + ", Authentication, AuthN, Birdhouse", classifiers=[ 'Development Status :: 2 - Pre-Alpha', 'Intended Audience :: Developers', diff --git a/tests/interfaces.py b/tests/interfaces.py index 2dc2d31d9..7eaa11afb 100644 --- a/tests/interfaces.py +++ b/tests/interfaces.py @@ -128,14 +128,7 @@ def setup_test_values(cls): cls.test_service_name = u'magpie-unittest-service-api' cls.test_service_type = u'api' - data = { - u'service_name': cls.test_service_name, - u'service_type': cls.test_service_type, - u'service_url': u'http://localhost:9000/dummy-api' - } - resp = utils.test_request(cls.url, 'POST', '/services', json=data, - headers=cls.json_headers, cookies=cls.cookies) - utils.check_response_basic_info(resp, 201, expected_method='POST') + utils.TestSetup.create_TestService(cls) resp = utils.test_request(cls.url, 'GET', '/services/{}'.format(cls.test_service_name), headers=cls.json_headers, cookies=cls.cookies) @@ -233,6 +226,8 @@ def check_GetUserResourcesPermissions(cls, user_name, resource_id=None, query=No @pytest.mark.users @unittest.skipUnless(runner.MAGPIE_TEST_USERS, reason=runner.MAGPIE_TEST_DISABLED_MESSAGE('users')) def test_GetCurrentUserResourcesPermissions(self): + utils.TestSetup.create_TestService(self) + utils.TestSetup.create_TestServiceResource(self) self.check_GetUserResourcesPermissions(get_constant('MAGPIE_LOGGED_USER')) @pytest.mark.users @@ -320,6 +315,8 @@ def test_GetCurrentUserResourcesPermissions_Queries(self): @pytest.mark.users @unittest.skipUnless(runner.MAGPIE_TEST_USERS, reason=runner.MAGPIE_TEST_DISABLED_MESSAGE('users')) def test_GetUserResourcesPermissions(self): + utils.TestSetup.create_TestService(self) + utils.TestSetup.create_TestServiceResource(self) self.check_GetUserResourcesPermissions(self.usr) @pytest.mark.users @@ -390,6 +387,8 @@ def test_GetCurrentUserGroups(self): @pytest.mark.users @unittest.skipUnless(runner.MAGPIE_TEST_USERS, reason=runner.MAGPIE_TEST_DISABLED_MESSAGE('users')) def test_GetUserInheritedResources(self): + utils.TestSetup.create_TestService(self) + utils.TestSetup.create_TestServiceResource(self) if LooseVersion(self.version) >= LooseVersion('0.7.0'): route = '/users/{usr}/inherited_resources'.format(usr=self.usr) else: @@ -418,7 +417,7 @@ def test_GetUserInheritedResources(self): utils.check_val_type(svc_dict['resources'], dict) if LooseVersion(self.version) >= LooseVersion('0.7.0'): utils.check_val_is_in('service_sync_type', svc_dict) - utils.check_val_type(svc_dict['service_sync_type'], six.string_types + tuple([type(None)])) + utils.check_val_type(svc_dict['service_sync_type'], utils.OptionalStringType) utils.check_val_not_in('service_url', svc_dict, msg="Services under user routes shouldn't show private url.") else: @@ -456,7 +455,7 @@ def test_GetUserServices(self): utils.check_val_type(svc_dict['permission_names'], list) if LooseVersion(self.version) >= LooseVersion('0.7.0'): utils.check_val_is_in('service_sync_type', svc_dict) - utils.check_val_type(svc_dict['service_sync_type'], six.string_types) + utils.check_val_type(svc_dict['service_sync_type'], utils.OptionalStringType) utils.check_val_not_in('service_url', svc_dict, msg="Services under user routes shouldn't show private url.") else: @@ -466,6 +465,8 @@ def test_GetUserServices(self): @pytest.mark.users @unittest.skipUnless(runner.MAGPIE_TEST_USERS, reason=runner.MAGPIE_TEST_DISABLED_MESSAGE('users')) def test_GetUserServiceResources(self): + utils.TestSetup.create_TestService(self) + utils.TestSetup.create_TestServiceResource(self) route = '/users/{usr}/services/{svc}/resources'.format(usr=self.usr, svc=self.test_service_name) resp = utils.test_request(self.url, 'GET', route, headers=self.json_headers, cookies=self.cookies) json_body = utils.check_response_basic_info(resp, 200, expected_method='GET') @@ -486,7 +487,7 @@ def test_GetUserServiceResources(self): utils.check_val_type(svc_dict['resources'], dict) if LooseVersion(self.version) >= LooseVersion('0.7.0'): utils.check_val_is_in('service_sync_type', svc_dict) - utils.check_val_type(svc_dict['service_sync_type'], six.string_types) + utils.check_val_type(svc_dict['service_sync_type'], utils.OptionalStringType) utils.check_val_not_in('service_url', svc_dict) else: utils.check_val_is_in('service_url', svc_dict) @@ -729,7 +730,7 @@ def test_GetGroupServices(self): utils.check_val_type(svc_dict['permission_names'], list) if LooseVersion(self.version) >= LooseVersion('0.7.0'): utils.check_val_is_in('service_sync_type', svc_dict) - utils.check_val_type(svc_dict['service_sync_type'], six.string_types + tuple([type(None)])) + utils.check_val_type(svc_dict['service_sync_type'], utils.OptionalStringType) utils.check_val_not_in('service_url', svc_dict) else: utils.check_val_is_in('service_url', svc_dict) @@ -738,6 +739,8 @@ def test_GetGroupServices(self): @pytest.mark.groups @unittest.skipUnless(runner.MAGPIE_TEST_USERS, reason=runner.MAGPIE_TEST_DISABLED_MESSAGE('groups')) def test_GetGroupServiceResources(self): + utils.TestSetup.create_TestService(self) + utils.TestSetup.create_TestServiceResource(self) route = '/groups/{grp}/services/{svc}/resources'.format(grp=self.grp, svc=self.test_service_name) resp = utils.test_request(self.url, 'GET', route, headers=self.json_headers, cookies=self.cookies) json_body = utils.check_response_basic_info(resp, 200, expected_method='GET') @@ -758,7 +761,7 @@ def test_GetGroupServiceResources(self): utils.check_val_type(svc_dict['resources'], dict) if LooseVersion(self.version) >= LooseVersion('0.7.0'): utils.check_val_is_in('service_sync_type', svc_dict) - utils.check_val_type(svc_dict['service_sync_type'], six.string_types) + utils.check_val_type(svc_dict['service_sync_type'], utils.OptionalStringType) utils.check_val_not_in('service_url', svc_dict) else: utils.check_val_is_in('service_url', svc_dict) @@ -767,6 +770,8 @@ def test_GetGroupServiceResources(self): @pytest.mark.services @unittest.skipUnless(runner.MAGPIE_TEST_SERVICES, reason=runner.MAGPIE_TEST_DISABLED_MESSAGE('services')) def test_GetServiceResources(self): + utils.TestSetup.create_TestService(self) + utils.TestSetup.create_TestServiceResource(self) route = '/services/{svc}/resources'.format(svc=self.test_service_name) resp = utils.test_request(self.url, 'GET', route, headers=self.json_headers, cookies=self.cookies) json_body = utils.check_response_basic_info(resp, 200, expected_method='GET') @@ -789,7 +794,7 @@ def test_GetServiceResources(self): utils.check_resource_children(svc_dict['resources'], svc_dict['resource_id'], svc_dict['resource_id']) if LooseVersion(self.version) >= LooseVersion('0.7.0'): utils.check_val_is_in('service_sync_type', svc_dict) - utils.check_val_type(svc_dict['service_sync_type'], six.string_types) + utils.check_val_type(svc_dict['service_sync_type'], utils.OptionalStringType) @pytest.mark.services @unittest.skipUnless(runner.MAGPIE_TEST_SERVICES, reason=runner.MAGPIE_TEST_DISABLED_MESSAGE('services')) @@ -809,6 +814,7 @@ def test_GetServicePermissions(self): @pytest.mark.services @unittest.skipUnless(runner.MAGPIE_TEST_SERVICES, reason=runner.MAGPIE_TEST_DISABLED_MESSAGE('services')) def test_PostServiceResources_DirectResource_NoParentID(self): + utils.TestSetup.create_TestService(self) resources_prior = utils.TestSetup.get_TestServiceDirectResources(self) resources_prior_ids = [res['resource_id'] for res in resources_prior] json_body = utils.TestSetup.create_TestServiceResource(self) @@ -825,6 +831,7 @@ def test_PostServiceResources_DirectResource_NoParentID(self): @pytest.mark.services @unittest.skipUnless(runner.MAGPIE_TEST_SERVICES, reason=runner.MAGPIE_TEST_DISABLED_MESSAGE('services')) def test_PostServiceResources_DirectResource_WithParentID(self): + utils.TestSetup.create_TestService(self) resources_prior = utils.TestSetup.get_TestServiceDirectResources(self) resources_prior_ids = [res['resource_id'] for res in resources_prior] service_id = utils.TestSetup.get_ExistingTestServiceInfo(self)['resource_id'] @@ -944,6 +951,7 @@ def test_ValidateDefaultServiceProviders(self): @pytest.mark.resources @unittest.skipUnless(runner.MAGPIE_TEST_RESOURCES, reason=runner.MAGPIE_TEST_DISABLED_MESSAGE('resources')) def test_PostResources_DirectServiceResource(self): + utils.TestSetup.create_TestService(self) service_info = utils.TestSetup.get_ExistingTestServiceInfo(self) service_resource_id = service_info['resource_id'] @@ -962,6 +970,7 @@ def test_PostResources_DirectServiceResource(self): @pytest.mark.resources @unittest.skipUnless(runner.MAGPIE_TEST_RESOURCES, reason=runner.MAGPIE_TEST_DISABLED_MESSAGE('resources')) def test_PostResources_DirectServiceResourceOptional(self): + utils.TestSetup.create_TestService(self) service_info = utils.TestSetup.get_ExistingTestServiceInfo(self) service_resource_id = service_info['resource_id'] diff --git a/tests/test_magpie_adapter.py b/tests/test_magpie_adapter.py index e69de29bb..1b9975e4d 100644 --- a/tests/test_magpie_adapter.py +++ b/tests/test_magpie_adapter.py @@ -0,0 +1,9 @@ +import pytest +import unittest + + +@unittest.skip("not implemented") +class TestAdapter(unittest.TestCase): + @pytest.skip("not implemented") + def test_adapter(self): + pass diff --git a/tests/test_register.py b/tests/test_register.py new file mode 100644 index 000000000..8d500a4c5 --- /dev/null +++ b/tests/test_register.py @@ -0,0 +1,9 @@ +import pytest +import unittest + + +@unittest.skip("not implemented") +class TestRegister(unittest.TestCase): + @pytest.skip("not implemented") + def test_register(self): + pass diff --git a/tests/utils.py b/tests/utils.py index 010e9ec20..759a02b80 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -9,6 +9,8 @@ from magpie import __meta__, db, services, magpiectl from magpie.constants import get_constant +OptionalStringType = six.string_types + tuple([type(None)]) + def config_setup_from_ini(config_ini_file_path): settings = db.get_settings_from_config_ini(config_ini_file_path) @@ -409,6 +411,7 @@ def get_AnyServiceOfTestServiceType(test_class): @staticmethod def create_TestServiceResource(test_class, data_override=None): + TestSetup.create_TestService(test_class) route = '/services/{svc}/resources'.format(svc=test_class.test_service_name) data = { "resource_name": test_class.test_resource_name, @@ -462,6 +465,20 @@ def delete_TestServiceResource(test_class, override_resource_name=None): check_val_equal(resp.status_code, 200) TestSetup.check_NonExistingTestServiceResource(test_class) + @staticmethod + def create_TestService(test_class): + data = { + u'service_name': test_class.test_service_name, + u'service_type': test_class.test_service_type, + u'service_url': u'http://localhost:9000/{}'.format(test_class.test_service_name) + } + resp = test_request(test_class.url, 'POST', '/services', json=data, + headers=test_class.json_headers, cookies=test_class.cookies, + expect_errors=True) + if resp.status_code == 409: + return + check_response_basic_info(resp, 201, expected_method='POST') + @staticmethod def check_NonExistingTestService(test_class): services_info = TestSetup.get_RegisteredServicesList(test_class) diff --git a/tox.ini b/tox.ini index c23f581b9..7b1a978ba 100644 --- a/tox.ini +++ b/tox.ini @@ -5,5 +5,4 @@ envlist = py27, py35, py36 setenv = PYTHONPATH = {toxinidir}:{toxinidir}/magpie commands = python setup.py test -deps = - -r{toxinidir}/requirements.txt +#deps =-r {toxinidir}/requirements.txt From 047f4510f3e9449050c4f63b57b5448dc1e8a921 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Tue, 29 Jan 2019 22:16:59 -0500 Subject: [PATCH 11/76] more testing --- .../api/management/service/service_utils.py | 13 +++++++ .../api/management/service/service_views.py | 16 ++------ tests/interfaces.py | 39 +++++++++++++------ tests/utils.py | 9 +++-- 4 files changed, 51 insertions(+), 26 deletions(-) diff --git a/magpie/api/management/service/service_utils.py b/magpie/api/management/service/service_utils.py index 592239f51..cd8d75c7c 100644 --- a/magpie/api/management/service/service_utils.py +++ b/magpie/api/management/service/service_utils.py @@ -7,6 +7,19 @@ from magpie.api.management.group.group_utils import create_group_resource_permission +def create_service(service_name, service_type, service_url, db_session): + """Generates an instance to register a new service.""" + # noinspection PyArgumentList + return evaluate_call(lambda: models.Service(resource_name=str(service_name), + resource_type=models.Service.resource_type_name, + url=str(service_url), type=str(service_type)), + fallback=lambda: db_session.rollback(), httpError=HTTPForbidden, + msgOnFail="Service creation for registration failed.", + content={u'service_name': str(service_name), + u'resource_type': models.Service.resource_type_name, + u'service_url': str(service_url), u'service_type': str(service_type)}) + + def get_services_by_type(service_type, db_session): verify_param(service_type, notNone=True, notEmpty=True, httpError=HTTPNotAcceptable, msgOnFail="Invalid `service_type` value '" + str(service_type) + "' specified") diff --git a/magpie/api/management/service/service_views.py b/magpie/api/management/service/service_views.py index 948ad94a4..f73b42fa9 100644 --- a/magpie/api/management/service/service_views.py +++ b/magpie/api/management/service/service_views.py @@ -60,25 +60,17 @@ def register_service(request): httpError=HTTPConflict, msgOnFail=Services_POST_ConflictResponseSchema.description, content={u'service_name': str(service_name)}, paramName=u'service_name') - # noinspection PyArgumentList - service = evaluate_call(lambda: models.Service(resource_name=str(service_name), - resource_type=models.Service.resource_type_name, - url=str(service_url), type=str(service_type)), - fallback=lambda: request.db.rollback(), httpError=HTTPForbidden, - msgOnFail="Service creation for registration failed.", - content={u'service_name': str(service_name), - u'resource_type': models.Service.resource_type_name, - u'service_url': str(service_url), u'service_type': str(service_type)}) - - def add_service_magpie_and_phoenix(svc, svc_push, db): + def _add_service_magpie_and_phoenix(svc, svc_push, db): db.add(svc) if svc_push and svc.type in SERVICES_PHOENIX_ALLOWED: sync_services_phoenix(db.query(models.Service)) - evaluate_call(lambda: add_service_magpie_and_phoenix(service, service_push, request.db), + service = create_service(service_name, service_type, service_url, db_session=request.db) + evaluate_call(lambda: _add_service_magpie_and_phoenix(service, service_push, request.db), fallback=lambda: request.db.rollback(), httpError=HTTPForbidden, msgOnFail=Services_POST_ForbiddenResponseSchema.description, content=format_service(service, show_private_url=True)) + return valid_http(httpSuccess=HTTPCreated, detail=Services_POST_CreatedResponseSchema.description, content={u'service': format_service(service, show_private_url=True)}) diff --git a/tests/interfaces.py b/tests/interfaces.py index 7eaa11afb..a3237a0a5 100644 --- a/tests/interfaces.py +++ b/tests/interfaces.py @@ -130,11 +130,6 @@ def setup_test_values(cls): cls.test_service_type = u'api' utils.TestSetup.create_TestService(cls) - resp = utils.test_request(cls.url, 'GET', '/services/{}'.format(cls.test_service_name), - headers=cls.json_headers, cookies=cls.cookies) - json_body = utils.check_response_basic_info(resp, 200, expected_method='GET') - cls.test_service_resource_id = json_body[cls.test_service_name]['resource_id'] - cls.test_resource_name = u'magpie-unittest-resource' test_service_res_perm_dict = service_type_dict[cls.test_service_type].resource_types_permissions test_service_resource_types = test_service_res_perm_dict.keys() @@ -213,8 +208,7 @@ def test_ValidateDefaultUsers(self): utils.check_val_is_in(get_constant('MAGPIE_ADMIN_USER'), users) @classmethod - def check_GetUserResourcesPermissions(cls, user_name, resource_id=None, query=None): - resource_id = resource_id or cls.test_service_resource_id + def check_GetUserResourcesPermissions(cls, user_name, resource_id, query=None): query = '?{}'.format(query) if query else '' route = '/users/{usr}/resources/{res_id}/permissions{q}'.format(res_id=resource_id, usr=user_name, q=query) resp = utils.test_request(cls.url, 'GET', route, headers=cls.json_headers, cookies=cls.cookies) @@ -242,8 +236,9 @@ def test_GetCurrentUserResourcesPermissions_Queries(self): # test-service | r | r-m | | r # |- test-resource (parent) | | r-m | | # |- test-resource (child) | | | r-m | + json_body = utils.TestSetup.create_TestService(self) + test_svc_res_id = json_body['service']['resource_id'] json_body = utils.TestSetup.create_TestServiceResource(self) - test_svc_res_id = self.test_service_resource_id test_parent_res_id = json_body['resource']['resource_id'] child_resource_name = self.test_resource_name + "-child" data_override = { @@ -316,8 +311,8 @@ def test_GetCurrentUserResourcesPermissions_Queries(self): @unittest.skipUnless(runner.MAGPIE_TEST_USERS, reason=runner.MAGPIE_TEST_DISABLED_MESSAGE('users')) def test_GetUserResourcesPermissions(self): utils.TestSetup.create_TestService(self) - utils.TestSetup.create_TestServiceResource(self) - self.check_GetUserResourcesPermissions(self.usr) + json_body = utils.TestSetup.create_TestServiceResource(self) + self.check_GetUserResourcesPermissions(self.usr, json_body['resource']['resource_id']) @pytest.mark.users @unittest.skipUnless(runner.MAGPIE_TEST_USERS, reason=runner.MAGPIE_TEST_DISABLED_MESSAGE('users')) @@ -393,7 +388,7 @@ def test_GetUserInheritedResources(self): route = '/users/{usr}/inherited_resources'.format(usr=self.usr) else: route = '/users/{usr}/resources?inherit=true'.format(usr=self.usr) - resp = utils.test_request(self.url, 'GET', route, headers=self.json_headers, cookies=self.cookies) + resp = utils.test_request(self.url, 'GET', route, headers=self.json_headers, cookies=self.cookies, timeout=20) json_body = utils.check_response_basic_info(resp, 200, expected_method='GET') utils.check_val_is_in('resources', json_body) utils.check_val_type(json_body['resources'], dict) @@ -767,6 +762,28 @@ def test_GetGroupServiceResources(self): utils.check_val_is_in('service_url', svc_dict) utils.check_val_type(svc_dict['service_url'], six.string_types) + @pytest.mark.services + @unittest.skipUnless(runner.MAGPIE_TEST_SERVICES, reason=runner.MAGPIE_TEST_DISABLED_MESSAGE('services')) + def test_PostService_ResponseFormat(self): + json_body = utils.TestSetup.create_TestService(self) + utils.check_val_is_in('service', json_body) + utils.check_val_type(json_body['service'], dict) + utils.check_val_is_in('public_url', json_body['service']) + utils.check_val_is_in('resource_id', json_body['service']) + utils.check_val_is_in('service_url', json_body['service']) + utils.check_val_is_in('service_name', json_body['service']) + utils.check_val_is_in('service_type', json_body['service']) + utils.check_val_is_in('permission_names', json_body['service']) + utils.check_val_type(json_body['service']['public_url'], six.string_types) + utils.check_val_type(json_body['service']['resource_id'], int) + utils.check_val_type(json_body['service']['service_url'], six.string_types) + utils.check_val_type(json_body['service']['service_name'], six.string_types) + utils.check_val_type(json_body['service']['service_type'], six.string_types) + utils.check_val_type(json_body['service']['permission_names'], list) + if LooseVersion(self.version) >= LooseVersion('0.7.0'): + utils.check_val_is_in('service_sync_type', json_body['service']) + utils.check_val_type(json_body['service']['service_sync_type'], utils.OptionalStringType) + @pytest.mark.services @unittest.skipUnless(runner.MAGPIE_TEST_SERVICES, reason=runner.MAGPIE_TEST_DISABLED_MESSAGE('services')) def test_GetServiceResources(self): diff --git a/tests/utils.py b/tests/utils.py index 759a02b80..a83e54e6c 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -382,7 +382,7 @@ def check_UpStatus(test_class, method, path): Verifies that the Magpie UI page at very least returned an Ok response with the displayed title. Validates that at the bare minimum, no underlying internal error occurred from the API or UI calls. """ - resp = test_request(test_class.url, method, path, cookies=test_class.cookies) + resp = test_request(test_class.url, method, path, cookies=test_class.cookies, timeout=10) check_val_equal(resp.status_code, 200) check_val_is_in('Content-Type', dict(resp.headers)) check_val_is_in('text/html', get_response_content_types_list(resp)) @@ -476,8 +476,11 @@ def create_TestService(test_class): headers=test_class.json_headers, cookies=test_class.cookies, expect_errors=True) if resp.status_code == 409: - return - check_response_basic_info(resp, 201, expected_method='POST') + resp = test_request(test_class.url, 'GET', '/services', + headers=test_class.json_headers, + cookies=test_class.cookies) + return check_response_basic_info(resp, 200, expected_method='GET') + return check_response_basic_info(resp, 201, expected_method='POST') @staticmethod def check_NonExistingTestService(test_class): From c6a42b7c72432996710c59504d5ecf4d705fa9b7 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Wed, 30 Jan 2019 11:15:18 -0500 Subject: [PATCH 12/76] fix res id missing svc registration --- magpie/api/api_rest_schemas.py | 13 ++ .../api/management/service/service_utils.py | 63 +++++-- .../api/management/service/service_views.py | 167 ++++++++---------- tests/interfaces.py | 9 +- 4 files changed, 143 insertions(+), 109 deletions(-) diff --git a/magpie/api/api_rest_schemas.py b/magpie/api/api_rest_schemas.py index cde413990..a3a08b006 100644 --- a/magpie/api/api_rest_schemas.py +++ b/magpie/api/api_rest_schemas.py @@ -965,6 +965,18 @@ class Services_POST_ConflictResponseSchema(colander.MappingSchema): body = BaseResponseBodySchema(code=HTTPConflict.code, description=description) +class Services_POST_UnprocessableEntityResponseSchema(colander.MappingSchema): + description = "Service creation for registration failed." + header = HeaderResponseSchema() + body = BaseResponseBodySchema(code=HTTPUnprocessableEntity.code, description=description) + + +class Services_POST_InternalServerErrorResponseSchema(colander.MappingSchema): + description = "Service registration status could not be validated." + header = HeaderResponseSchema() + body = BaseResponseBodySchema(code=HTTPInternalServerError.code, description=description) + + class Service_PUT_ResponseBodySchema(colander.MappingSchema): service_name = colander.SchemaNode( colander.String(), @@ -2357,6 +2369,7 @@ class Version_GET_OkResponseSchema(colander.MappingSchema): '401': UnauthorizedResponseSchema(), '403': Services_POST_ForbiddenResponseSchema(), '409': Services_POST_ConflictResponseSchema(), + '422': Services_POST_UnprocessableEntityResponseSchema(), } Service_GET_responses = { '200': Service_GET_OkResponseSchema(), diff --git a/magpie/api/management/service/service_utils.py b/magpie/api/management/service/service_utils.py index cd8d75c7c..86a14d63f 100644 --- a/magpie/api/management/service/service_utils.py +++ b/magpie/api/management/service/service_utils.py @@ -1,28 +1,61 @@ -from magpie import models +from magpie.api import api_except as ax +from magpie.api.api_rest_schemas import * +from magpie.api.management.group.group_utils import create_group_resource_permission +from magpie.api.management.service.service_formats import format_service from magpie.constants import get_constant -from magpie.register import SERVICES_PHOENIX_ALLOWED +from magpie.definitions.sqlalchemy_definitions import Session +from magpie.definitions.pyramid_definitions import * from magpie.definitions.ziggurat_definitions import * +from magpie.register import sync_services_phoenix, SERVICES_PHOENIX_ALLOWED from magpie.services import service_type_dict -from magpie.api.api_except import * -from magpie.api.management.group.group_utils import create_group_resource_permission +from magpie import models +from typing import AnyStr +import logging +LOGGER = logging.getLogger(__name__) -def create_service(service_name, service_type, service_url, db_session): +def create_service(service_name, service_type, service_url, service_push, db_session): + # type: (AnyStr, AnyStr, AnyStr, bool, Session) -> HTTPException """Generates an instance to register a new service.""" + + def _add_service_magpie_and_phoenix(svc, svc_push, db): + db.add(svc) + if svc_push and svc.type in SERVICES_PHOENIX_ALLOWED: + sync_services_phoenix(db.query(models.Service)) + + # sometimes, resource ID is not updated, fetch the service to obtain it + if not svc.resource_id: + svc = ax.evaluate_call(lambda: models.Service.by_service_name(service_name, db_session=db_session), + fallback=lambda: db_session.rollback(), httpError=HTTPInternalServerError, + msgOnFail=Services_POST_InternalServerErrorResponseSchema.description, + content={u'service_name': str(service_name), u'resource_id': svc.resource_id}) + ax.verify_param(svc.resource_id, notNone=True, ofType=int, httpError=HTTPInternalServerError, + msgOnFail=Services_POST_InternalServerErrorResponseSchema.description, + content={u'service_name': str(service_name), u'resource_id': svc.resource_id}, + paramName=u'service_name') + return svc + # noinspection PyArgumentList - return evaluate_call(lambda: models.Service(resource_name=str(service_name), - resource_type=models.Service.resource_type_name, - url=str(service_url), type=str(service_type)), - fallback=lambda: db_session.rollback(), httpError=HTTPForbidden, - msgOnFail="Service creation for registration failed.", - content={u'service_name': str(service_name), - u'resource_type': models.Service.resource_type_name, - u'service_url': str(service_url), u'service_type': str(service_type)}) + service = ax.evaluate_call(lambda: models.Service(resource_name=str(service_name), + resource_type=models.Service.resource_type_name, + url=str(service_url), type=str(service_type)), + fallback=lambda: db_session.rollback(), httpError=HTTPForbidden, + msgOnFail=Services_POST_UnprocessableEntityResponseSchema.description, + content={u'service_name': str(service_name), + u'resource_type': models.Service.resource_type_name, + u'service_url': str(service_url), u'service_type': str(service_type)}) + + service = ax.evaluate_call(lambda: _add_service_magpie_and_phoenix(service, service_push, db_session), + fallback=lambda: db_session.rollback(), httpError=HTTPForbidden, + msgOnFail=Services_POST_ForbiddenResponseSchema.description, + content=format_service(service, show_private_url=True)) + return ax.valid_http(httpSuccess=HTTPCreated, detail=Services_POST_CreatedResponseSchema.description, + content={u'service': format_service(service, show_private_url=True)}) def get_services_by_type(service_type, db_session): - verify_param(service_type, notNone=True, notEmpty=True, httpError=HTTPNotAcceptable, - msgOnFail="Invalid `service_type` value '" + str(service_type) + "' specified") + ax.verify_param(service_type, notNone=True, notEmpty=True, httpError=HTTPNotAcceptable, + msgOnFail="Invalid `service_type` value '" + str(service_type) + "' specified") services = db_session.query(models.Service).filter(models.Service.type == service_type) return sorted(services) diff --git a/magpie/api/management/service/service_views.py b/magpie/api/management/service/service_views.py index f73b42fa9..24f93d36a 100644 --- a/magpie/api/management/service/service_views.py +++ b/magpie/api/management/service/service_views.py @@ -1,13 +1,12 @@ from magpie.api.management.resource.resource_utils import create_resource, delete_resource -from magpie.api.management.service.service_formats import * -from magpie.api.management.service.service_utils import * -from magpie.api.api_requests import * +from magpie.api.management.service import service_formats as sf, service_utils as su +from magpie.api import api_requests as ar, api_except as ax from magpie.api.api_rest_schemas import * from magpie.definitions.pyramid_definitions import view_config from magpie.common import str2bool -from magpie.register import sync_services_phoenix -from magpie.models import resource_tree_service +from magpie.register import sync_services_phoenix, SERVICES_PHOENIX_ALLOWED from magpie.services import service_type_dict +from magpie import models @ServiceTypesAPI.get(tags=[ServicesTag], response_schemas=ServiceTypes_GET_responses) @@ -30,19 +29,19 @@ def get_services_runner(request): if not service_type_filter: service_types = service_type_dict.keys() else: - verify_param(service_type_filter, paramCompare=service_type_dict.keys(), isIn=True, httpError=HTTPNotAcceptable, - msgOnFail=Services_GET_NotAcceptableResponseSchema.description, - content={u'service_type': str(service_type_filter)}, contentType='application/json') + ax.verify_param(service_type_filter, paramCompare=service_type_dict.keys(), isIn=True, + httpError=HTTPNotAcceptable, msgOnFail=Services_GET_NotAcceptableResponseSchema.description, + content={u'service_type': str(service_type_filter)}, contentType='application/json') service_types = [service_type_filter] for service_type in service_types: - services = get_services_by_type(service_type, db_session=request.db) + services = su.get_services_by_type(service_type, db_session=request.db) json_response[service_type] = {} for service in services: - json_response[service_type][service.resource_name] = format_service(service, show_private_url=True) + json_response[service_type][service.resource_name] = sf.format_service(service, show_private_url=True) - return valid_http(httpSuccess=HTTPOk, detail=Services_GET_OkResponseSchema.description, - content={u'services': json_response}) + return ax.valid_http(httpSuccess=HTTPOk, content={u'services': json_response}, + detail=Services_GET_OkResponseSchema.description) @ServicesAPI.post(schema=Services_POST_RequestBodySchema(), tags=[ServicesTag], @@ -50,55 +49,42 @@ def get_services_runner(request): @view_config(route_name=ServicesAPI.name, request_method='POST') def register_service(request): """Registers a new service.""" - service_name = get_value_multiformat_post_checked(request, 'service_name') - service_url = get_value_multiformat_post_checked(request, 'service_url') - service_type = get_value_multiformat_post_checked(request, 'service_type') - service_push = str2bool(get_multiformat_post(request, 'service_push')) - verify_param(service_type, isIn=True, paramCompare=service_type_dict.keys(), - httpError=HTTPBadRequest, msgOnFail=Services_POST_BadRequestResponseSchema.description) - verify_param(models.Service.by_service_name(service_name, db_session=request.db), isNone=True, - httpError=HTTPConflict, msgOnFail=Services_POST_ConflictResponseSchema.description, - content={u'service_name': str(service_name)}, paramName=u'service_name') - - def _add_service_magpie_and_phoenix(svc, svc_push, db): - db.add(svc) - if svc_push and svc.type in SERVICES_PHOENIX_ALLOWED: - sync_services_phoenix(db.query(models.Service)) - - service = create_service(service_name, service_type, service_url, db_session=request.db) - evaluate_call(lambda: _add_service_magpie_and_phoenix(service, service_push, request.db), - fallback=lambda: request.db.rollback(), httpError=HTTPForbidden, - msgOnFail=Services_POST_ForbiddenResponseSchema.description, - content=format_service(service, show_private_url=True)) - - return valid_http(httpSuccess=HTTPCreated, detail=Services_POST_CreatedResponseSchema.description, - content={u'service': format_service(service, show_private_url=True)}) + service_name = ar.get_value_multiformat_post_checked(request, 'service_name') + service_url = ar.get_value_multiformat_post_checked(request, 'service_url') + service_type = ar.get_value_multiformat_post_checked(request, 'service_type') + service_push = str2bool(ar.get_multiformat_post(request, 'service_push')) + ax.verify_param(service_type, isIn=True, paramCompare=service_type_dict.keys(), + httpError=HTTPBadRequest, msgOnFail=Services_POST_BadRequestResponseSchema.description) + ax.verify_param(models.Service.by_service_name(service_name, db_session=request.db), isNone=True, + httpError=HTTPConflict, msgOnFail=Services_POST_ConflictResponseSchema.description, + content={u'service_name': str(service_name)}, paramName=u'service_name') + return su.create_service(service_name, service_type, service_url, service_push, db_session=request.db) @ServiceAPI.put(schema=Service_PUT_RequestBodySchema(), tags=[ServicesTag], response_schemas=Service_PUT_responses) @view_config(route_name=ServiceAPI.name, request_method='PUT') def update_service(request): """Update a service information.""" - service = get_service_matchdict_checked(request) - service_push = str2bool(get_multiformat_post(request, 'service_push', default=False)) + service = ar.get_service_matchdict_checked(request) + service_push = str2bool(ar.get_multiformat_post(request, 'service_push', default=False)) def select_update(new_value, old_value): return new_value if new_value is not None and not new_value == '' else old_value # None/Empty values are accepted in case of unspecified - svc_name = select_update(get_multiformat_post(request, 'service_name'), service.resource_name) - svc_url = select_update(get_multiformat_post(request, 'service_url'), service.url) - verify_param(svc_name == service.resource_name and svc_url == service.url, notEqual=True, paramCompare=True, - httpError=HTTPBadRequest, msgOnFail=Service_PUT_BadRequestResponseSchema.description) + svc_name = select_update(ar.get_multiformat_post(request, 'service_name'), service.resource_name) + svc_url = select_update(ar.get_multiformat_post(request, 'service_url'), service.url) + ax.verify_param(svc_name == service.resource_name and svc_url == service.url, notEqual=True, paramCompare=True, + httpError=HTTPBadRequest, msgOnFail=Service_PUT_BadRequestResponseSchema.description) if svc_name != service.resource_name: all_svc_names = list() for svc_type in service_type_dict: - for svc in get_services_by_type(svc_type, db_session=request.db): + for svc in su.get_services_by_type(svc_type, db_session=request.db): all_svc_names.extend(svc.resource_name) - verify_param(svc_name, notIn=True, paramCompare=all_svc_names, httpError=HTTPConflict, - msgOnFail=Service_PUT_ConflictResponseSchema.description, - content={u'service_name': str(svc_name)}) + ax.verify_param(svc_name, notIn=True, paramCompare=all_svc_names, httpError=HTTPConflict, + msgOnFail=Service_PUT_ConflictResponseSchema.description, + content={u'service_name': str(svc_name)}) def update_service_magpie_and_phoenix(_svc, new_name, new_url, svc_push, db_session): _svc.resource_name = new_name @@ -106,61 +92,62 @@ def update_service_magpie_and_phoenix(_svc, new_name, new_url, svc_push, db_sess has_getcap = 'getcapabilities' in service_type_dict[_svc.type].permission_names if svc_push and svc.type in SERVICES_PHOENIX_ALLOWED and has_getcap: # (re)apply getcapabilities to updated service to ensure updated push - add_service_getcapabilities_perms(_svc, db_session) + su.add_service_getcapabilities_perms(_svc, db_session) sync_services_phoenix(db_session.query(models.Service)) # push all services - old_svc_content = format_service(service, show_private_url=True) + old_svc_content = sf.format_service(service, show_private_url=True) err_svc_content = {u'service': old_svc_content, u'new_service_name': svc_name, u'new_service_url': svc_url} - evaluate_call(lambda: update_service_magpie_and_phoenix(service, svc_name, svc_url, service_push, request.db), - fallback=lambda: request.db.rollback(), - httpError=HTTPForbidden, msgOnFail=Service_PUT_ForbiddenResponseSchema.description, - content=err_svc_content) - return valid_http(httpSuccess=HTTPOk, detail=Service_PUT_OkResponseSchema.description, - content={u'service': format_service(service, show_private_url=True)}) + ax.evaluate_call(lambda: update_service_magpie_and_phoenix(service, svc_name, svc_url, service_push, request.db), + fallback=lambda: request.db.rollback(), + httpError=HTTPForbidden, msgOnFail=Service_PUT_ForbiddenResponseSchema.description, + content=err_svc_content) + return ax.valid_http(httpSuccess=HTTPOk, detail=Service_PUT_OkResponseSchema.description, + content={u'service': sf.format_service(service, show_private_url=True)}) @ServiceAPI.get(tags=[ServicesTag], response_schemas=Service_GET_responses) @view_config(route_name=ServiceAPI.name, request_method='GET') def get_service(request): """Get a service information.""" - service = get_service_matchdict_checked(request) - return valid_http(httpSuccess=HTTPOk, detail=Service_GET_OkResponseSchema.description, - content={service.resource_name: format_service(service, show_private_url=True)}) + service = ar.get_service_matchdict_checked(request) + return ax.valid_http(httpSuccess=HTTPOk, detail=Service_GET_OkResponseSchema.description, + content={service.resource_name: sf.format_service(service, show_private_url=True)}) @ServiceAPI.delete(schema=Service_DELETE_RequestSchema(), tags=[ServicesTag], response_schemas=Service_DELETE_responses) @view_config(route_name=ServiceAPI.name, request_method='DELETE') def unregister_service(request): """Unregister a service.""" - service = get_service_matchdict_checked(request) - service_push = str2bool(get_multiformat_delete(request, 'service_push', default=False)) - svc_content = format_service(service, show_private_url=True) - evaluate_call(lambda: resource_tree_service.delete_branch(resource_id=service.resource_id, db_session=request.db), - fallback=lambda: request.db.rollback(), httpError=HTTPForbidden, - msgOnFail="Delete service from resource tree failed.", content=svc_content) + service = ar.get_service_matchdict_checked(request) + service_push = str2bool(ar.get_multiformat_delete(request, 'service_push', default=False)) + svc_content = sf.format_service(service, show_private_url=True) + svc_res_id = service.resource_id + ax.evaluate_call(lambda: models.resource_tree_service.delete_branch(resource_id=svc_res_id, db_session=request.db), + fallback=lambda: request.db.rollback(), httpError=HTTPForbidden, + msgOnFail="Delete service from resource tree failed.", content=svc_content) def remove_service_magpie_and_phoenix(svc, svc_push, db_session): db_session.delete(svc) if svc_push and svc.type in SERVICES_PHOENIX_ALLOWED: sync_services_phoenix(db_session.query(models.Service)) - evaluate_call(lambda: remove_service_magpie_and_phoenix(service, service_push, request.db), - fallback=lambda: request.db.rollback(), httpError=HTTPForbidden, - msgOnFail=Service_DELETE_ForbiddenResponseSchema.description, content=svc_content) - return valid_http(httpSuccess=HTTPOk, detail=Service_DELETE_OkResponseSchema.description) + ax.evaluate_call(lambda: remove_service_magpie_and_phoenix(service, service_push, request.db), + fallback=lambda: request.db.rollback(), httpError=HTTPForbidden, + msgOnFail=Service_DELETE_ForbiddenResponseSchema.description, content=svc_content) + return ax.valid_http(httpSuccess=HTTPOk, detail=Service_DELETE_OkResponseSchema.description) @ServicePermissionsAPI.get(tags=[ServicesTag], response_schemas=ServicePermissions_GET_responses) @view_config(route_name=ServicePermissionsAPI.name, request_method='GET') def get_service_permissions(request): """List all applicable permissions for a service.""" - service = get_service_matchdict_checked(request) - svc_content = format_service(service, show_private_url=True) - svc_perms = evaluate_call(lambda: service_type_dict[service.type].permission_names, - fallback=request.db.rollback(), httpError=HTTPNotAcceptable, content=svc_content, - msgOnFail=ServicePermissions_GET_NotAcceptableResponseSchema.description) - return valid_http(httpSuccess=HTTPOk, detail=ServicePermissions_GET_OkResponseSchema.description, - content={u'permission_names': sorted(svc_perms)}) + service = ar.get_service_matchdict_checked(request) + svc_content = sf.format_service(service, show_private_url=True) + svc_perms = ax.evaluate_call(lambda: service_type_dict[service.type].permission_names, + fallback=request.db.rollback(), httpError=HTTPNotAcceptable, content=svc_content, + msgOnFail=ServicePermissions_GET_NotAcceptableResponseSchema.description) + return ax.valid_http(httpSuccess=HTTPOk, detail=ServicePermissions_GET_OkResponseSchema.description, + content={u'permission_names': sorted(svc_perms)}) @ServiceResourceAPI.delete(schema=ServiceResource_DELETE_RequestSchema(), tags=[ServicesTag], @@ -175,10 +162,10 @@ def delete_service_resource_view(request): @view_config(route_name=ServiceResourcesAPI.name, request_method='GET') def get_service_resources_view(request): """List all resources registered under a service.""" - service = get_service_matchdict_checked(request) - svc_res_json = format_service_resources(service, db_session=request.db, display_all=True, show_private_url=True) - return valid_http(httpSuccess=HTTPOk, detail=ServiceResources_GET_OkResponseSchema.description, - content={str(service.resource_name): svc_res_json}) + service = ar.get_service_matchdict_checked(request) + svc_res_json = sf.format_service_resources(service, db_session=request.db, display_all=True, show_private_url=True) + return ax.valid_http(httpSuccess=HTTPOk, detail=ServiceResources_GET_OkResponseSchema.description, + content={str(service.resource_name): svc_res_json}) @ServiceResourcesAPI.post(schema=ServiceResources_POST_RequestSchema, tags=[ServicesTag], @@ -186,11 +173,11 @@ def get_service_resources_view(request): @view_config(route_name=ServiceResourcesAPI.name, request_method='POST') def create_service_direct_resource(request): """Register a new resource directly under a service.""" - service = get_service_matchdict_checked(request) - resource_name = get_multiformat_post(request, 'resource_name') - resource_display_name = get_multiformat_post(request, 'resource_display_name', default=resource_name) - resource_type = get_multiformat_post(request, 'resource_type') - parent_id = get_multiformat_post(request, 'parent_id') # no check because None/empty is allowed + service = ar.get_service_matchdict_checked(request) + resource_name = ar.get_multiformat_post(request, 'resource_name') + resource_display_name = ar.get_multiformat_post(request, 'resource_display_name', default=resource_name) + resource_type = ar.get_multiformat_post(request, 'resource_type') + parent_id = ar.get_multiformat_post(request, 'parent_id') # no check because None/empty is allowed if not parent_id: parent_id = service.resource_id return create_resource(resource_name, resource_display_name, resource_type, @@ -201,11 +188,11 @@ def create_service_direct_resource(request): @view_config(route_name=ServiceResourceTypesAPI.name, request_method='GET') def get_service_type_resource_types(request): """List all resources under a specific service type.""" - service_type = get_value_matchdict_checked(request, 'service_type') - verify_param(service_type, paramCompare=service_type_dict.keys(), isIn=True, httpError=HTTPNotFound, - msgOnFail=ServiceResourceTypes_GET_NotFoundResponseSchema.description) - resource_types = evaluate_call(lambda: service_type_dict[service_type].resource_types, - httpError=HTTPForbidden, content={u'service_type': str(service_type)}, - msgOnFail=ServiceResourceTypes_GET_ForbiddenResponseSchema.description) - return valid_http(httpSuccess=HTTPOk, detail=ServiceResourceTypes_GET_OkResponseSchema.description, - content={u'resource_types': resource_types}) + service_type = ar.get_value_matchdict_checked(request, 'service_type') + ax.verify_param(service_type, paramCompare=service_type_dict.keys(), isIn=True, httpError=HTTPNotFound, + msgOnFail=ServiceResourceTypes_GET_NotFoundResponseSchema.description) + resource_types = ax.evaluate_call(lambda: service_type_dict[service_type].resource_types, + httpError=HTTPForbidden, content={u'service_type': str(service_type)}, + msgOnFail=ServiceResourceTypes_GET_ForbiddenResponseSchema.description) + return ax.valid_http(httpSuccess=HTTPOk, detail=ServiceResourceTypes_GET_OkResponseSchema.description, + content={u'resource_types': resource_types}) diff --git a/tests/interfaces.py b/tests/interfaces.py index a3237a0a5..c830680a9 100644 --- a/tests/interfaces.py +++ b/tests/interfaces.py @@ -221,8 +221,9 @@ def check_GetUserResourcesPermissions(cls, user_name, resource_id, query=None): @unittest.skipUnless(runner.MAGPIE_TEST_USERS, reason=runner.MAGPIE_TEST_DISABLED_MESSAGE('users')) def test_GetCurrentUserResourcesPermissions(self): utils.TestSetup.create_TestService(self) - utils.TestSetup.create_TestServiceResource(self) - self.check_GetUserResourcesPermissions(get_constant('MAGPIE_LOGGED_USER')) + json_body = utils.TestSetup.create_TestServiceResource(self) + res_id = json_body['resource']['resource_id'] + self.check_GetUserResourcesPermissions(get_constant('MAGPIE_LOGGED_USER'), res_id) @pytest.mark.users @unittest.skipUnless(runner.MAGPIE_TEST_USERS, reason=runner.MAGPIE_TEST_DISABLED_MESSAGE('users')) @@ -768,14 +769,14 @@ def test_PostService_ResponseFormat(self): json_body = utils.TestSetup.create_TestService(self) utils.check_val_is_in('service', json_body) utils.check_val_type(json_body['service'], dict) - utils.check_val_is_in('public_url', json_body['service']) utils.check_val_is_in('resource_id', json_body['service']) + utils.check_val_is_in('public_url', json_body['service']) utils.check_val_is_in('service_url', json_body['service']) utils.check_val_is_in('service_name', json_body['service']) utils.check_val_is_in('service_type', json_body['service']) utils.check_val_is_in('permission_names', json_body['service']) - utils.check_val_type(json_body['service']['public_url'], six.string_types) utils.check_val_type(json_body['service']['resource_id'], int) + utils.check_val_type(json_body['service']['public_url'], six.string_types) utils.check_val_type(json_body['service']['service_url'], six.string_types) utils.check_val_type(json_body['service']['service_name'], six.string_types) utils.check_val_type(json_body['service']['service_type'], six.string_types) From 28c7ac856df0277e67f04c49d5894f6eef338057 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Wed, 30 Jan 2019 14:23:48 -0500 Subject: [PATCH 13/76] more test fixes + readme status --- HISTORY.rst | 1 + README.rst | 20 ++++++++ magpie/magpie.ini | 4 +- tests/interfaces.py | 71 ++++++++++++++++---------- tests/test_magpie_adapter.py | 2 +- tests/test_register.py | 2 +- tests/utils.py | 98 ++++++++++++++++++++++++++++++------ 7 files changed, 152 insertions(+), 46 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index fc4a2f0f0..d7ac9504b 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -9,6 +9,7 @@ History * add permissions config to auto-generate user/group rules on startup * fix many invalid or erroneous swagger specifications * attempt db creation on first migration if not existing +* add continuous integration testing and deployment 0.8.x --------------------- diff --git a/README.rst b/README.rst index 227870640..0e6e09c66 100644 --- a/README.rst +++ b/README.rst @@ -16,6 +16,10 @@ Behind the scene, it uses `Ziggurat-Foundations`_ and `Authomatic`_. * - dependencies - | |py_ver| |requires| + * - build status + - | |travis_latest| |travis_tag| + * - docker status + - | |docker_build_mode| |docker_build_status| * - releases - | |version| |commits-since| @@ -35,6 +39,22 @@ Behind the scene, it uses `Ziggurat-Foundations`_ and `Authomatic`_. :alt: Requirements Status :target: https://requires.io/github/Ouranosinc/Magpie/requirements/?branch=master +.. |travis_latest| image:: https://img.shields.io/travis/Ouranosinc/Magpie/master.svg?label=master + :alt: Travis-CI Build Status (master branch) + :target: https://travis-ci.com/Ouranosinc/Magpie + +.. |travis_tag| image:: https://img.shields.io/travis/Ouranosinc/Magpie/0.9.0.svg?label=0.9.0 + :alt: Travis-CI Build Status (latest tag) + :target: https://github.com/Ouranosinc/Magpie/tree/0.9.0 + +.. |docker_build_mode| image:: https://img.shields.io/docker/automated/pavics/magpie.svg?label=build + :alt: Docker Build Status (latest tag) + :target: https://hub.docker.com/r/pavics/magpie/builds + +.. |docker_build_status| image:: https://img.shields.io/docker/build/pavics/magpie.svg?label=status + :alt: Docker Build Status (latest tag) + :target: https://hub.docker.com/r/pavics/magpie/builds + .. end-badges diff --git a/magpie/magpie.ini b/magpie/magpie.ini index 4498f2639..5e86fc031 100644 --- a/magpie/magpie.ini +++ b/magpie/magpie.ini @@ -22,7 +22,7 @@ pyramid.includes = pyramid_tm ziggurat_foundations.ext.pyramid.sign_in ziggurat_ # magpie magpie.port = 2001 -magpie.url = http://localhost:2001/magpie +magpie.url = http://localhost:2001 magpie.max_restart = 5 # This secret should be the same in Twitcher ! magpie.secret = seekrit @@ -53,7 +53,7 @@ prefix = /magpie use = egg:gunicorn#main host = localhost port=2001 -timeout=5000 +timeout=10 workers=3 threads=4 diff --git a/tests/interfaces.py b/tests/interfaces.py index c830680a9..b1468fa82 100644 --- a/tests/interfaces.py +++ b/tests/interfaces.py @@ -105,13 +105,15 @@ def tearDownClass(cls): pyramid.testing.tearDown() def tearDown(self): + self.check_requirements() # re-login as required in case test logged out the user with permissions utils.TestSetup.delete_TestServiceResource(self) utils.TestSetup.delete_TestService(self) utils.TestSetup.delete_TestUser(self) @classmethod def check_requirements(cls): - headers, cookies = utils.check_or_try_login_user(cls.url, cls.usr, cls.pwd, version=cls.version) + headers, cookies = utils.check_or_try_login_user(cls.url, cls.usr, cls.pwd, + use_ui_form_submit=True, version=cls.version) assert headers and cookies, cls.require assert cls.headers and cls.cookies, cls.require @@ -537,27 +539,30 @@ def test_PutUsers_username(self): body = utils.check_response_basic_info(resp, 200, expected_method='GET') utils.check_val_equal(body['user']['user_name'], new_name) + # validate removed previous user name + route = '/users/{usr}'.format(usr=self.test_user_name) + resp = utils.test_request(self.url, 'GET', route, headers=self.json_headers, cookies=self.cookies, + expect_errors=True) + utils.check_response_basic_info(resp, 404, expected_method='GET') + # validate effective new user name - data = {'user_name': new_name, 'password': self.test_user_name} - resp = utils.test_request(self.url, 'POST', '/signin', headers=self.json_headers, data=data) - utils.check_response_basic_info(resp, 200, expected_method='POST') - resp = utils.test_request(self.url, 'GET', '/session', headers=self.json_headers, cookies=resp.cookies) + utils.check_or_try_logout_user(self.url) + headers, cookies = utils.check_or_try_login_user(self.url, username=new_name, password=self.test_user_name, + use_ui_form_submit=True, version=self.version) + resp = utils.test_request(self.url, 'GET', '/session', headers=headers, cookies=cookies) body = utils.check_response_basic_info(resp, 200, expected_method='GET') utils.check_val_equal(body['authenticated'], True) utils.check_val_equal(body['user']['user_name'], new_name) # validate ineffective previous user name - route = '/users/{usr}'.format(usr=self.test_user_name) - resp = utils.test_request(self.url, 'GET', route, headers=self.json_headers, cookies=self.cookies, - expect_errors=True) - utils.check_response_basic_info(resp, 404, expected_method='GET') - data = {'user_name': self.test_user_name, 'password': self.test_user_name} - resp = utils.test_request(self.url, 'POST', '/signin', headers=self.json_headers, data=data, - expect_errors=True) - if LooseVersion(self.version) >= LooseVersion('0.7.8'): - utils.check_response_basic_info(resp, 401, expected_method='POST') - else: - utils.check_response_basic_info(resp, 400, expected_method='POST') + utils.check_or_try_logout_user(self.url) + headers, cookies = utils.check_or_try_login_user(self.url, + username=self.test_user_name, password=self.test_user_name, + use_ui_form_submit=True, version=self.version) + utils.check_val_equal(cookies, {}, msg="Cookies should be empty from login failure.") + resp = utils.test_request(self.url, 'GET', '/session', headers=headers, cookies=cookies) + body = utils.check_response_basic_info(resp, 200, expected_method='GET') + utils.check_val_equal(body['authenticated'], False) @pytest.mark.users @unittest.skipUnless(runner.MAGPIE_TEST_USERS, reason=runner.MAGPIE_TEST_DISABLED_MESSAGE('users')) @@ -583,23 +588,34 @@ def test_PutUsers_password(self): route = '/users/{usr}'.format(usr=self.test_user_name) resp = utils.test_request(self.url, 'PUT', route, headers=self.json_headers, cookies=self.cookies, data=data) utils.check_response_basic_info(resp, 200, expected_method='PUT') + utils.check_or_try_logout_user(self.url) # validate that the new password is effective - data = {'user_name': self.test_user_name, 'password': new_password} - resp = utils.test_request(self.url, 'POST', '/signin', headers=self.json_headers, data=data) - utils.check_response_basic_info(resp, 200, expected_method='POST') - resp = utils.test_request(self.url, 'GET', '/session', headers=self.json_headers, cookies=resp.cookies) + #data = {'user_name': self.test_user_name, 'password': new_password} + #resp = utils.test_request(self.url, 'POST', '/signin', data=data, + # headers=self.json_headers, allow_redirects=True) + #utils.check_response_basic_info(resp, 200, expected_method='POST') + + headers, cookies = utils.check_or_try_login_user(self.url, username=self.test_user_name, password=new_password, + use_ui_form_submit=True, version=self.version) + resp = utils.test_request(self.url, 'GET', '/session', headers=headers, cookies=cookies) body = utils.check_response_basic_info(resp, 200, expected_method='GET') utils.check_val_equal(body['authenticated'], True) utils.check_val_equal(body['user']['user_name'], self.test_user_name) + utils.check_or_try_logout_user(self.url) # validate that previous password is ineffective - data = {'user_name': self.test_user_name, 'password': old_password} - resp = utils.test_request(self.url, 'POST', '/signin', headers=self.json_headers, data=data, expect_errors=True) - if LooseVersion(self.version) >= LooseVersion('0.7.8'): - utils.check_response_basic_info(resp, 401, expected_method='POST') - else: - utils.check_response_basic_info(resp, 400, expected_method='POST') + #data = {'user_name': self.test_user_name, 'password': old_password} + #resp = utils.test_request(self.url, 'POST', '/signin', headers=self.json_headers, data=data, expect_errors=True) + #if LooseVersion(self.version) >= LooseVersion('0.7.8'): + # utils.check_response_basic_info(resp, 401, expected_method='POST') + #else: + # utils.check_response_basic_info(resp, 400, expected_method='POST') + headers, cookies = utils.check_or_try_login_user(self.url, username=self.test_user_name, password=old_password, + use_ui_form_submit=True, version=self.version) + resp = utils.test_request(self.url, 'GET', '/session', headers=headers, cookies=cookies) + body = utils.check_response_basic_info(resp, 200, expected_method='GET') + utils.check_val_equal(body['authenticated'], False) @pytest.mark.users @unittest.skipUnless(runner.MAGPIE_TEST_USERS, reason=runner.MAGPIE_TEST_DISABLED_MESSAGE('users')) @@ -1035,7 +1051,8 @@ def test_PostResources_MissingParentID(self): resp = utils.test_request(self.url, 'POST', '/resources', headers=self.json_headers, cookies=self.cookies, data=data, expect_errors=True) json_body = utils.check_response_basic_info(resp, 422, expected_method='POST') - utils.check_error_param_structure(json_body, param_name='parent_id', param_value=repr(None), version=self.version) + utils.check_error_param_structure(json_body, version=self.version, + param_name='parent_id', param_value=repr(None)) @pytest.mark.resources @unittest.skipUnless(runner.MAGPIE_TEST_RESOURCES, reason=runner.MAGPIE_TEST_DISABLED_MESSAGE('resources')) diff --git a/tests/test_magpie_adapter.py b/tests/test_magpie_adapter.py index 1b9975e4d..d2956b0f4 100644 --- a/tests/test_magpie_adapter.py +++ b/tests/test_magpie_adapter.py @@ -2,8 +2,8 @@ import unittest +@pytest.skip("not implemented") @unittest.skip("not implemented") class TestAdapter(unittest.TestCase): - @pytest.skip("not implemented") def test_adapter(self): pass diff --git a/tests/test_register.py b/tests/test_register.py index 8d500a4c5..a8d7b07b4 100644 --- a/tests/test_register.py +++ b/tests/test_register.py @@ -2,8 +2,8 @@ import unittest +@pytest.skip("not implemented") @unittest.skip("not implemented") class TestRegister(unittest.TestCase): - @pytest.skip("not implemented") def test_register(self): pass diff --git a/tests/utils.py b/tests/utils.py index a83e54e6c..112165c8b 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -3,13 +3,21 @@ import six from six.moves.urllib.parse import urlparse from distutils.version import LooseVersion +from pyramid.response import Response from pyramid.testing import setUp as PyramidSetUp from webtest import TestApp from webtest.response import TestResponse +from webob.headers import ResponseHeaders from magpie import __meta__, db, services, magpiectl from magpie.constants import get_constant +from typing import AnyStr, Dict, List, Optional, Tuple, Union OptionalStringType = six.string_types + tuple([type(None)]) +HeadersType = Union[Dict[AnyStr, AnyStr], List[Tuple[AnyStr, AnyStr]]] +CookiesType = Union[Dict[AnyStr, AnyStr], List[Tuple[AnyStr, AnyStr]]] +OptionalHeaderCookiesType = Union[Tuple[None, None], Tuple[HeadersType, CookiesType]] +TestAppOrUrlType = Union[AnyStr, TestApp] +ResponseType = Union[TestResponse, Response] def config_setup_from_ini(config_ini_file_path): @@ -22,6 +30,7 @@ def get_test_magpie_app(): # parse settings from ini file to pass them to the application config = config_setup_from_ini(get_constant('MAGPIE_INI_FILE_PATH')) config.include('ziggurat_foundations.ext.pyramid.sign_in') + config.include('ziggurat_foundations.ext.pyramid.get_user') config.registry.settings['magpie.db_migration_disabled'] = True # scan dependencies config.include('magpie') @@ -42,6 +51,22 @@ def get_headers(app_or_url, header_dict): return header_dict +def get_header(header_name, header_container): + # type: (AnyStr, Optional[Union[HeadersType, ResponseHeaders]]) -> Union[AnyStr, None] + if header_container is None: + return None + headers = header_container + if isinstance(headers, ResponseHeaders): + headers = dict(headers) + if isinstance(headers, dict): + headers = header_container.items() + header_name = header_name.lower().replace('-', '_') + for h, v in headers: + if h.lower().replace('-', '_') == header_name: + return v + return None + + def get_response_content_types_list(response): return [ct.strip() for ct in response.headers['Content-Type'].split(';')] @@ -88,9 +113,11 @@ def test_request(app_or_url, method, path, timeout=5, allow_redirects=True, **kw if cookies and not app_or_url.cookies: app_or_url.cookies.update(cookies) - kwargs['params'] = json_body + # obtain Content-Type header if specified to ensure it is properly applied + kwargs['content_type'] = get_header('Content-Type', kwargs.get('headers')) # convert JSON body as required + kwargs['params'] = json_body if json_body is not None: kwargs.update({'params': json.dumps(json_body, cls=json.JSONEncoder)}) if status and status >= 300: @@ -115,33 +142,47 @@ def test_request(app_or_url, method, path, timeout=5, allow_redirects=True, **kw return requests.request(method, url, timeout=timeout, allow_redirects=allow_redirects, **kwargs) -def check_or_try_login_user(app_or_url, username=None, password=None, provider='ziggurat', headers=None, - use_ui_form_submit=False, version=__meta__.__version__): +def get_session_user(app_or_url, headers=None): + # type: (TestAppOrUrlType, Optional[HeadersType]) -> ResponseType + if not headers: + headers = get_headers(app_or_url, {'Accept': 'application/json', 'Content-Type': 'application/json'}) + if isinstance(app_or_url, TestApp): + resp = app_or_url.get('/session', headers=headers) + else: + resp = requests.get('{}/session'.format(app_or_url), headers=headers) + if resp.status_code != 200: + raise Exception('cannot retrieve logged in user information') + return resp + + +def check_or_try_login_user(app_or_url, # type: TestAppOrUrlType + username=None, # type: Optional[AnyStr] + password=None, # type: Optional[AnyStr] + provider='ziggurat', # type: Optional[AnyStr] + headers=None, # type: Optional[Dict[AnyStr, AnyStr]] + use_ui_form_submit=False, # type: Optional[bool] + version=__meta__.__version__, # type: Optional[AnyStr] + ): # type: (...) -> OptionalHeaderCookiesType """ Verifies that the required user is already logged in (or none is if username=None), or tries to login him otherwise. + Validates that the logged user (if any), matched the one specified with `username`. - :param app_or_url: `webtest.TestApp` instance of the test application or remote server URL to call with `requests` + :param app_or_url: instance of the test application or remote server URL to call :param username: name of the user to login or None otherwise :param password: password to use for login if the user was not already logged in :param provider: provider string to use for login (default: ziggurat, ie: magpie's local signin) :param headers: headers to include in the test request - :param use_ui_form_submit: use Magpie UI login 'form' to obtain cookies (required for local WebTest.App login) + :param use_ui_form_submit: use Magpie UI login 'form' to obtain cookies + (required for local `WebTest.App` login, ignored by requests using URL) :param version: server or local app version to evaluate responses with backward compatibility :return: headers and cookies of the user session or (None, None) - :raise: Exception on any login/logout failure as required by the caller's specifications (username/password) + :raise: Exception on any login failure as required by the caller's specifications (username/password) """ headers = headers or {} - - if isinstance(app_or_url, TestApp): - resp = app_or_url.get('/session', headers=headers) - else: - resp = requests.get('{}/session'.format(app_or_url), headers=headers) + resp = get_session_user(app_or_url, headers) body = get_json_body(resp) - if resp.status_code != 200: - raise Exception('cannot retrieve logged in user information') - resp_cookies = None auth = body.get('authenticated', False) if auth is False and username is None: @@ -185,6 +226,33 @@ def check_or_try_login_user(app_or_url, username=None, password=None, provider=' return resp.headers, resp_cookies +def check_or_try_logout_user(app_or_url): + # type: (TestAppOrUrlType) -> None + """ + Verifies that any user is logged out, or tries to logout him otherwise. + + :param app_or_url: instance of the test application or remote server URL to call + :raise: Exception on any logout failure or incapability to validate logout + """ + + def _is_logged_out(): + resp = get_session_user(app_or_url) + body = get_json_body(resp) + auth = body.get('authenticated', False) + return not auth + + if _is_logged_out(): + return + resp_logout = test_request(app_or_url, 'GET', '/ui/logout', allow_redirects=True) + if isinstance(app_or_url, TestApp): + app_or_url.reset() # clear app cookies + if resp_logout.status_code >= 400: + raise Exception("cannot validate logout") + if _is_logged_out(): + return + raise Exception("logout did not succeed") + + def format_test_val_ref(val, ref, pre='Fail', msg=None): _msg = '({0}) Test value: `{1}`, Reference value: `{2}`'.format(pre, val, ref) if isinstance(msg, six.string_types): @@ -382,7 +450,7 @@ def check_UpStatus(test_class, method, path): Verifies that the Magpie UI page at very least returned an Ok response with the displayed title. Validates that at the bare minimum, no underlying internal error occurred from the API or UI calls. """ - resp = test_request(test_class.url, method, path, cookies=test_class.cookies, timeout=10) + resp = test_request(test_class.url, method, path, cookies=test_class.cookies, timeout=20) check_val_equal(resp.status_code, 200) check_val_is_in('Content-Type', dict(resp.headers)) check_val_is_in('text/html', get_response_content_types_list(resp)) From a6788346b6e5def7bbe2431ad63189a93ee25e81 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Wed, 30 Jan 2019 15:49:48 -0500 Subject: [PATCH 14/76] logging configurable & fixed tests --- HISTORY.rst | 1 + env/magpie.env.example | 1 + magpie/common.py | 10 ++++++++++ magpie/constants.py | 12 +++++++++++- magpie/db.py | 11 +---------- magpie/magpie.ini | 2 +- magpie/magpiectl.py | 9 +++++++++ tests/utils.py | 5 +++-- 8 files changed, 37 insertions(+), 14 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index d7ac9504b..4bf1e3a21 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -10,6 +10,7 @@ History * fix many invalid or erroneous swagger specifications * attempt db creation on first migration if not existing * add continuous integration testing and deployment +* reduce excessive sqlalchemy logging using `MAGPIE_LOG_LEVEL >= INFO` 0.8.x --------------------- diff --git a/env/magpie.env.example b/env/magpie.env.example index e393c0e2a..6e29eb6ec 100644 --- a/env/magpie.env.example +++ b/env/magpie.env.example @@ -9,6 +9,7 @@ MAGPIE_ADMIN_PASSWORD=qwerty MAGPIE_ANONYMOUS_USER=anonymous MAGPIE_USERS_GROUP=users MAGPIE_CRON_LOG=~/magpie_cron.log +MAGPIE_LOG_LEVEL=INFO PHOENIX_USER=phoenix PHOENIX_PASSWORD=qwerty PHOENIX_PORT=8443 diff --git a/magpie/common.py b/magpie/common.py index f05e89baa..4c42a1753 100644 --- a/magpie/common.py +++ b/magpie/common.py @@ -2,6 +2,8 @@ # -*- coding: utf-8 -*- from __future__ import unicode_literals from distutils.dir_util import mkpath +# noinspection PyCompatibility +import configparser import logging import types import six @@ -49,3 +51,11 @@ def make_dirs(path): for subdir in mkpath(dir_path): if not os.path.isdir(subdir): os.mkdir(subdir) + + +def get_settings_from_config_ini(config_ini_path, ini_main_section_name='app:magpie_app'): + parser = configparser.ConfigParser() + parser.optionxform = lambda option: option # preserve case of config (ziggurat requires it for 'User' model) + parser.read([config_ini_path]) + settings = dict(parser.items(ini_main_section_name)) + return settings diff --git a/magpie/constants.py b/magpie/constants.py index 1fcbb37db..cf3ed503d 100644 --- a/magpie/constants.py +++ b/magpie/constants.py @@ -4,7 +4,7 @@ import shutil import dotenv import logging -from magpie.common import str2bool, raise_log, print_log +from magpie.common import str2bool, raise_log, print_log, get_settings_from_config_ini logger = logging.getLogger(__name__) # =========================== @@ -41,6 +41,15 @@ logger.warn("Failed to open environment files [MAGPIE_ENV_DIR={}].".format(MAGPIE_ENV_DIR)) pass +# get default configurations from ini file +_default_log_lvl = 'INFO' +# noinspection PyBroadException +try: + _settings = get_settings_from_config_ini(MAGPIE_INI_FILE_PATH, ini_main_section_name='logger_magpie') + _default_log_lvl = _settings.get('level', _default_log_lvl) +except Exception: + pass + # =========================== # variables from magpie.env # =========================== @@ -58,6 +67,7 @@ MAGPIE_EDITOR_GROUP = os.getenv('MAGPIE_EDITOR_GROUP', 'editors') MAGPIE_USERS_GROUP = os.getenv('MAGPIE_USERS_GROUP', 'users') MAGPIE_CRON_LOG = os.getenv('MAGPIE_CRON_LOG', '~/magpie-cron.log') +MAGPIE_LOG_LEVEL = os.getenv('MAGPIE_LOG_LEVEL', _default_log_lvl) PHOENIX_USER = os.getenv('PHOENIX_USER', 'phoenix') PHOENIX_PASSWORD = os.getenv('PHOENIX_PASSWORD', 'qwerty') PHOENIX_PORT = int(os.getenv('PHOENIX_PORT', 8443)) diff --git a/magpie/db.py b/magpie/db.py index 89ad0995c..1042f2031 100644 --- a/magpie/db.py +++ b/magpie/db.py @@ -1,10 +1,9 @@ #!/usr/bin/python # -*- coding: utf-8 -*- from magpie import constants +from magpie.common import get_settings_from_config_ini from magpie.definitions.alembic_definitions import * from magpie.definitions.sqlalchemy_definitions import * -# noinspection PyCompatibility -import configparser import transaction import inspect import zope.sqlalchemy @@ -80,14 +79,6 @@ def get_db_session_from_config_ini(config_ini_path, ini_main_section_name='app:m return get_db_session_from_settings(settings) -def get_settings_from_config_ini(config_ini_path, ini_main_section_name='app:magpie_app'): - parser = configparser.ConfigParser() - parser.optionxform = lambda option: option # preserve case of config (ziggurat requires it for 'User' model) - parser.read([config_ini_path]) - settings = dict(parser.items(ini_main_section_name)) - return settings - - def run_database_migration(): logger.info("Using file '{}' for migration.".format(constants.MAGPIE_ALEMBIC_INI_FILE_PATH)) alembic_args = ['-c', constants.MAGPIE_ALEMBIC_INI_FILE_PATH, 'upgrade', 'heads'] diff --git a/magpie/magpie.ini b/magpie/magpie.ini index 5e86fc031..d65575bf7 100644 --- a/magpie/magpie.ini +++ b/magpie/magpie.ini @@ -83,7 +83,7 @@ qualname = magpie [handler_console] class = StreamHandler args = (sys.stderr,) -level = DEBUG +level = INFO formatter = generic [formatter_generic] diff --git a/magpie/magpiectl.py b/magpie/magpiectl.py index 05210531c..c3a5ba763 100644 --- a/magpie/magpiectl.py +++ b/magpie/magpiectl.py @@ -34,6 +34,15 @@ def main(global_config=None, **settings): settings['magpie.root'] = constants.MAGPIE_ROOT settings['magpie.module'] = constants.MAGPIE_MODULE_DIR + # suppress sqlalchemy logging if not in debug for magpie + log_lvl = get_constant('MAGPIE_LOG_LEVEL', settings, 'magpie.log_level', default_value=logging.INFO, + raise_missing=False, print_missing=False, raise_not_set=False) + log_lvl = logging.getLevelName(log_lvl) if isinstance(log_lvl, int) else log_lvl + if log_lvl.upper() != 'DEBUG': + sa_log = logging.getLogger('sqlalchemy.engine.base.Engine') + sa_log.setLevel(logging.WARN) # WARN to avoid INFO logs + LOGGER.setLevel(log_lvl) + # migrate db as required and check if database is ready if not settings.get('magpie.db_migration_disabled', False): print_log('Running database migration (as required) ...') diff --git a/tests/utils.py b/tests/utils.py index 112165c8b..7ad64ada3 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -8,7 +8,8 @@ from webtest import TestApp from webtest.response import TestResponse from webob.headers import ResponseHeaders -from magpie import __meta__, db, services, magpiectl +from magpie import __meta__, services, magpiectl +from magpie.common import get_settings_from_config_ini from magpie.constants import get_constant from typing import AnyStr, Dict, List, Optional, Tuple, Union @@ -21,7 +22,7 @@ def config_setup_from_ini(config_ini_file_path): - settings = db.get_settings_from_config_ini(config_ini_file_path) + settings = get_settings_from_config_ini(config_ini_file_path) config = PyramidSetUp(settings=settings) return config From cfd0b709ec20bb4ac2afe476e7cabcfffbf8b1ce Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Wed, 30 Jan 2019 16:12:39 -0500 Subject: [PATCH 15/76] update travis --- .travis.yml | 5 ++++- Makefile | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index c799747b8..ebd3cad4c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,6 +5,8 @@ python: - "3.6" sudo: false cache: pip +# includes PR when base branch = master +if: branch = master OR tag IS present env: matrix: - env TARGET=test-local @@ -14,8 +16,9 @@ before_install: - python --version - uname -a - lsb_release -a + - make sysinstall install: - - make install + - make install install-dev - make version script: - make $TARGET diff --git a/Makefile b/Makefile index 1f02d612d..fe2a7d78b 100644 --- a/Makefile +++ b/Makefile @@ -86,7 +86,7 @@ clean-build: clean-docs: @echo "Cleaning doc artifacts..." - "$(MAKE)" -C "$(CUR_DIR)/docs" clean + "$(MAKE)" -C "$(CUR_DIR)/docs" clean || true clean-pyc: @echo "Cleaning Python artifacts..." From 98feba83c2519e3f0cc0076ae1a044068476dde0 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Wed, 30 Jan 2019 16:30:05 -0500 Subject: [PATCH 16/76] remove missing file ref conda.env --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index fe2a7d78b..bf049f977 100644 --- a/Makefile +++ b/Makefile @@ -265,4 +265,4 @@ conda_config: conda-base .PHONY: conda-env conda-env: conda-base @test -d "$(CONDA_ENV_PATH)" || (echo "Creating conda environment at '$(CONDA_ENV_PATH)'..." && \ - "$(CONDA_HOME)/bin/conda" env create --file "$(CUR_DIR)/conda-env.yml" -n "$(APP_NAME)") + "$(CONDA_HOME)/bin/conda" env create -n "$(APP_NAME)") From 193a59fd14d6612ddc11b7840dd227f4e5c9481c Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Wed, 30 Jan 2019 17:03:02 -0500 Subject: [PATCH 17/76] travis build update --- .travis.yml | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index ebd3cad4c..36688823f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,18 +4,26 @@ python: - "3.5" - "3.6" sudo: false -cache: pip +cache: + - pip + - directories: + - $HOME/conda + - $HOME/downloads # includes PR when base branch = master if: branch = master OR tag IS present env: + global: + - CONDA_HOME=$HOME/conda + - DOWNLOAD_CACHE=$HOME/downloads matrix: - - env TARGET=test-local - - env TARGET=test-remote - - env TARGET=coverage + - APP_NAME=magpie-$TRAVIS_PYTHON_VERSION TARGET=test-local + - APP_NAME=magpie-$TRAVIS_PYTHON_VERSION TARGET=test-remote + - APP_NAME=magpie-$TRAVIS_PYTHON_VERSION TARGET=coverage before_install: - python --version - uname -a - lsb_release -a + - hash -r - make sysinstall install: - make install install-dev @@ -23,9 +31,7 @@ install: script: - make $TARGET notifications: - email: - on_success: never - on_failure: always + email: false jobs: include: - stage: deploy From 2732f44b0b209f65b93b68a4690af005c84862d3 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Wed, 30 Jan 2019 17:05:49 -0500 Subject: [PATCH 18/76] print env --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 36688823f..1592136ff 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,6 +24,7 @@ before_install: - uname -a - lsb_release -a - hash -r + - env - make sysinstall install: - make install install-dev From 04af476a213043ddef498da8f53d1e77cc705297 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Wed, 30 Jan 2019 17:11:03 -0500 Subject: [PATCH 19/76] alternate conda env setup --- .travis.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1592136ff..056c50aff 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,13 +16,14 @@ env: - CONDA_HOME=$HOME/conda - DOWNLOAD_CACHE=$HOME/downloads matrix: - - APP_NAME=magpie-$TRAVIS_PYTHON_VERSION TARGET=test-local - - APP_NAME=magpie-$TRAVIS_PYTHON_VERSION TARGET=test-remote - - APP_NAME=magpie-$TRAVIS_PYTHON_VERSION TARGET=coverage + - TARGET=test-local + - TARGET=test-remote + - TARGET=coverage before_install: - python --version - uname -a - lsb_release -a + - export APP_NAME=magpie-${TRAVIS_PYTHON_VERSION} - hash -r - env - make sysinstall From f0fa44118325012fa44b5b63b1191141892f079a Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Wed, 30 Jan 2019 17:26:10 -0500 Subject: [PATCH 20/76] debug travis dir contents --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index 056c50aff..ddc72c529 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,8 +24,11 @@ before_install: - uname -a - lsb_release -a - export APP_NAME=magpie-${TRAVIS_PYTHON_VERSION} + - export PATH="$(CONDA_HOME)/bin":$PATH - hash -r - env + - ls -al "$(CONDA_HOME)" + - ls -al "$(DOWNLOAD_CACHE)" - make sysinstall install: - make install install-dev From 0af8e778e08f42851c268afca62ec893907dfbf2 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Wed, 30 Jan 2019 18:10:42 -0500 Subject: [PATCH 21/76] fix vars --- .travis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index ddc72c529..94141ee04 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,11 +24,11 @@ before_install: - uname -a - lsb_release -a - export APP_NAME=magpie-${TRAVIS_PYTHON_VERSION} - - export PATH="$(CONDA_HOME)/bin":$PATH + - export PATH="${CONDA_HOME}/bin":$PATH - hash -r - env - - ls -al "$(CONDA_HOME)" - - ls -al "$(DOWNLOAD_CACHE)" + - ls -al "${CONDA_HOME}" + - ls -al "${DOWNLOAD_CACHE}" - make sysinstall install: - make install install-dev From d06b10bb870ceb4b1a7c5afee0e358346c137bc3 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Wed, 30 Jan 2019 18:32:46 -0500 Subject: [PATCH 22/76] force conda-base install --- .travis.yml | 3 +-- Makefile | 4 +++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 94141ee04..c3740ffbf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,8 +27,7 @@ before_install: - export PATH="${CONDA_HOME}/bin":$PATH - hash -r - env - - ls -al "${CONDA_HOME}" - - ls -al "${DOWNLOAD_CACHE}" + - make conda-base - make sysinstall install: - make install install-dev diff --git a/Makefile b/Makefile index bf049f977..c97f52a0e 100644 --- a/Makefile +++ b/Makefile @@ -245,8 +245,10 @@ docker-push: docker-build .PHONY: conda-base conda-base: - @test -d "$(CONDA_HOME)" || test -d "$(DOWNLOAD_CACHE)" || mkdir "$(DOWNLOAD_CACHE)" + @test -d "$(CONDA_HOME)" || test -d "$(DOWNLOAD_CACHE)" || \ + echo "Creating download directory: $(DOWNLOAD_CACHE)" && mkdir "$(DOWNLOAD_CACHE)" @test -d "$(CONDA_HOME)" || test -f "$(DOWNLOAD_CACHE)/$(FN)" || \ + echo "Fetching conda distribution from: $(CONDA_URL)/$(FN)" && \ curl "$(CONDA_URL)/$(FN)" --insecure --output "$(DOWNLOAD_CACHE)/$(FN)" @test -d "$(CONDA_HOME)" || (bash "$(DOWNLOAD_CACHE)/$(FN)" -b -p "$(CONDA_HOME)" && \ echo "Make sure to add '$(CONDA_HOME)/bin' to your PATH variable in '~/.bashrc'.") From 9c9bf513d3e1a8daa0e93bccc7761478ab77b58b Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Wed, 30 Jan 2019 18:44:41 -0500 Subject: [PATCH 23/76] allow overrides makefile vars --- .travis.yml | 2 +- Makefile | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index c3740ffbf..f2dd66587 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,7 +23,7 @@ before_install: - python --version - uname -a - lsb_release -a - - export APP_NAME=magpie-${TRAVIS_PYTHON_VERSION} + - export CONDA_ENV=magpie-${TRAVIS_PYTHON_VERSION} - export PATH="${CONDA_HOME}/bin":$PATH - hash -r - env diff --git a/Makefile b/Makefile index c97f52a0e..24fd43c3f 100644 --- a/Makefile +++ b/Makefile @@ -21,7 +21,7 @@ CONDA_ENV ?= $(APP_NAME) CONDA_HOME ?= $(HOME)/conda CONDA_ENVS_DIR ?= $(CONDA_HOME)/envs CONDA_ENV_PATH := $(CONDA_ENVS_DIR)/$(CONDA_ENV) -DOWNLOAD_CACHE := $(APP_ROOT)/downloads +DOWNLOAD_CACHE ?= $(APP_ROOT)/downloads # choose conda installer depending on your OS CONDA_URL = https://repo.continuum.io/miniconda @@ -246,7 +246,7 @@ docker-push: docker-build .PHONY: conda-base conda-base: @test -d "$(CONDA_HOME)" || test -d "$(DOWNLOAD_CACHE)" || \ - echo "Creating download directory: $(DOWNLOAD_CACHE)" && mkdir "$(DOWNLOAD_CACHE)" + echo "Creating download directory: $(DOWNLOAD_CACHE)" && mkdir -p "$(DOWNLOAD_CACHE)" @test -d "$(CONDA_HOME)" || test -f "$(DOWNLOAD_CACHE)/$(FN)" || \ echo "Fetching conda distribution from: $(CONDA_URL)/$(FN)" && \ curl "$(CONDA_URL)/$(FN)" --insecure --output "$(DOWNLOAD_CACHE)/$(FN)" @@ -267,4 +267,4 @@ conda_config: conda-base .PHONY: conda-env conda-env: conda-base @test -d "$(CONDA_ENV_PATH)" || (echo "Creating conda environment at '$(CONDA_ENV_PATH)'..." && \ - "$(CONDA_HOME)/bin/conda" env create -n "$(APP_NAME)") + "$(CONDA_HOME)/bin/conda" env create -n "$(CONDA_ENV)") From 1055957bb8ddd56d0ed315ddcf5ee1ed6f5db3ed Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Wed, 30 Jan 2019 18:59:30 -0500 Subject: [PATCH 24/76] update makefile download conda --- Makefile | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 24fd43c3f..d2bd7ad19 100644 --- a/Makefile +++ b/Makefile @@ -246,12 +246,13 @@ docker-push: docker-build .PHONY: conda-base conda-base: @test -d "$(CONDA_HOME)" || test -d "$(DOWNLOAD_CACHE)" || \ - echo "Creating download directory: $(DOWNLOAD_CACHE)" && mkdir -p "$(DOWNLOAD_CACHE)" + (echo "Creating download directory: $(DOWNLOAD_CACHE)" && mkdir -p "$(DOWNLOAD_CACHE)") @test -d "$(CONDA_HOME)" || test -f "$(DOWNLOAD_CACHE)/$(FN)" || \ - echo "Fetching conda distribution from: $(CONDA_URL)/$(FN)" && \ - curl "$(CONDA_URL)/$(FN)" --insecure --output "$(DOWNLOAD_CACHE)/$(FN)" - @test -d "$(CONDA_HOME)" || (bash "$(DOWNLOAD_CACHE)/$(FN)" -b -p "$(CONDA_HOME)" && \ - echo "Make sure to add '$(CONDA_HOME)/bin' to your PATH variable in '~/.bashrc'.") + (echo "Fetching conda distribution from: $(CONDA_URL)/$(FN)" && \ + curl "$(CONDA_URL)/$(FN)" --insecure --output "$(DOWNLOAD_CACHE)/$(FN)") + @test -f "$(CONDA_HOME)/bin/conda" || \ + (bash "$(DOWNLOAD_CACHE)/$(FN)" -b -p "$(CONDA_HOME)" && \ + echo "Make sure to add '$(CONDA_HOME)/bin' to your PATH variable in '~/.bashrc'.") .PHONY: conda-cfg conda_config: conda-base @@ -266,5 +267,6 @@ conda_config: conda-base .PHONY: conda-env conda-env: conda-base - @test -d "$(CONDA_ENV_PATH)" || (echo "Creating conda environment at '$(CONDA_ENV_PATH)'..." && \ - "$(CONDA_HOME)/bin/conda" env create -n "$(CONDA_ENV)") + @test -d "$(CONDA_ENV_PATH)" || \ + (echo "Creating conda environment at '$(CONDA_ENV_PATH)'..." && \ + "$(CONDA_HOME)/bin/conda" env create -n "$(CONDA_ENV)") From 3cdf9a3fa695d4e445d5905313f6ad272219ea12 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Wed, 30 Jan 2019 19:10:33 -0500 Subject: [PATCH 25/76] test conda bin for download --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index d2bd7ad19..f791e2efd 100644 --- a/Makefile +++ b/Makefile @@ -247,7 +247,7 @@ docker-push: docker-build conda-base: @test -d "$(CONDA_HOME)" || test -d "$(DOWNLOAD_CACHE)" || \ (echo "Creating download directory: $(DOWNLOAD_CACHE)" && mkdir -p "$(DOWNLOAD_CACHE)") - @test -d "$(CONDA_HOME)" || test -f "$(DOWNLOAD_CACHE)/$(FN)" || \ + @test -d "$(CONDA_HOME)/bin/conda" || test -f "$(DOWNLOAD_CACHE)/$(FN)" || \ (echo "Fetching conda distribution from: $(CONDA_URL)/$(FN)" && \ curl "$(CONDA_URL)/$(FN)" --insecure --output "$(DOWNLOAD_CACHE)/$(FN)") @test -f "$(CONDA_HOME)/bin/conda" || \ From b9231c09f05dd4464a0771600577e688000b161e Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Wed, 30 Jan 2019 19:29:18 -0500 Subject: [PATCH 26/76] enforce conda install even if dir exists --- Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index f791e2efd..33b25aa59 100644 --- a/Makefile +++ b/Makefile @@ -245,13 +245,13 @@ docker-push: docker-build .PHONY: conda-base conda-base: - @test -d "$(CONDA_HOME)" || test -d "$(DOWNLOAD_CACHE)" || \ + @test -f "$(CONDA_HOME)/bin/conda" || test -d "$(DOWNLOAD_CACHE)" || \ (echo "Creating download directory: $(DOWNLOAD_CACHE)" && mkdir -p "$(DOWNLOAD_CACHE)") - @test -d "$(CONDA_HOME)/bin/conda" || test -f "$(DOWNLOAD_CACHE)/$(FN)" || \ + @test -f "$(CONDA_HOME)/bin/conda" || test -f "$(DOWNLOAD_CACHE)/$(FN)" || \ (echo "Fetching conda distribution from: $(CONDA_URL)/$(FN)" && \ curl "$(CONDA_URL)/$(FN)" --insecure --output "$(DOWNLOAD_CACHE)/$(FN)") @test -f "$(CONDA_HOME)/bin/conda" || \ - (bash "$(DOWNLOAD_CACHE)/$(FN)" -b -p "$(CONDA_HOME)" && \ + (bash "$(DOWNLOAD_CACHE)/$(FN)" -b -u -p "$(CONDA_HOME)" && \ echo "Make sure to add '$(CONDA_HOME)/bin' to your PATH variable in '~/.bashrc'.") .PHONY: conda-cfg From 7f49a9e4eb741fcada147a316d45a48392782274 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Wed, 30 Jan 2019 20:04:50 -0500 Subject: [PATCH 27/76] update conda create --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 33b25aa59..d5eb8cb92 100644 --- a/Makefile +++ b/Makefile @@ -269,4 +269,4 @@ conda_config: conda-base conda-env: conda-base @test -d "$(CONDA_ENV_PATH)" || \ (echo "Creating conda environment at '$(CONDA_ENV_PATH)'..." && \ - "$(CONDA_HOME)/bin/conda" env create -n "$(CONDA_ENV)") + "$(CONDA_HOME)/bin/conda" create -n "$(CONDA_ENV)") From 1268c706cea14ef9f525d5c41623329bbcda26e4 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Wed, 30 Jan 2019 20:07:39 -0500 Subject: [PATCH 28/76] YES install it conda... --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index d5eb8cb92..ce3437de5 100644 --- a/Makefile +++ b/Makefile @@ -269,4 +269,4 @@ conda_config: conda-base conda-env: conda-base @test -d "$(CONDA_ENV_PATH)" || \ (echo "Creating conda environment at '$(CONDA_ENV_PATH)'..." && \ - "$(CONDA_HOME)/bin/conda" create -n "$(CONDA_ENV)") + "$(CONDA_HOME)/bin/conda" create -y -n "$(CONDA_ENV)") From 27b3bd237c05103340c6f5719c9a8bcf1de438b4 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Wed, 30 Jan 2019 20:18:59 -0500 Subject: [PATCH 29/76] force conda activate env --- .travis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index f2dd66587..fb506bf93 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,12 +28,12 @@ before_install: - hash -r - env - make conda-base - - make sysinstall + - bash -c "conda activate $CONDA_ENV && make sysinstall" install: - - make install install-dev + - bash -c "conda activate $CONDA_ENV && make install install-dev" - make version script: - - make $TARGET + - bash -c "conda activate $CONDA_ENV && make $TARGET" notifications: email: false jobs: From bead440c9ecab4e35ce6e5bdea3223485b6bd70a Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Wed, 30 Jan 2019 20:43:13 -0500 Subject: [PATCH 30/76] revert bash && print info --- .travis.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index fb506bf93..3e141c4e3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,12 +28,15 @@ before_install: - hash -r - env - make conda-base - - bash -c "conda activate $CONDA_ENV && make sysinstall" + - make sysinstall + - echo $CONDA_PREFIX + - conda info + - conda env list install: - - bash -c "conda activate $CONDA_ENV && make install install-dev" + - make install install-dev - make version script: - - bash -c "conda activate $CONDA_ENV && make $TARGET" + - make $TARGET notifications: email: false jobs: From 2d539e5441945a97a611700864aca94e340be766 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Thu, 31 Jan 2019 10:09:41 -0500 Subject: [PATCH 31/76] travis conda config --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 3e141c4e3..609e7451e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,11 +26,13 @@ before_install: - export CONDA_ENV=magpie-${TRAVIS_PYTHON_VERSION} - export PATH="${CONDA_HOME}/bin":$PATH - hash -r + - conda config --set always_yes yes --set changeps1 no + - conda update -q conda + - conda info -a - env - make conda-base - make sysinstall - echo $CONDA_PREFIX - - conda info - conda env list install: - make install install-dev From c269ba3d661e0ad60d60256b67ea0f83770bfed3 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Thu, 31 Jan 2019 10:18:03 -0500 Subject: [PATCH 32/76] activate env --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 609e7451e..dad302d9d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -34,6 +34,7 @@ before_install: - make sysinstall - echo $CONDA_PREFIX - conda env list + - conda activate $CONDA_ENV install: - make install install-dev - make version From f6635a1c716e6d33432c83c451e38344ec434cfb Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Thu, 31 Jan 2019 10:55:18 -0500 Subject: [PATCH 33/76] coverage report & init conda --- .travis.yml | 4 ++++ Makefile | 6 +++++- README.rst | 6 +++++- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index dad302d9d..9fb639739 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,7 +33,9 @@ before_install: - make conda-base - make sysinstall - echo $CONDA_PREFIX + - echo $CONDA_ENV - conda env list + - conda init bash - conda activate $CONDA_ENV install: - make install install-dev @@ -52,3 +54,5 @@ jobs: skip_existing: true on: tags: true +after_success: + - bash <(curl -s https://codecov.io/bash) || echo "Codecov did not collect coverage reports" diff --git a/Makefile b/Makefile index ce3437de5..3493dc928 100644 --- a/Makefile +++ b/Makefile @@ -66,7 +66,8 @@ help: @echo " migrate: run postgres database migration with alembic" @echo " start: start magpie instance with gunicorn" @echo " Test and coverage" - @echo " coverage: check code coverage quickly with the default Python" + @echo " coverage: check code coverage and generate a report" + @echo " coverage-show: check code coverage and generate a report served on a web interface" @echo " lint: check style with flake8" @echo " test: run tests quickly with the default Python" @echo " test-local: run only local tests with the default Python" @@ -131,6 +132,9 @@ coverage: install-dev install @echo "Running coverage analysis..." @bash -c 'source "$(CONDA_HOME)/bin/activate" "$(CONDA_ENV)"; coverage run --source magpie setup.py test || true' @bash -c 'source "$(CONDA_HOME)/bin/activate" "$(CONDA_ENV)"; coverage report -m' + +.PHONY: coverage-show +coverage-show: coverage @bash -c 'source "$(CONDA_HOME)/bin/activate" "$(CONDA_ENV)"; coverage html -d coverage' "$(BROWSER)" "$(CUR_DIR)/coverage/index.html" diff --git a/README.rst b/README.rst index 0e6e09c66..96d1e7ec0 100644 --- a/README.rst +++ b/README.rst @@ -17,7 +17,7 @@ Behind the scene, it uses `Ziggurat-Foundations`_ and `Authomatic`_. * - dependencies - | |py_ver| |requires| * - build status - - | |travis_latest| |travis_tag| + - | |travis_latest| |travis_tag| |coverage| * - docker status - | |docker_build_mode| |docker_build_status| * - releases @@ -47,6 +47,10 @@ Behind the scene, it uses `Ziggurat-Foundations`_ and `Authomatic`_. :alt: Travis-CI Build Status (latest tag) :target: https://github.com/Ouranosinc/Magpie/tree/0.9.0 +.. |coverage| image:: https://img.shields.io/codecov/c/gh/Ouranosinc/Magpie.svg?label=coverage + :alt: Travis-CI CodeCov Coverage + :target: https://codecov.io/gh/Ouranosinc/Magpie + .. |docker_build_mode| image:: https://img.shields.io/docker/automated/pavics/magpie.svg?label=build :alt: Docker Build Status (latest tag) :target: https://hub.docker.com/r/pavics/magpie/builds From 9cd4cc9e021a82f3325f4b1d3fd6fdbc0b26a188 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Thu, 31 Jan 2019 10:57:42 -0500 Subject: [PATCH 34/76] remove conda activate travis doesnt like --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 9fb639739..cbe81a69a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -36,7 +36,6 @@ before_install: - echo $CONDA_ENV - conda env list - conda init bash - - conda activate $CONDA_ENV install: - make install install-dev - make version From 56eca20506c789673bfde952cc00acbc3809ff8c Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Thu, 31 Jan 2019 14:33:45 -0500 Subject: [PATCH 35/76] adjust openid req for py2/3 --- requirements-py2.txt | 2 ++ requirements-py3.txt | 2 ++ requirements.txt | 1 - setup.py | 54 ++++++++++++++++++++------------------------ 4 files changed, 29 insertions(+), 30 deletions(-) create mode 100644 requirements-py2.txt create mode 100644 requirements-py3.txt diff --git a/requirements-py2.txt b/requirements-py2.txt new file mode 100644 index 000000000..f405c0315 --- /dev/null +++ b/requirements-py2.txt @@ -0,0 +1,2 @@ +# authomatic doesn't properly install valid version of openid +python-openid diff --git a/requirements-py3.txt b/requirements-py3.txt new file mode 100644 index 000000000..4d0036091 --- /dev/null +++ b/requirements-py3.txt @@ -0,0 +1,2 @@ +# authomatic doesn't properly install valid version of openid +python3-openid diff --git a/requirements.txt b/requirements.txt index b10e1887f..f5a373951 100644 --- a/requirements.txt +++ b/requirements.txt @@ -31,4 +31,3 @@ typing # until fix merged and deployed #-e git+https://github.com/fmigneault/authomatic.git@httplib-port#egg=Authomatic https://github.com/fmigneault/authomatic/archive/httplib-port.zip#egg=Authomatic - diff --git a/setup.py b/setup.py index 49892ef4c..ef936d4fd 100644 --- a/setup.py +++ b/setup.py @@ -7,13 +7,12 @@ MAGPIE_MODULE_DIR = os.path.join(MAGPIE_ROOT, 'magpie') sys.path.insert(0, MAGPIE_MODULE_DIR) -from setuptools import find_packages +from setuptools import find_packages # noqa: F401 try: from setuptools import setup except ImportError: from distutils.core import setup - -from magpie import __meta__ +from magpie import __meta__ # noqa: F401 with open('README.rst') as readme_file: README = readme_file.read() @@ -21,31 +20,28 @@ with open('HISTORY.rst') as history_file: HISTORY = history_file.read().replace('.. :changelog:', '') -LINKS = set() # See https://github.com/pypa/pip/issues/3610 -REQUIREMENTS = set() # use set to have unique packages by name -with open('requirements.txt', 'r') as requirements_file: - for line in requirements_file: - if 'git+https' in line: - pkg = line.split('#')[-1] - LINKS.add(line.strip()) - REQUIREMENTS.add(pkg.replace('egg=', '').rstrip()) - elif line.startswith('http'): - LINKS.add(line.strip()) - else: - REQUIREMENTS.add(line.strip()) -TEST_REQUIREMENTS = set() -with open('requirements-dev.txt', 'r') as requirements_file: - for line in requirements_file: - if 'git+https' in line: - pkg = line.split('#')[-1] - LINKS.add(line.strip()) - REQUIREMENTS.add(pkg.replace('egg=', '').rstrip()) - elif line.startswith('http'): - LINKS.add(line.strip()) - else: - REQUIREMENTS.add(line.strip()) +def _parse_requirements(file_path, requirements, links): + with open(file_path, 'r') as requirements_file: + for line in requirements_file: + if 'git+https' in line: + pkg = line.split('#')[-1] + links.add(line.strip()) + requirements.add(pkg.replace('egg=', '').rstrip()) + elif line.startswith('http'): + links.add(line.strip()) + else: + requirements.add(line.strip()) + +# See https://github.com/pypa/pip/issues/3610 +# use set to have unique packages by name +LINKS = set() +REQUIREMENTS = set() +TEST_REQUIREMENTS = set() +_parse_requirements('requirements.txt', REQUIREMENTS, LINKS) +_parse_requirements('requirements-py{}.txt'.format(sys.version[0]), REQUIREMENTS, LINKS) +_parse_requirements('requirements-dev.txt', TEST_REQUIREMENTS, LINKS) LINKS = list(LINKS) REQUIREMENTS = list(REQUIREMENTS) TEST_REQUIREMENTS = list(TEST_REQUIREMENTS) @@ -94,9 +90,9 @@ zip_safe=False, # -- self - tests -------------------------------------------------------- - #test_suite='nose.collector', - #test_suite='tests.test_runner', - #test_loader='tests.test_runner:run_suite', + # test_suite='nose.collector', + # test_suite='tests.test_runner', + # test_loader='tests.test_runner:run_suite', tests_require=TEST_REQUIREMENTS, # -- script entry points ----------------------------------------------- From 974eb99821df54dfd463d00f570870d410f11cb0 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Thu, 31 Jan 2019 14:50:00 -0500 Subject: [PATCH 36/76] travis add postgres and start magpie instance --- .travis.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index cbe81a69a..ff41974a1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,9 +16,13 @@ env: - CONDA_HOME=$HOME/conda - DOWNLOAD_CACHE=$HOME/downloads matrix: - - TARGET=test-local - - TARGET=test-remote - - TARGET=coverage + - TEST_TARGET=test-local START_TARGET= + - TEST_TARGET=test-remote START_TARGET=start + - TEST_TARGET=coverage START_TARGET= +addons: + postgresql: "9.6" +before_script: + - psql -c 'create database magpie;' -U postgres before_install: - python --version - uname -a @@ -40,7 +44,7 @@ install: - make install install-dev - make version script: - - make $TARGET + - make $START_TARGET $TEST_TARGET notifications: email: false jobs: From 364b0b287f7500991757aa027c9de93386c03643 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Thu, 31 Jan 2019 14:52:33 -0500 Subject: [PATCH 37/76] remove failing conda direct calls --- .travis.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index ff41974a1..dad7b10f5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -30,16 +30,16 @@ before_install: - export CONDA_ENV=magpie-${TRAVIS_PYTHON_VERSION} - export PATH="${CONDA_HOME}/bin":$PATH - hash -r - - conda config --set always_yes yes --set changeps1 no - - conda update -q conda - - conda info -a + #- conda config --set always_yes yes --set changeps1 no + #- conda update -q conda + #- conda info -a - env - make conda-base - make sysinstall - echo $CONDA_PREFIX - echo $CONDA_ENV - - conda env list - - conda init bash + #- conda env list + #- conda init bash install: - make install install-dev - make version From 27b57034965cd184b5bada8ea339813d6b742448 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Thu, 31 Jan 2019 14:58:03 -0500 Subject: [PATCH 38/76] python 3 compat --- magpie/api/login/login.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/magpie/api/login/login.py b/magpie/api/login/login.py index b4b371628..cff548614 100644 --- a/magpie/api/login/login.py +++ b/magpie/api/login/login.py @@ -19,7 +19,7 @@ default_provider = get_constant('MAGPIE_DEFAULT_PROVIDER') MAGPIE_INTERNAL_PROVIDERS = {default_provider: default_provider.capitalize()} MAGPIE_EXTERNAL_PROVIDERS = get_provider_names() -MAGPIE_PROVIDER_KEYS = MAGPIE_INTERNAL_PROVIDERS.keys() + MAGPIE_EXTERNAL_PROVIDERS.keys() +MAGPIE_PROVIDER_KEYS = list(MAGPIE_INTERNAL_PROVIDERS.keys()) + list(MAGPIE_EXTERNAL_PROVIDERS.keys()) def process_sign_in_external(request, username, provider): From b9ea1afb328616324743add2e241ef493e003497 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Thu, 31 Jan 2019 15:00:36 -0500 Subject: [PATCH 39/76] separate make start/cron, call each in docker --- Dockerfile | 2 +- Makefile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8c92648c6..2c0c5b528 100644 --- a/Dockerfile +++ b/Dockerfile @@ -31,4 +31,4 @@ RUN touch ~/magpie_cron_status.log # set /etc/environment so that cron runs using the environment variables set by docker RUN env >> /etc/environment -CMD make start +CMD make start cron diff --git a/Makefile b/Makefile index 3493dc928..9079addbf 100644 --- a/Makefile +++ b/Makefile @@ -220,7 +220,7 @@ cron: cron .PHONY: start -start: cron install +start: install @echo "Starting Magpie..." exec gunicorn -b 0.0.0.0:2001 --paste "$(CUR_DIR)/magpie/magpie.ini" --workers 10 --preload From 1306a2248bfe3c5938c6a67f37de2a80b9188a33 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Thu, 31 Jan 2019 15:21:56 -0500 Subject: [PATCH 40/76] allow override magpie db migration --- .travis.yml | 3 +++ ci/magpie.env | 1 + magpie/magpiectl.py | 3 ++- tests/utils.py | 2 +- 4 files changed, 7 insertions(+), 2 deletions(-) create mode 100644 ci/magpie.env diff --git a/.travis.yml b/.travis.yml index dad7b10f5..3f400cff2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -40,6 +40,9 @@ before_install: - echo $CONDA_ENV #- conda env list #- conda init bash + #==== magpie env and constants === + - mkdir -p ./env + - cp -f ./ci/magpie.env ./env/magpie.env install: - make install install-dev - make version diff --git a/ci/magpie.env b/ci/magpie.env new file mode 100644 index 000000000..17ea167d8 --- /dev/null +++ b/ci/magpie.env @@ -0,0 +1 @@ +MAGPIE_DB_MIGRATION=False diff --git a/magpie/magpiectl.py b/magpie/magpiectl.py index c3a5ba763..a56da0175 100644 --- a/magpie/magpiectl.py +++ b/magpie/magpiectl.py @@ -44,7 +44,8 @@ def main(global_config=None, **settings): LOGGER.setLevel(log_lvl) # migrate db as required and check if database is ready - if not settings.get('magpie.db_migration_disabled', False): + if get_constant('MAGPIE_DB_MIGRATION', settings, 'magpie.db_migration', True, + raise_missing=False, raise_not_set=False, print_missing=True): print_log('Running database migration (as required) ...') try: with warnings.catch_warnings(): diff --git a/tests/utils.py b/tests/utils.py index 7ad64ada3..52aeec710 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -32,7 +32,7 @@ def get_test_magpie_app(): config = config_setup_from_ini(get_constant('MAGPIE_INI_FILE_PATH')) config.include('ziggurat_foundations.ext.pyramid.sign_in') config.include('ziggurat_foundations.ext.pyramid.get_user') - config.registry.settings['magpie.db_migration_disabled'] = True + config.registry.settings['magpie.db_migration'] = False # scan dependencies config.include('magpie') # create the test application From 6652434883af90ab2e79f58113f59d7cec07b292 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Thu, 31 Jan 2019 15:29:51 -0500 Subject: [PATCH 41/76] MAPGIE_DB_MIGRATION var && travis CONDA_PREFIX --- .travis.yml | 1 + ci/magpie.env | 2 +- magpie/magpiectl.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3f400cff2..788c10ea3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,6 +28,7 @@ before_install: - uname -a - lsb_release -a - export CONDA_ENV=magpie-${TRAVIS_PYTHON_VERSION} + - export CONDA_PREFIX=$HOME/conda/envs/magpie-${TRAVIS_PYTHON_VERSION} - export PATH="${CONDA_HOME}/bin":$PATH - hash -r #- conda config --set always_yes yes --set changeps1 no diff --git a/ci/magpie.env b/ci/magpie.env index 17ea167d8..bfbd55379 100644 --- a/ci/magpie.env +++ b/ci/magpie.env @@ -1 +1 @@ -MAGPIE_DB_MIGRATION=False +MAGPIE_DB_MIGRATION=True diff --git a/magpie/magpiectl.py b/magpie/magpiectl.py index a56da0175..b20b5b084 100644 --- a/magpie/magpiectl.py +++ b/magpie/magpiectl.py @@ -54,7 +54,7 @@ def main(global_config=None, **settings): except ImportError: pass except Exception as e: - raise_log('Database migration failed [{}]'.format(str(e))) + raise_log('Database migration failed [{}]'.format(str(e)), exception=e) if not db.is_database_ready(): time.sleep(2) raise_log('Database not ready') From 297e9cfd87d94c986b2b89ecee30c585403adea1 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Thu, 31 Jan 2019 15:50:40 -0500 Subject: [PATCH 42/76] raise-log check message --- magpie/common.py | 6 ++++++ magpie/magpiectl.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/magpie/common.py b/magpie/common.py index 4c42a1753..012fd59a8 100644 --- a/magpie/common.py +++ b/magpie/common.py @@ -2,6 +2,9 @@ # -*- coding: utf-8 -*- from __future__ import unicode_literals from distutils.dir_util import mkpath +from typing import AnyStr, Optional, Type +# noinspection PyProtectedMember +from logging import _loggerClass as LoggerType # noinspection PyCompatibility import configparser import logging @@ -18,7 +21,10 @@ def print_log(msg, logger=LOGGER, level=logging.INFO): def raise_log(msg, exception=Exception, logger=LOGGER, level=logging.ERROR): + # type: (AnyStr, Optional[Type[Exception]], Optional[LoggerType], Optional[int]) -> None logger.log(level, msg) + if not hasattr(exception, 'message'): + exception = Exception raise exception(msg) diff --git a/magpie/magpiectl.py b/magpie/magpiectl.py index b20b5b084..e8a3cea69 100644 --- a/magpie/magpiectl.py +++ b/magpie/magpiectl.py @@ -54,7 +54,7 @@ def main(global_config=None, **settings): except ImportError: pass except Exception as e: - raise_log('Database migration failed [{}]'.format(str(e)), exception=e) + raise_log('Database migration failed [{}]'.format(str(e)), exception=RuntimeError) if not db.is_database_ready(): time.sleep(2) raise_log('Database not ready') From c108aa4e6cb85cffdf155e9659c0e5356321649e Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Thu, 31 Jan 2019 15:53:43 -0500 Subject: [PATCH 43/76] pytest mark skip instead of radical skip --- tests/test_magpie_adapter.py | 2 +- tests/test_register.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_magpie_adapter.py b/tests/test_magpie_adapter.py index d2956b0f4..d44142176 100644 --- a/tests/test_magpie_adapter.py +++ b/tests/test_magpie_adapter.py @@ -2,7 +2,7 @@ import unittest -@pytest.skip("not implemented") +@pytest.mark.skip("not implemented") @unittest.skip("not implemented") class TestAdapter(unittest.TestCase): def test_adapter(self): diff --git a/tests/test_register.py b/tests/test_register.py index a8d7b07b4..b76924df4 100644 --- a/tests/test_register.py +++ b/tests/test_register.py @@ -2,7 +2,7 @@ import unittest -@pytest.skip("not implemented") +@pytest.mark.skip("not implemented") @unittest.skip("not implemented") class TestRegister(unittest.TestCase): def test_register(self): From 0aea75ad19043ca1b077ed47f7cd535ef3f8088d Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Thu, 31 Jan 2019 16:09:12 -0500 Subject: [PATCH 44/76] avoid invalid .message --- magpie/alembic/env.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/magpie/alembic/env.py b/magpie/alembic/env.py index ad4de2e92..a76bbfe82 100644 --- a/magpie/alembic/env.py +++ b/magpie/alembic/env.py @@ -69,7 +69,7 @@ def run_migrations_online(): # see for details: # https://stackoverflow.com/questions/6506578 db_name = get_constant('MAGPIE_POSTGRES_DB') - if 'database "{}" does not exist'.format(db_name) not in ex.message: + if 'database "{}" does not exist'.format(db_name) not in str(ex): raise # any error is OperationalError, so validate only missing db error db_default_postgres_url = get_db_url( username='postgres', # only postgres user can connect to default 'postgres' db From 74dd6bd20a63e37099af6b2a1a2f5f6bcab62282 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Thu, 31 Jan 2019 16:21:16 -0500 Subject: [PATCH 45/76] make sure conda install env with expected py version --- .travis.yml | 4 +++- Makefile | 7 ++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 788c10ea3..00c8550fa 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,9 +27,11 @@ before_install: - python --version - uname -a - lsb_release -a + - export PYTHON_VERSION=${TRAVIS_PYTHON_VERSION} - export CONDA_ENV=magpie-${TRAVIS_PYTHON_VERSION} - export CONDA_PREFIX=$HOME/conda/envs/magpie-${TRAVIS_PYTHON_VERSION} - - export PATH="${CONDA_HOME}/bin":$PATH + - export PATH=${CONDA_HOME}/bin:$PATH + - conda update --yes conda - hash -r #- conda config --set always_yes yes --set changeps1 no #- conda update -q conda diff --git a/Makefile b/Makefile index 9079addbf..076314608 100644 --- a/Makefile +++ b/Makefile @@ -22,14 +22,15 @@ CONDA_HOME ?= $(HOME)/conda CONDA_ENVS_DIR ?= $(CONDA_HOME)/envs CONDA_ENV_PATH := $(CONDA_ENVS_DIR)/$(CONDA_ENV) DOWNLOAD_CACHE ?= $(APP_ROOT)/downloads +PYTHON_VERSION ?= 2.7 # choose conda installer depending on your OS CONDA_URL = https://repo.continuum.io/miniconda OS_NAME := $(shell uname -s || echo "unknown") ifeq "$(OS_NAME)" "Linux" -FN := Miniconda3-latest-Linux-x86_64.sh +FN := Miniconda-latest-Linux-x86_64.sh else ifeq "$(OS_NAME)" "Darwin" -FN := Miniconda3-latest-MacOSX-x86_64.sh +FN := Miniconda-latest-MacOSX-x86_64.sh else FN := unknown endif @@ -273,4 +274,4 @@ conda_config: conda-base conda-env: conda-base @test -d "$(CONDA_ENV_PATH)" || \ (echo "Creating conda environment at '$(CONDA_ENV_PATH)'..." && \ - "$(CONDA_HOME)/bin/conda" create -y -n "$(CONDA_ENV)") + "$(CONDA_HOME)/bin/conda" create -y -n "$(CONDA_ENV)" python=$PYTHON_VERSION) From 4d1d60afeb506f477aa79e52b6d5e797a53f5ddc Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Thu, 31 Jan 2019 16:26:49 -0500 Subject: [PATCH 46/76] conda being a pain --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 00c8550fa..b44c40753 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,7 +31,6 @@ before_install: - export CONDA_ENV=magpie-${TRAVIS_PYTHON_VERSION} - export CONDA_PREFIX=$HOME/conda/envs/magpie-${TRAVIS_PYTHON_VERSION} - export PATH=${CONDA_HOME}/bin:$PATH - - conda update --yes conda - hash -r #- conda config --set always_yes yes --set changeps1 no #- conda update -q conda From 2998a5b1b0bfe5da64cee67736a3e28fc55ae31a Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Thu, 31 Jan 2019 16:32:50 -0500 Subject: [PATCH 47/76] postgres connection setup for travis --- .travis.yml | 5 +++++ ci/magpie.env | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/.travis.yml b/.travis.yml index b44c40753..042a4ba25 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,6 +21,11 @@ env: - TEST_TARGET=coverage START_TARGET= addons: postgresql: "9.6" +postgres: + adapter: postgresql + database: magpie + username: postgres + password: qwerty before_script: - psql -c 'create database magpie;' -U postgres before_install: diff --git a/ci/magpie.env b/ci/magpie.env index bfbd55379..b9a9a0cb0 100644 --- a/ci/magpie.env +++ b/ci/magpie.env @@ -1 +1,6 @@ MAGPIE_DB_MIGRATION=True +MAGPIE_POSTGRES_USER=postgres +MAGPIE_POSTGRES_PASSWORD=qwerty +MAGPIE_POSTGRES_HOST=localhost +MAGPIE_POSTGRES_PORT=5432 +MAGPIE_POSTGRES_DB=magpie From e5671feeca228992428f19337d9c6bd590bea3f7 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Thu, 31 Jan 2019 17:20:10 -0500 Subject: [PATCH 48/76] override python --- Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 076314608..a8e28d6e8 100644 --- a/Makefile +++ b/Makefile @@ -22,15 +22,15 @@ CONDA_HOME ?= $(HOME)/conda CONDA_ENVS_DIR ?= $(CONDA_HOME)/envs CONDA_ENV_PATH := $(CONDA_ENVS_DIR)/$(CONDA_ENV) DOWNLOAD_CACHE ?= $(APP_ROOT)/downloads -PYTHON_VERSION ?= 2.7 +PYTHON_VERSION ?= `python -c 'import sys; print(sys.version[:5])'` # choose conda installer depending on your OS CONDA_URL = https://repo.continuum.io/miniconda OS_NAME := $(shell uname -s || echo "unknown") ifeq "$(OS_NAME)" "Linux" -FN := Miniconda-latest-Linux-x86_64.sh +FN := Miniconda3-latest-Linux-x86_64.sh else ifeq "$(OS_NAME)" "Darwin" -FN := Miniconda-latest-MacOSX-x86_64.sh +FN := Miniconda3-latest-MacOSX-x86_64.sh else FN := unknown endif From 543ac6d7fac36de6dc23bcfdf68558e52ede52e3 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Thu, 31 Jan 2019 17:24:25 -0500 Subject: [PATCH 49/76] python version... --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index a8e28d6e8..05feb085c 100644 --- a/Makefile +++ b/Makefile @@ -274,4 +274,4 @@ conda_config: conda-base conda-env: conda-base @test -d "$(CONDA_ENV_PATH)" || \ (echo "Creating conda environment at '$(CONDA_ENV_PATH)'..." && \ - "$(CONDA_HOME)/bin/conda" create -y -n "$(CONDA_ENV)" python=$PYTHON_VERSION) + "$(CONDA_HOME)/bin/conda" create -y -n "$(CONDA_ENV)" python=$(PYTHON_VERSION)) From 382fe7fc38d111ae53069853d5f6b8f04301f3da Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Thu, 31 Jan 2019 17:36:06 -0500 Subject: [PATCH 50/76] install gunicorn & env settings --- .travis.yml | 1 + ci/magpie.env | 2 ++ 2 files changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index 042a4ba25..5614222f5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -45,6 +45,7 @@ before_install: - make sysinstall - echo $CONDA_PREFIX - echo $CONDA_ENV + - pip install gunicorn #- conda env list #- conda init bash #==== magpie env and constants === diff --git a/ci/magpie.env b/ci/magpie.env index b9a9a0cb0..b25c37787 100644 --- a/ci/magpie.env +++ b/ci/magpie.env @@ -4,3 +4,5 @@ MAGPIE_POSTGRES_PASSWORD=qwerty MAGPIE_POSTGRES_HOST=localhost MAGPIE_POSTGRES_PORT=5432 MAGPIE_POSTGRES_DB=magpie +PHOENIX_PUSH=false +HOSTNAME=localhost From 73f1bc4f650cc8be242976c793992172438d8faa Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Thu, 31 Jan 2019 17:54:59 -0500 Subject: [PATCH 51/76] force install pkgs --- .travis.yml | 4 +++- Makefile | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5614222f5..ef0e523ae 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,7 +29,7 @@ postgres: before_script: - psql -c 'create database magpie;' -U postgres before_install: - - python --version + - python -V - uname -a - lsb_release -a - export PYTHON_VERSION=${TRAVIS_PYTHON_VERSION} @@ -41,6 +41,8 @@ before_install: #- conda update -q conda #- conda info -a - env + - python -V + - which python - make conda-base - make sysinstall - echo $CONDA_PREFIX diff --git a/Makefile b/Makefile index 05feb085c..2e4475942 100644 --- a/Makefile +++ b/Makefile @@ -208,7 +208,7 @@ sysinstall: clean conda-env .PHONY: install install: sysinstall @echo "Installing Magpie..." - @bash -c 'source "$(CONDA_HOME)/bin/activate" "$(CONDA_ENV)"; pip install "$(CUR_DIR)"' + @bash -c 'source "$(CONDA_HOME)/bin/activate" "$(CONDA_ENV)"; pip install --force-reinstall "$(CUR_DIR)"' .PHONY: install-dev install-dev: conda-env From 44e1912cd633c8dc80dd54d5a7fd518dac15db58 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Thu, 31 Jan 2019 18:41:14 -0500 Subject: [PATCH 52/76] force override versions --- .travis.yml | 1 + Makefile | 7 ++++++- requirements.txt | 7 ++++--- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index ef0e523ae..fa9e3c298 100644 --- a/.travis.yml +++ b/.travis.yml @@ -43,6 +43,7 @@ before_install: - env - python -V - which python + - which pip - make conda-base - make sysinstall - echo $CONDA_PREFIX diff --git a/Makefile b/Makefile index 2e4475942..d6dd42b8d 100644 --- a/Makefile +++ b/Makefile @@ -208,7 +208,12 @@ sysinstall: clean conda-env .PHONY: install install: sysinstall @echo "Installing Magpie..." - @bash -c 'source "$(CONDA_HOME)/bin/activate" "$(CONDA_ENV)"; pip install --force-reinstall "$(CUR_DIR)"' + # TODO: remove when merged + # --- ensure fix is applied + @bash -c 'source "$(CONDA_HOME)/bin/activate" "$(CONDA_ENV)"; \ + pip install --force-reinstall "https://github.com/fmigneault/authomatic/archive/httplib-port.zip#egg=Authomatic"' + # --- + @bash -c 'source "$(CONDA_HOME)/bin/activate" "$(CONDA_ENV)"; pip install --upgrade "$(CUR_DIR)"' .PHONY: install-dev install-dev: conda-env diff --git a/requirements.txt b/requirements.txt index f5a373951..f8ed624d5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -27,7 +27,8 @@ threddsclient==0.3.4 humanize requests_file typing -#authomatic==0.1.0.post1 -# until fix merged and deployed +# TODO: remove when merged +# until fix merged and deployed (https://github.com/authomatic/authomatic/pull/195) +# authomatic==0.1.0.post1 #-e git+https://github.com/fmigneault/authomatic.git@httplib-port#egg=Authomatic -https://github.com/fmigneault/authomatic/archive/httplib-port.zip#egg=Authomatic +https://github.com/fmigneault/authomatic/archive/httplib-port.zip#egg=Authomatic>=0.1.0.post2 From a633fb14383f8ea0b1601aa70f8a23e82e0cb09a Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Thu, 31 Jan 2019 18:59:39 -0500 Subject: [PATCH 53/76] verify conda env before start/test --- .travis.yml | 24 +++++++++++++----------- magpie/__meta__.py | 3 +++ setup.py | 7 ++++--- 3 files changed, 20 insertions(+), 14 deletions(-) diff --git a/.travis.yml b/.travis.yml index fa9e3c298..14f754ef5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,8 +26,6 @@ postgres: database: magpie username: postgres password: qwerty -before_script: - - psql -c 'create database magpie;' -U postgres before_install: - python -V - uname -a @@ -37,26 +35,30 @@ before_install: - export CONDA_PREFIX=$HOME/conda/envs/magpie-${TRAVIS_PYTHON_VERSION} - export PATH=${CONDA_HOME}/bin:$PATH - hash -r - #- conda config --set always_yes yes --set changeps1 no - #- conda update -q conda - #- conda info -a - env - - python -V - - which python - - which pip - make conda-base - make sysinstall - echo $CONDA_PREFIX - echo $CONDA_ENV - - pip install gunicorn - #- conda env list - #- conda init bash #==== magpie env and constants === - mkdir -p ./env - cp -f ./ci/magpie.env ./env/magpie.env install: + - pip install gunicorn paste - make install install-dev - make version +before_script: + - psql -c 'create database magpie;' -U postgres + - echo $CONDA_PREFIX + - echo $CONDA_ENV + - export PYTHON_VERSION=${TRAVIS_PYTHON_VERSION} + - export CONDA_ENV=magpie-${TRAVIS_PYTHON_VERSION} + - export CONDA_PREFIX=$HOME/conda/envs/magpie-${TRAVIS_PYTHON_VERSION} + - export PATH=${CONDA_HOME}/bin:$PATH + - hash -r + - env + - echo $CONDA_PREFIX + - echo $CONDA_ENV script: - make $START_TARGET $TEST_TARGET notifications: diff --git a/magpie/__meta__.py b/magpie/__meta__.py index 0bba03bf6..60887e128 100644 --- a/magpie/__meta__.py +++ b/magpie/__meta__.py @@ -11,3 +11,6 @@ __url__ = 'https://github.com/Ouranosinc/Magpie' __docker__ = 'https://hub.docker.com/r/pavics/magpie' __description__ = "Magpie is a service for AuthN and AuthZ based on Ziggurat-Foundations" +__platforms__ = ['linux_x86_64'] +__natural_language__ = 'English' +__license__ = "ISCL" diff --git a/setup.py b/setup.py index ef936d4fd..e98d9548e 100644 --- a/setup.py +++ b/setup.py @@ -69,16 +69,17 @@ def _parse_requirements(file_path, requirements, links): contact=__meta__.__maintainer__, contact_email=__meta__.__email__, url=__meta__.__url__, - platforms=['linux_x86_64'], - license="ISCL", + platforms=__meta__.__platforms__, + license=__meta__.__license__, keywords=__meta__.__title__ + ", Authentication, AuthN, Birdhouse", classifiers=[ - 'Development Status :: 2 - Pre-Alpha', + 'Development Status :: 3 - Alpha', 'Intended Audience :: Developers', 'License :: OSI Approved :: ISC License (ISCL)', 'Natural Language :: English', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', ], # -- Package structure ------------------------------------------------- From 42e89101f09fddc1f3da5ecfb57b2958281e9767 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Thu, 31 Jan 2019 19:10:59 -0500 Subject: [PATCH 54/76] try enforce install directly --- .travis.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.travis.yml b/.travis.yml index 14f754ef5..fa2f24792 100644 --- a/.travis.yml +++ b/.travis.yml @@ -44,7 +44,14 @@ before_install: - mkdir -p ./env - cp -f ./ci/magpie.env ./env/magpie.env install: + # TODO: remove when fixed and merged (see requirements.txt and Makefile) - pip install gunicorn paste + - | + if [ "${TRAVIS_PYTHON_VERSION}" -eq "2.7" ]; then + ${CONDA_PREFIX}/bin/pip install --upgrade python-openid; + else + ${CONDA_PREFIX}/bin/pip install --upgrade python3-openid; + fi - make install install-dev - make version before_script: From 49e813c163ed3dc943d1074c5cdc1c29e74bab1f Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Thu, 31 Jan 2019 19:19:14 -0500 Subject: [PATCH 55/76] explicit to the max openid version install --- .travis.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index fa2f24792..e4b6a502d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -48,10 +48,13 @@ install: - pip install gunicorn paste - | if [ "${TRAVIS_PYTHON_VERSION}" -eq "2.7" ]; then - ${CONDA_PREFIX}/bin/pip install --upgrade python-openid; + ${CONDA_PREFIX}/bin/pip install --upgrade --force-reinstall python-openid && \ + ${CONDA_PREFIX}/bin/pip uninstall python3-openid; else - ${CONDA_PREFIX}/bin/pip install --upgrade python3-openid; + ${CONDA_PREFIX}/bin/pip install --upgrade --force-reinstall python3-openid && \ + ${CONDA_PREFIX}/bin/pip uninstall python-openid; fi + - echo "OpenID version: `${CONDA_PREFIX}/bin/python -c 'import openid; print(openid.__version__)'`" - make install install-dev - make version before_script: From 22fcbd5741e4bb1804b439e7d85d3c77fd2b8484 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Thu, 31 Jan 2019 19:24:18 -0500 Subject: [PATCH 56/76] travis travis travis --- .travis.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index e4b6a502d..78f8d81f0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -45,7 +45,7 @@ before_install: - cp -f ./ci/magpie.env ./env/magpie.env install: # TODO: remove when fixed and merged (see requirements.txt and Makefile) - - pip install gunicorn paste + - ${CONDA_PREFIX}/bin/pip install gunicorn paste - | if [ "${TRAVIS_PYTHON_VERSION}" -eq "2.7" ]; then ${CONDA_PREFIX}/bin/pip install --upgrade --force-reinstall python-openid && \ @@ -54,7 +54,6 @@ install: ${CONDA_PREFIX}/bin/pip install --upgrade --force-reinstall python3-openid && \ ${CONDA_PREFIX}/bin/pip uninstall python-openid; fi - - echo "OpenID version: `${CONDA_PREFIX}/bin/python -c 'import openid; print(openid.__version__)'`" - make install install-dev - make version before_script: From 3824bd1816c89dae1c7cc79f41c405775ee6beea Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Thu, 31 Jan 2019 19:30:03 -0500 Subject: [PATCH 57/76] yes do it travis --- .travis.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 78f8d81f0..008313e3f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -48,11 +48,11 @@ install: - ${CONDA_PREFIX}/bin/pip install gunicorn paste - | if [ "${TRAVIS_PYTHON_VERSION}" -eq "2.7" ]; then - ${CONDA_PREFIX}/bin/pip install --upgrade --force-reinstall python-openid && \ - ${CONDA_PREFIX}/bin/pip uninstall python3-openid; + ${CONDA_PREFIX}/bin/pip install -y --upgrade --force-reinstall python-openid && \ + ${CONDA_PREFIX}/bin/pip uninstall -y python3-openid; else - ${CONDA_PREFIX}/bin/pip install --upgrade --force-reinstall python3-openid && \ - ${CONDA_PREFIX}/bin/pip uninstall python-openid; + ${CONDA_PREFIX}/bin/pip install -y --upgrade --force-reinstall python3-openid && \ + ${CONDA_PREFIX}/bin/pip uninstall -y python-openid; fi - make install install-dev - make version From d3dd15aa6cd835510980fcd941a4ff47dceeb96b Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Thu, 31 Jan 2019 19:33:23 -0500 Subject: [PATCH 58/76] yes --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 008313e3f..ee4260a0a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -48,10 +48,10 @@ install: - ${CONDA_PREFIX}/bin/pip install gunicorn paste - | if [ "${TRAVIS_PYTHON_VERSION}" -eq "2.7" ]; then - ${CONDA_PREFIX}/bin/pip install -y --upgrade --force-reinstall python-openid && \ + ${CONDA_PREFIX}/bin/pip install --upgrade --force-reinstall python-openid && \ ${CONDA_PREFIX}/bin/pip uninstall -y python3-openid; else - ${CONDA_PREFIX}/bin/pip install -y --upgrade --force-reinstall python3-openid && \ + ${CONDA_PREFIX}/bin/pip install --upgrade --force-reinstall python3-openid && \ ${CONDA_PREFIX}/bin/pip uninstall -y python-openid; fi - make install install-dev From e28ea7e3477b391316b9b349b5f3d2b5c9f3930b Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Thu, 31 Jan 2019 19:37:48 -0500 Subject: [PATCH 59/76] openid reqs --- .travis.yml | 16 ++++++++-------- requirements.txt | 1 - 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index ee4260a0a..aaaaee1a1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -46,14 +46,14 @@ before_install: install: # TODO: remove when fixed and merged (see requirements.txt and Makefile) - ${CONDA_PREFIX}/bin/pip install gunicorn paste - - | - if [ "${TRAVIS_PYTHON_VERSION}" -eq "2.7" ]; then - ${CONDA_PREFIX}/bin/pip install --upgrade --force-reinstall python-openid && \ - ${CONDA_PREFIX}/bin/pip uninstall -y python3-openid; - else - ${CONDA_PREFIX}/bin/pip install --upgrade --force-reinstall python3-openid && \ - ${CONDA_PREFIX}/bin/pip uninstall -y python-openid; - fi + #- | + # if [ "${TRAVIS_PYTHON_VERSION}" -eq "2.7" ]; then + # ${CONDA_PREFIX}/bin/pip install --upgrade --force-reinstall python-openid && \ + # ${CONDA_PREFIX}/bin/pip uninstall -y python3-openid; + # else + # ${CONDA_PREFIX}/bin/pip install --upgrade --force-reinstall python3-openid && \ + # ${CONDA_PREFIX}/bin/pip uninstall -y python-openid; + # fi - make install install-dev - make version before_script: diff --git a/requirements.txt b/requirements.txt index f8ed624d5..98be8bd63 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,7 +8,6 @@ ziggurat-foundations==0.8.1 pyramid_tm==2.2.1 pyramid_chameleon==0.3 pyramid_mako>=1.0.2 -python-openid==2.2.5 requests psycopg2>=2.7.1 lxml>=3.7 From d32f8c8f88ee6e4cabc5514ae6409834fde918df Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Thu, 31 Jan 2019 19:50:04 -0500 Subject: [PATCH 60/76] more requirements --- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 98be8bd63..26f705585 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,9 +14,10 @@ lxml>=3.7 bcrypt==3.1.3 futures==3.1.1 zope.sqlalchemy==1.0 -gunicorn==19.8.1 alembic==0.9.6 +gunicorn paste +pastedeploy python-dotenv # api docs cornice_swagger>=0.7.0 From 621bdd7483ee5fdaca7f8287fa8af0ee266e9df8 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Thu, 31 Jan 2019 19:53:40 -0500 Subject: [PATCH 61/76] make start with env --- .travis.yml | 1 - Makefile | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index aaaaee1a1..2a0f4c5e2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -45,7 +45,6 @@ before_install: - cp -f ./ci/magpie.env ./env/magpie.env install: # TODO: remove when fixed and merged (see requirements.txt and Makefile) - - ${CONDA_PREFIX}/bin/pip install gunicorn paste #- | # if [ "${TRAVIS_PYTHON_VERSION}" -eq "2.7" ]; then # ${CONDA_PREFIX}/bin/pip install --upgrade --force-reinstall python-openid && \ diff --git a/Makefile b/Makefile index d6dd42b8d..5e960ea70 100644 --- a/Makefile +++ b/Makefile @@ -228,7 +228,8 @@ cron: .PHONY: start start: install @echo "Starting Magpie..." - exec gunicorn -b 0.0.0.0:2001 --paste "$(CUR_DIR)/magpie/magpie.ini" --workers 10 --preload + @bash -c 'source "$(CONDA_HOME)/bin/activate" "$(CONDA_ENV)"; \ + exec gunicorn -b 0.0.0.0:2001 --paste "$(CUR_DIR)/magpie/magpie.ini" --workers 10 --preload' .PHONY: version version: From d874e1de5c74ee12aef6600ab29da66748928fdb Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Thu, 31 Jan 2019 20:00:46 -0500 Subject: [PATCH 62/76] override them openid versions --- .travis.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2a0f4c5e2..ac3aafb79 100644 --- a/.travis.yml +++ b/.travis.yml @@ -45,14 +45,14 @@ before_install: - cp -f ./ci/magpie.env ./env/magpie.env install: # TODO: remove when fixed and merged (see requirements.txt and Makefile) - #- | - # if [ "${TRAVIS_PYTHON_VERSION}" -eq "2.7" ]; then - # ${CONDA_PREFIX}/bin/pip install --upgrade --force-reinstall python-openid && \ - # ${CONDA_PREFIX}/bin/pip uninstall -y python3-openid; - # else - # ${CONDA_PREFIX}/bin/pip install --upgrade --force-reinstall python3-openid && \ - # ${CONDA_PREFIX}/bin/pip uninstall -y python-openid; - # fi + - | + if [ "${TRAVIS_PYTHON_VERSION}" -eq "2.7" ]; then + ${CONDA_PREFIX}/bin/pip install --upgrade --force-reinstall python-openid && \ + ${CONDA_PREFIX}/bin/pip uninstall -y python3-openid; + else + ${CONDA_PREFIX}/bin/pip install --upgrade --force-reinstall python3-openid && \ + ${CONDA_PREFIX}/bin/pip uninstall -y python-openid; + fi - make install install-dev - make version before_script: From 01795c9fa529e93791b01edc37d7dedf027901ea Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Thu, 31 Jan 2019 20:28:39 -0500 Subject: [PATCH 63/76] run detached make start for tests to work --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 5e960ea70..a6216a276 100644 --- a/Makefile +++ b/Makefile @@ -229,7 +229,7 @@ cron: start: install @echo "Starting Magpie..." @bash -c 'source "$(CONDA_HOME)/bin/activate" "$(CONDA_ENV)"; \ - exec gunicorn -b 0.0.0.0:2001 --paste "$(CUR_DIR)/magpie/magpie.ini" --workers 10 --preload' + exec gunicorn -b 0.0.0.0:2001 --paste "$(CUR_DIR)/magpie/magpie.ini" --workers 10 --preload &' .PHONY: version version: From 318569040a91b1f06c1dedb5f5a27f65e2e06358 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Thu, 31 Jan 2019 20:57:55 -0500 Subject: [PATCH 64/76] travis test url + py3 support unicode & json decode handling --- ci/magpie.env | 1 + magpie/api/api_except.py | 16 +++++++++++++--- magpie/api/api_generic.py | 12 +++++++++--- 3 files changed, 23 insertions(+), 6 deletions(-) diff --git a/ci/magpie.env b/ci/magpie.env index b25c37787..e914f4ba2 100644 --- a/ci/magpie.env +++ b/ci/magpie.env @@ -6,3 +6,4 @@ MAGPIE_POSTGRES_PORT=5432 MAGPIE_POSTGRES_DB=magpie PHOENIX_PUSH=false HOSTNAME=localhost +MAGPIE_TEST_REMOTE_SERVER_URL=http://localhost:2001 diff --git a/magpie/api/api_except.py b/magpie/api/api_except.py index 3836e490f..7b8800a73 100644 --- a/magpie/api/api_except.py +++ b/magpie/api/api_except.py @@ -1,6 +1,7 @@ from magpie.common import islambda, isclass from pyramid.httpexceptions import * from sys import exc_info +import six # control variables to avoid infinite recursion in case of # major programming error to avoid application hanging @@ -8,6 +9,7 @@ RAISE_RECURSIVE_SAFEGUARD_COUNT = 0 +# noinspection PyPep8Naming def verify_param(param, paramCompare=None, httpError=HTTPNotAcceptable, httpKWArgs=None, msgOnFail="", content=None, contentType='application/json', notNone=False, notEmpty=False, notIn=False, notEqual=False, isTrue=False, isFalse=False, @@ -116,6 +118,7 @@ def verify_param(param, paramCompare=None, httpError=HTTPNotAcceptable, httpKWAr raise_http(httpError, httpKWArgs=httpKWArgs, detail=msgOnFail, content=content, contentType=contentType) +# noinspection PyPep8Naming def evaluate_call(call, fallback=None, httpError=HTTPInternalServerError, httpKWArgs=None, msgOnFail="", content=None, contentType='application/json'): """ @@ -182,6 +185,7 @@ def evaluate_call(call, fallback=None, httpError=HTTPInternalServerError, httpKW contentType=contentType) +# noinspection PyPep8Naming def valid_http(httpSuccess=HTTPOk, httpKWArgs=None, detail="", content=None, contentType='application/json'): """ Returns successful HTTP with standardized information formatted with content type. @@ -206,6 +210,7 @@ def valid_http(httpSuccess=HTTPOk, httpKWArgs=None, detail="", content=None, con return resp +# noinspection PyPep8Naming def raise_http(httpError=HTTPInternalServerError, httpKWArgs=None, detail="", content=None, contentType='application/json', nothrow=False): """ @@ -242,10 +247,12 @@ def raise_http(httpError=HTTPInternalServerError, httpKWArgs=None, # reset counter for future calls (don't accumulate for different requests) # following raise is the last in the chain since it wasn't triggered by other functions RAISE_RECURSIVE_SAFEGUARD_COUNT = 0 - if nothrow: return resp + if nothrow: + return resp raise resp +# noinspection PyPep8Naming def validate_params(httpClass, httpBase, detail, content, contentType): """ Validates parameter types and formats required by `valid_http` and `raise_http`. @@ -262,7 +269,7 @@ def validate_params(httpClass, httpBase, detail, content, contentType): # verify input arguments, raise `HTTPInternalServerError` with caller info if invalid # cannot be done within a try/except because it would always trigger with `raise_http` content = dict() if content is None else content - detail = repr(detail) if type(detail) not in [str, unicode] else detail + detail = repr(detail) if not isinstance(detail, six.string_types) else detail if not isclass(httpClass): raise_http(httpError=HTTPInternalServerError, detail="Object specified is not of type `HTTPError`", @@ -275,8 +282,9 @@ def validate_params(httpClass, httpBase, detail, content, contentType): # if it derives from `HTTPException`, it *could* be different than base (ex: 2xx instead of 4xx codes) # return 'unknown error' (520) if not of lowest level base `HTTPException`, otherwise use the available code httpBase = tuple(httpBase if hasattr(httpBase, '__iter__') else [httpBase]) + # noinspection PyUnresolvedReferences httpCode = httpClass.code if issubclass(httpClass, httpBase) else \ - httpClass.code if issubclass(httpClass, HTTPException) else 520 + httpClass.code if issubclass(httpClass, HTTPException) else 520 # noqa: F401 if not issubclass(httpClass, httpBase): raise_http(httpError=HTTPInternalServerError, detail="Invalid `httpBase` derived class specified", @@ -296,6 +304,7 @@ def validate_params(httpClass, httpBase, detail, content, contentType): return httpCode, detail, content +# noinspection PyPep8Naming def format_content_json_str(httpCode, detail, content, contentType): """ Inserts the code, details, content and type within the body using json format. @@ -324,6 +333,7 @@ def format_content_json_str(httpCode, detail, content, contentType): return json_body +# noinspection PyPep8Naming def generate_response_http_format(httpClass, httpKWArgs, jsonContent, outputType='text/plain'): """ Formats the HTTP response output according to desired `outputType` using provided HTTP code and content. diff --git a/magpie/api/api_generic.py b/magpie/api/api_generic.py index b21466b5b..92d0ac644 100644 --- a/magpie/api/api_generic.py +++ b/magpie/api/api_generic.py @@ -1,6 +1,7 @@ from magpie.definitions.pyramid_definitions import HTTPUnauthorized, HTTPNotFound, HTTPInternalServerError from magpie.api.api_except import raise_http, HTTPServerError from magpie.api import api_rest_schemas as s +from simplejson import JSONDecodeError # @notfound_view_config() @@ -31,9 +32,14 @@ def get_request_info(request, default_msg="undefined"): content = {u'route_name': str(request.upath_info), u'request_url': str(request.url), u'detail': default_msg, u'method': request.method} if hasattr(request, 'exception'): - if hasattr(request.exception, 'json'): - if type(request.exception.json) is dict: - content.update(request.exception.json) + # handle error raised simply by checking for 'json' property in python 3 when body is invalid + has_json = False + try: + has_json = hasattr(request.exception, 'json') + except JSONDecodeError: + pass + if has_json and isinstance(request.exception.json, dict): + content.update(request.exception.json) elif isinstance(request.exception, HTTPServerError) and hasattr(request.exception, 'message'): content.update({u'exception': str(request.exception.message)}) elif hasattr(request, 'matchdict'): From 62e4d74f5b574beda23d09af1717009c89365d54 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Fri, 1 Feb 2019 12:58:41 -0500 Subject: [PATCH 65/76] test only instances with setupclass --- ci/magpie.env | 2 ++ magpie/adapter/__init__.py | 10 ++++------ magpie/adapter/magpieowssecurity.py | 17 +++++++++++++++-- magpie/adapter/magpieservice.py | 2 +- tests/interfaces.py | 21 ++++++++++----------- tests/test_magpie_adapter.py | 9 --------- tests/test_magpie_api.py | 24 ++++++++++++++++++------ tests/test_magpie_ui.py | 3 ++- 8 files changed, 52 insertions(+), 36 deletions(-) delete mode 100644 tests/test_magpie_adapter.py diff --git a/ci/magpie.env b/ci/magpie.env index e914f4ba2..c864e7305 100644 --- a/ci/magpie.env +++ b/ci/magpie.env @@ -7,3 +7,5 @@ MAGPIE_POSTGRES_DB=magpie PHOENIX_PUSH=false HOSTNAME=localhost MAGPIE_TEST_REMOTE_SERVER_URL=http://localhost:2001 +MAGPIE_TEST_ADMIN_USERNAME=admin +MAGPIE_TEST_ADMIN_PASSWORD=qwerty diff --git a/magpie/adapter/__init__.py b/magpie/adapter/__init__.py index eeb0ae50b..33674067f 100644 --- a/magpie/adapter/__init__.py +++ b/magpie/adapter/__init__.py @@ -1,17 +1,15 @@ -from magpie.definitions.pyramid_definitions import * -from magpie.definitions.ziggurat_definitions import * -from magpie.definitions.twitcher_definitions import * -from magpie.adapter.magpieowssecurity import * +from magpie.definitions.twitcher_definitions import DefaultAdapter, AdapterInterface, owsproxy +from magpie.adapter.magpieowssecurity import MagpieOWSSecurity from magpie.adapter.magpieservice import MagpieServiceStore from magpie.models import get_user from magpie.security import auth_config_from_settings -from magpie.db import * +from magpie.db import get_session_factory, get_tm_session, get_engine from magpie import __meta__ import logging LOGGER = logging.getLogger("TWITCHER") -# noinspection PyAbstractClass +# noinspection PyAbstractClass, PyMethodMayBeStatic class MagpieAdapter(AdapterInterface): def describe_adapter(self): return {"name": self.__class__.__name__, "version": __meta__.__version__} diff --git a/magpie/adapter/magpieowssecurity.py b/magpie/adapter/magpieowssecurity.py index 9701de516..f3c235437 100644 --- a/magpie/adapter/magpieowssecurity.py +++ b/magpie/adapter/magpieowssecurity.py @@ -1,7 +1,20 @@ from magpie.api.api_except import evaluate_call, verify_param from magpie.constants import get_constant -from magpie.definitions.pyramid_definitions import * -from magpie.definitions.twitcher_definitions import * +from magpie.definitions.pyramid_definitions import ( + HTTPOk, + HTTPNotFound, + HTTPForbidden, + IAuthenticationPolicy, + IAuthorizationPolicy, + asbool, +) +from magpie.definitions.twitcher_definitions import ( + OWSSecurityInterface, + OWSAccessForbidden, + parse_service_name, + get_twitcher_configuration, + TWITCHER_CONFIGURATION_DEFAULT, +) from magpie.models import Service from magpie.services import service_factory from magpie.utils import get_magpie_url diff --git a/magpie/adapter/magpieservice.py b/magpie/adapter/magpieservice.py index a14e4c3e8..7332f45ab 100644 --- a/magpie/adapter/magpieservice.py +++ b/magpie/adapter/magpieservice.py @@ -2,7 +2,7 @@ Store adapters to read data from magpie. """ -from magpie.definitions.twitcher_definitions import * +from magpie.definitions.twitcher_definitions import ServiceStore, Service, ServiceNotFound from magpie.definitions.pyramid_definitions import HTTPOk, asbool from magpie.utils import get_admin_cookies, get_magpie_url import requests diff --git a/tests/interfaces.py b/tests/interfaces.py index b1468fa82..3605c0b52 100644 --- a/tests/interfaces.py +++ b/tests/interfaces.py @@ -21,6 +21,8 @@ class TestMagpieAPI_NoAuth_Interface(unittest.TestCase): Derived classes must implement ``setUpClass`` accordingly to generate the Magpie test application. """ + __test__ = False + @classmethod def setUpClass(cls): raise NotImplementedError @@ -77,6 +79,8 @@ class TestMagpieAPI_UsersAuth_Interface(unittest.TestCase): Derived classes must implement ``setUpClass`` accordingly to generate the Magpie test application. """ + __test__ = False + @classmethod def setUpClass(cls): raise NotImplementedError @@ -96,6 +100,8 @@ class TestMagpieAPI_AdminAuth_Interface(unittest.TestCase): Derived classes must implement ``setUpClass`` accordingly to generate the Magpie test application. """ + __test__ = False + @classmethod def setUpClass(cls): raise NotImplementedError @@ -591,11 +597,6 @@ def test_PutUsers_password(self): utils.check_or_try_logout_user(self.url) # validate that the new password is effective - #data = {'user_name': self.test_user_name, 'password': new_password} - #resp = utils.test_request(self.url, 'POST', '/signin', data=data, - # headers=self.json_headers, allow_redirects=True) - #utils.check_response_basic_info(resp, 200, expected_method='POST') - headers, cookies = utils.check_or_try_login_user(self.url, username=self.test_user_name, password=new_password, use_ui_form_submit=True, version=self.version) resp = utils.test_request(self.url, 'GET', '/session', headers=headers, cookies=cookies) @@ -605,12 +606,6 @@ def test_PutUsers_password(self): utils.check_or_try_logout_user(self.url) # validate that previous password is ineffective - #data = {'user_name': self.test_user_name, 'password': old_password} - #resp = utils.test_request(self.url, 'POST', '/signin', headers=self.json_headers, data=data, expect_errors=True) - #if LooseVersion(self.version) >= LooseVersion('0.7.8'): - # utils.check_response_basic_info(resp, 401, expected_method='POST') - #else: - # utils.check_response_basic_info(resp, 400, expected_method='POST') headers, cookies = utils.check_or_try_login_user(self.url, username=self.test_user_name, password=old_password, use_ui_form_submit=True, version=self.version) resp = utils.test_request(self.url, 'GET', '/session', headers=headers, cookies=cookies) @@ -1079,6 +1074,8 @@ class TestMagpieUI_NoAuth_Interface(unittest.TestCase): Derived classes must implement ``setUpClass`` accordingly to generate the Magpie test application. """ + __test__ = False + @classmethod def setUpClass(cls): raise NotImplementedError @@ -1168,6 +1165,8 @@ class TestMagpieUI_AdminAuth_Interface(unittest.TestCase): Derived classes must implement ``setUpClass`` accordingly to generate the Magpie test application. """ + __test__ = False + @classmethod def setUpClass(cls): raise NotImplementedError diff --git a/tests/test_magpie_adapter.py b/tests/test_magpie_adapter.py deleted file mode 100644 index d44142176..000000000 --- a/tests/test_magpie_adapter.py +++ /dev/null @@ -1,9 +0,0 @@ -import pytest -import unittest - - -@pytest.mark.skip("not implemented") -@unittest.skip("not implemented") -class TestAdapter(unittest.TestCase): - def test_adapter(self): - pass diff --git a/tests/test_magpie_api.py b/tests/test_magpie_api.py index 5f13e8141..ef656874c 100644 --- a/tests/test_magpie_api.py +++ b/tests/test_magpie_api.py @@ -22,12 +22,14 @@ @pytest.mark.local @unittest.skipUnless(runner.MAGPIE_TEST_API, reason=runner.MAGPIE_TEST_DISABLED_MESSAGE('api')) @unittest.skipUnless(runner.MAGPIE_TEST_LOCAL, reason=runner.MAGPIE_TEST_DISABLED_MESSAGE('local')) -class TestMagpieAPI_NoAuth_Local(ti.TestMagpieAPI_NoAuth_Interface): +class TestMagpieAPI_NoAuth_Local(ti.TestMagpieAPI_NoAuth_Interface, unittest.TestCase): """ Test any operation that do not require user AuthN/AuthZ. Use a local Magpie test application. """ + __test__ = True + @classmethod def setUpClass(cls): cls.app = utils.get_test_magpie_app() @@ -50,6 +52,8 @@ class TestMagpieAPI_UsersAuth_Local(ti.TestMagpieAPI_UsersAuth_Interface): Use a local Magpie test application. """ + __test__ = True + @classmethod def setUpClass(cls): cls.app = utils.get_test_magpie_app() @@ -65,6 +69,8 @@ class TestMagpieAPI_AdminAuth_Local(ti.TestMagpieAPI_AdminAuth_Interface): Use a local Magpie test application. """ + __test__ = True + @classmethod def setUpClass(cls): cls.app = utils.get_test_magpie_app() @@ -80,7 +86,7 @@ def setUpClass(cls): # NOTE: localhost magpie has to be running for following login call to work cls.headers, cls.cookies = utils.check_or_try_login_user(cls.app, cls.usr, cls.pwd, use_ui_form_submit=True, version=cls.version) - cls.require = "cannot run tests without logged in '{}' user".format(cls.grp) + cls.require = "cannot run tests without logged in user with '{}' permissions".format(cls.grp) cls.check_requirements() cls.setup_test_values() @@ -89,12 +95,14 @@ def setUpClass(cls): @pytest.mark.remote @unittest.skipUnless(runner.MAGPIE_TEST_API, reason=runner.MAGPIE_TEST_DISABLED_MESSAGE('api')) @unittest.skipUnless(runner.MAGPIE_TEST_REMOTE, reason=runner.MAGPIE_TEST_DISABLED_MESSAGE('remote')) -class TestMagpieAPI_NoAuth_Remote(ti.TestMagpieAPI_NoAuth_Interface): +class TestMagpieAPI_NoAuth_Remote(ti.TestMagpieAPI_NoAuth_Interface, unittest.TestCase): """ Test any operation that do not require user AuthN/AuthZ. Use an already running remote bird server. """ + __test__ = True + @classmethod def setUpClass(cls): cls.url = get_constant('MAGPIE_TEST_REMOTE_SERVER_URL') @@ -110,12 +118,14 @@ def setUpClass(cls): @pytest.mark.remote @unittest.skipUnless(runner.MAGPIE_TEST_API, reason=runner.MAGPIE_TEST_DISABLED_MESSAGE('api')) @unittest.skipUnless(runner.MAGPIE_TEST_LOCAL, reason=runner.MAGPIE_TEST_DISABLED_MESSAGE('remote')) -class TestMagpieAPI_UsersAuth_Remote(ti.TestMagpieAPI_UsersAuth_Interface): +class TestMagpieAPI_UsersAuth_Remote(ti.TestMagpieAPI_UsersAuth_Interface, unittest.TestCase): """ Test any operation that require at least 'Users' group AuthN/AuthZ. Use an already running remote bird server. """ + __test__ = True + @classmethod def setUpClass(cls): cls.app = utils.get_test_magpie_app() @@ -125,12 +135,14 @@ def setUpClass(cls): @pytest.mark.remote @unittest.skipUnless(runner.MAGPIE_TEST_API, reason=runner.MAGPIE_TEST_DISABLED_MESSAGE('api')) @unittest.skipUnless(runner.MAGPIE_TEST_REMOTE, reason=runner.MAGPIE_TEST_DISABLED_MESSAGE('remote')) -class TestMagpieAPI_AdminAuth_Remote(ti.TestMagpieAPI_AdminAuth_Interface): +class TestMagpieAPI_AdminAuth_Remote(ti.TestMagpieAPI_AdminAuth_Interface, unittest.TestCase): """ Test any operation that require at least 'Administrator' group AuthN/AuthZ. Use an already running remote bird server. """ + __test__ = True + @classmethod def setUpClass(cls): cls.grp = get_constant('MAGPIE_ADMIN_GROUP') @@ -138,7 +150,7 @@ def setUpClass(cls): cls.pwd = get_constant('MAGPIE_TEST_ADMIN_PASSWORD') cls.url = get_constant('MAGPIE_TEST_REMOTE_SERVER_URL') cls.headers, cls.cookies = utils.check_or_try_login_user(cls.url, cls.usr, cls.pwd) - cls.require = "cannot run tests without logged in '{}' user".format(cls.grp) + cls.require = "cannot run tests without logged in user with '{}' permissions".format(cls.grp) cls.json_headers = utils.get_headers(cls.url, {'Accept': 'application/json', 'Content-Type': 'application/json'}) cls.version = utils.TestSetup.get_Version(cls) diff --git a/tests/test_magpie_ui.py b/tests/test_magpie_ui.py index 37ed1d881..9c561829a 100644 --- a/tests/test_magpie_ui.py +++ b/tests/test_magpie_ui.py @@ -56,6 +56,7 @@ class TestMagpieUI_AdminAuth_Local(ti.TestMagpieUI_AdminAuth_Interface): @classmethod def setUpClass(cls): + cls.grp = get_constant('MAGPIE_ADMIN_GROUP') cls.usr = get_constant('MAGPIE_TEST_ADMIN_USERNAME') cls.pwd = get_constant('MAGPIE_TEST_ADMIN_PASSWORD') cls.app = utils.get_test_magpie_app() @@ -67,7 +68,7 @@ def setUpClass(cls): # TODO: fix UI views so that they can be 'found' directly in the WebTest.TestApp # NOTE: localhost magpie has to be running for following login call to work cls.headers, cls.cookies = utils.check_or_try_login_user(cls.url, cls.usr, cls.pwd, use_ui_form_submit=True) - cls.require = "cannot run tests without logged in '{}' user".format(get_constant('MAGPIE_ADMIN_GROUP')) + cls.require = "cannot run tests without logged in user with '{}' permissions".format(cls.grp) cls.check_requirements() cls.test_user = get_constant('MAGPIE_ANONYMOUS_USER') From 88f3c3a87c96e85ab241668bd23b8b8e4b387be2 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Fri, 1 Feb 2019 15:21:10 -0500 Subject: [PATCH 66/76] fixes for py3 and UI --- HISTORY.rst | 1 + magpie/api/api_rest_schemas.py | 2 +- .../management/resource/resource_formats.py | 9 +- .../api/management/resource/resource_utils.py | 9 +- magpie/api/management/service/__init__.py | 2 +- .../api/management/service/service_formats.py | 4 +- .../api/management/service/service_utils.py | 4 +- .../api/management/service/service_views.py | 4 +- magpie/ui/login/views.py | 14 +- magpie/ui/management/__init__.py | 29 +-- magpie/ui/management/views.py | 245 ++++++++++-------- magpie/ui/utils.py | 7 + 12 files changed, 173 insertions(+), 157 deletions(-) create mode 100644 magpie/ui/utils.py diff --git a/HISTORY.rst b/HISTORY.rst index 4bf1e3a21..9351fffac 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -11,6 +11,7 @@ History * attempt db creation on first migration if not existing * add continuous integration testing and deployment * reduce excessive sqlalchemy logging using `MAGPIE_LOG_LEVEL >= INFO` +* use schema API route definitions for UI calls 0.8.x --------------------- diff --git a/magpie/api/api_rest_schemas.py b/magpie/api/api_rest_schemas.py index a3a08b006..5f18de3c7 100644 --- a/magpie/api/api_rest_schemas.py +++ b/magpie/api/api_rest_schemas.py @@ -212,7 +212,7 @@ def service_api_route_info(service_api): ServiceAPI = Service( path='/services/{service_name}', name='Service') -ServiceTypesAPI = Service( +ServiceTypeAPI = Service( path='/services/types/{service_type}', name='ServiceTypes') ServicePermissionsAPI = Service( diff --git a/magpie/api/management/resource/resource_formats.py b/magpie/api/management/resource/resource_formats.py index 6a2ab252a..4a5595d20 100644 --- a/magpie/api/management/resource/resource_formats.py +++ b/magpie/api/management/resource/resource_formats.py @@ -7,14 +7,9 @@ def format_resource(resource, permissions=None, basic_info=False): def fmt_res(res, perms, info): - resource_name = str(res.resource_name) - resource_display_name = resource_name - if res.resource_display_name: - resource_display_name = res.resource_display_name.encode('utf-8') - result = { - u'resource_name': resource_name, - u'resource_display_name': resource_display_name, + u'resource_name': str(res.resource_name), + u'resource_display_name': str(res.resource_display_name or res.resource_name), u'resource_type': str(res.resource_type), u'resource_id': res.resource_id } diff --git a/magpie/api/management/resource/resource_utils.py b/magpie/api/management/resource/resource_utils.py index efd6856d9..5aefcccde 100644 --- a/magpie/api/management/resource/resource_utils.py +++ b/magpie/api/management/resource/resource_utils.py @@ -59,15 +59,14 @@ def check_valid_service_resource(parent_resource, resource_type, db_session): def crop_tree_with_permission(children, resource_id_list): - for child_id, child_dict in children.items(): + for child_id, child_dict in list(children.items()): new_children = child_dict[u'children'] children_returned, resource_id_list = crop_tree_with_permission(new_children, resource_id_list) - is_in_resource_id_list = child_id in resource_id_list - if not is_in_resource_id_list and not children_returned: + if child_id not in resource_id_list and not children_returned: children.pop(child_id) - elif is_in_resource_id_list: + elif child_id in resource_id_list: resource_id_list.remove(child_id) - return children, resource_id_list + return dict(children), list(resource_id_list) def get_resource_path(resource_id, db_session): diff --git a/magpie/api/management/service/__init__.py b/magpie/api/management/service/__init__.py index e459959b7..826a8bf36 100644 --- a/magpie/api/management/service/__init__.py +++ b/magpie/api/management/service/__init__.py @@ -9,7 +9,7 @@ def includeme(config): # Add all the rest api routes config.add_route(**service_api_route_info(ServicesAPI)) config.add_route(**service_api_route_info(ServiceAPI)) - config.add_route(**service_api_route_info(ServiceTypesAPI)) + config.add_route(**service_api_route_info(ServiceTypeAPI)) config.add_route(**service_api_route_info(ServicePermissionsAPI)) config.add_route(**service_api_route_info(ServiceResourcesAPI)) config.add_route(**service_api_route_info(ServiceResourceAPI)) diff --git a/magpie/api/management/service/service_formats.py b/magpie/api/management/service/service_formats.py index 10688ca79..f485a573f 100644 --- a/magpie/api/management/service/service_formats.py +++ b/magpie/api/management/service/service_formats.py @@ -12,7 +12,7 @@ def fmt_svc(svc, perms): u'public_url': str(get_twitcher_protected_service_url(svc.resource_name)), u'service_name': str(svc.resource_name), u'service_type': str(svc.type), - u'service_sync_type': svc.sync_type, + u'service_sync_type': str(svc.sync_type), u'resource_id': svc.resource_id, u'permission_names': sorted(service_type_dict[svc.type].permission_names if perms is None else perms) } @@ -33,7 +33,7 @@ def format_service_resources(service, db_session, service_perms=None, def fmt_svc_res(svc, db, svc_perms, res_perms, show_all): tree = get_resource_children(svc, db) if not show_all: - tree, resource_id_list_remain = crop_tree_with_permission(tree, res_perms.keys()) + tree, resource_id_list_remain = crop_tree_with_permission(tree, list(res_perms.keys())) svc_perms = service_type_dict[svc.type].permission_names if svc_perms is None else svc_perms svc_res = format_service(svc, svc_perms, show_private_url=show_private_url) diff --git a/magpie/api/management/service/service_utils.py b/magpie/api/management/service/service_utils.py index 86a14d63f..e4e3b6aa1 100644 --- a/magpie/api/management/service/service_utils.py +++ b/magpie/api/management/service/service_utils.py @@ -57,12 +57,12 @@ def get_services_by_type(service_type, db_session): ax.verify_param(service_type, notNone=True, notEmpty=True, httpError=HTTPNotAcceptable, msgOnFail="Invalid `service_type` value '" + str(service_type) + "' specified") services = db_session.query(models.Service).filter(models.Service.type == service_type) - return sorted(services) + return sorted(services, key=lambda svc: svc.resource_name) def add_service_getcapabilities_perms(service, db_session, group_name=None): if service.type in SERVICES_PHOENIX_ALLOWED \ - and 'getcapabilities' in service_type_dict[service.type].permission_names: + and 'getcapabilities' in service_type_dict[service.type].permission_names: # noqa: F401 if group_name is None: group_name = get_constant('MAGPIE_ANONYMOUS_USER') group = GroupService.by_group_name(group_name, db_session=db_session) diff --git a/magpie/api/management/service/service_views.py b/magpie/api/management/service/service_views.py index 24f93d36a..1be910f5a 100644 --- a/magpie/api/management/service/service_views.py +++ b/magpie/api/management/service/service_views.py @@ -9,8 +9,8 @@ from magpie import models -@ServiceTypesAPI.get(tags=[ServicesTag], response_schemas=ServiceTypes_GET_responses) -@view_config(route_name=ServiceTypesAPI.name, request_method='GET') +@ServiceTypeAPI.get(tags=[ServicesTag], response_schemas=ServiceTypes_GET_responses) +@view_config(route_name=ServiceTypeAPI.name, request_method='GET') def get_services_by_type_view(request): """List all registered services from a specific type.""" return get_services_runner(request) diff --git a/magpie/ui/login/views.py b/magpie/ui/login/views.py index a632730a4..85ae1d967 100755 --- a/magpie/ui/login/views.py +++ b/magpie/ui/login/views.py @@ -1,7 +1,7 @@ from magpie.definitions.pyramid_definitions import * -from magpie.ui.management import check_response +from magpie.ui.utils import check_response from magpie.ui.home import add_template_data -from magpie.api.api_rest_schemas import SigninAPI +from magpie.api import api_rest_schemas as schemas import requests @@ -11,12 +11,12 @@ def __init__(self, request): self.magpie_url = self.request.registry.settings['magpie.url'] def get_internal_providers(self): - resp = requests.get(self.magpie_url + '/providers') + resp = requests.get('{}{}'.format(self.magpie_url, schemas.ProvidersAPI.path)) check_response(resp) return resp.json()['providers']['internal'] def get_external_providers(self): - resp = requests.get(self.magpie_url + '/providers') + resp = requests.get('{}{}'.format(self.magpie_url, schemas.ProvidersAPI.path)) check_response(resp) return resp.json()['providers']['external'] @@ -33,7 +33,7 @@ def login(self): try: if 'submit' in self.request.POST: - signin_url = '{}/signin'.format(self.magpie_url) + signin_url = '{}{}'.format(self.magpie_url, schemas.SigninAPI.path) data_to_send = {} for key in self.request.POST: data_to_send[key] = self.request.POST.get(key) @@ -49,7 +49,7 @@ def login(self): pyr_res = Response(body=response.content, headers=response.headers) for cookie in response.cookies: pyr_res.set_cookie(name=cookie.name, value=cookie.value, overwrite=True) - is_external = response.url != '{}{}'.format(self.magpie_url, SigninAPI.path) + is_external = response.url != '{}{}'.format(self.magpie_url, schemas.SigninAPI.path) if is_external: return HTTPFound(response.url, headers=pyr_res.headers) return HTTPFound(location=self.request.route_url('home'), headers=pyr_res.headers) @@ -66,5 +66,5 @@ def login(self): @view_config(route_name='logout', renderer='templates/login.mako', permission=NO_PERMISSION_REQUIRED) def logout(self): # Flush cookies and return to home - requests.get('{url}/signout'.format(url=self.magpie_url)) + requests.get('{url}{path}'.format(url=self.magpie_url, path=schemas.SignoutAPI.path)) return HTTPFound(location=self.request.route_url('home'), headers=forget(self.request)) diff --git a/magpie/ui/management/__init__.py b/magpie/ui/management/__init__.py index b82893c29..63ea95e68 100644 --- a/magpie/ui/management/__init__.py +++ b/magpie/ui/management/__init__.py @@ -1,25 +1,18 @@ -from pyramid.httpexceptions import exception_response - import logging logger = logging.getLogger(__name__) -def check_response(response): - if response.status_code >= 400: - raise exception_response(response.status_code, body=response.text) - return response - - def includeme(config): + from magpie.ui.management.views import ManagementViews logger.info('Adding management ...') - config.add_route('view_groups', '/ui/groups') - config.add_route('add_group', '/ui/groups/add') - config.add_route('edit_group', '/ui/groups/{group_name}/{cur_svc_type}') - config.add_route('view_users', '/ui/users') - config.add_route('add_user', '/ui/users/add') - config.add_route('edit_user', '/ui/users/{user_name}/{cur_svc_type}') - config.add_route('view_services', '/ui/services/{cur_svc_type}') - config.add_route('add_service', '/ui/services/{cur_svc_type}/add') - config.add_route('edit_service', '/ui/services/{cur_svc_type}/{service_name}') - config.add_route('add_resource', '/ui/services/{cur_svc_type}/{service_name}/add/{resource_id}') + config.add_route(ManagementViews.view_groups.__name__, '/ui/groups') + config.add_route(ManagementViews.add_group.__name__, '/ui/groups/add') + config.add_route(ManagementViews.edit_group.__name__, '/ui/groups/{group_name}/{cur_svc_type}') + config.add_route(ManagementViews.view_users.__name__, '/ui/users') + config.add_route(ManagementViews.add_user.__name__, '/ui/users/add') + config.add_route(ManagementViews.edit_user.__name__, '/ui/users/{user_name}/{cur_svc_type}') + config.add_route(ManagementViews.view_services.__name__, '/ui/services/{cur_svc_type}') + config.add_route(ManagementViews.add_service.__name__, '/ui/services/{cur_svc_type}/add') + config.add_route(ManagementViews.edit_service.__name__, '/ui/services/{cur_svc_type}/{service_name}') + config.add_route(ManagementViews.add_resource.__name__, '/ui/services/{cur_svc_type}/{service_name}/add/{resource_id}') config.scan() diff --git a/magpie/ui/management/views.py b/magpie/ui/management/views.py index c1bebc4f5..9e652fec1 100644 --- a/magpie/ui/management/views.py +++ b/magpie/ui/management/views.py @@ -3,13 +3,14 @@ import humanize +from magpie.api import api_rest_schemas as schemas from magpie.definitions.pyramid_definitions import * from magpie.constants import get_constant from magpie.common import str2bool from magpie.helpers.sync_resources import OUT_OF_SYNC from magpie.services import service_type_dict from magpie.models import resource_type_dict, remote_resource_tree_service -from magpie.ui.management import check_response +from magpie.ui.utils import check_response from magpie.helpers import sync_resources from magpie.ui.home import add_template_data from magpie import register, __meta__ @@ -24,14 +25,17 @@ def __init__(self, request): self.magpie_url = get_constant('magpie.url', settings=self.request.registry.settings, raise_missing=True, raise_not_set=True) + def get_url(self, path): + return '{url}{path}'.format(url=self.magpie_url, path=path) + def get_all_groups(self, first_default_group=None): try: - resp_groups = requests.get('{url}/groups'.format(url=self.magpie_url), cookies=self.request.cookies) + resp = requests.get(self.get_url(schemas.GroupsAPI.path), cookies=self.request.cookies) except Exception: raise HTTPBadRequest(detail='Bad Json response') - check_response(resp_groups) + check_response(resp) try: - groups = list(resp_groups.json()['group_names']) + groups = list(resp.json()['group_names']) if type(first_default_group) is str and first_default_group in groups: groups.remove(first_default_group) groups.insert(0, first_default_group) @@ -41,26 +45,27 @@ def get_all_groups(self, first_default_group=None): def get_group_users(self, group_name): try: - resp_group_users = requests.get('{url}/groups/{grp}/users'.format(url=self.magpie_url, grp=group_name), - cookies=self.request.cookies) - check_response(resp_group_users) - return resp_group_users.json()['user_names'] + path = schemas.GroupAPI.path.format(group_name=group_name) + resp = requests.get(self.get_url(path), cookies=self.request.cookies) + check_response(resp) + return resp.json()['user_names'] except Exception as e: raise HTTPBadRequest(detail=e.message) def get_user_groups(self, user_name): try: - resp_user_groups = requests.get('{url}/users/{usr}/groups'.format(url=self.magpie_url, usr=user_name), - cookies=self.request.cookies) - check_response(resp_user_groups) - return resp_user_groups.json()['group_names'] + path = schemas.UserGroupsAPI.path.format(user_name=user_name) + resp = requests.get(self.get_url(path), cookies=self.request.cookies) + check_response(resp) + return resp.json()['group_names'] except Exception as e: raise HTTPBadRequest(detail=e.message) def get_user_names(self): - resp_users = requests.get('{url}/users'.format(url=self.magpie_url), cookies=self.request.cookies) try: - return resp_users.json()['user_names'] + resp = requests.get(self.get_url(schemas.UsersAPI.path), cookies=self.request.cookies) + check_response(resp) + return resp.json()['user_names'] except Exception as e: raise HTTPBadRequest(detail=e.message) @@ -69,13 +74,13 @@ def get_user_emails(self): try: emails = list() for user in user_names: - resp_user = requests.get('{url}/users/{usr}'.format(url=self.magpie_url, usr=user), - cookies=self.request.cookies) - check_response(resp_user) + path = schemas.UserAPI.path.format(user_name=user) + resp = requests.get(self.get_url(path), cookies=self.request.cookies) + check_response(resp) if LooseVersion(__meta__.__version__) >= LooseVersion('0.6.3'): - user_email = resp_user.json()['user']['email'] + user_email = resp.json()['user']['email'] else: - user_email = resp_user.json()['email'] + user_email = resp.json()['email'] emails.append(user_email) return emails except Exception as e: @@ -86,18 +91,18 @@ def get_resource_types(self): :return: dictionary of all resources as {id: 'resource_type'} :rtype: dict """ - resp_resources = requests.get('{url}/resources'.format(url=self.magpie_url), cookies=self.request.cookies) - check_response(resp_resources) - res_dic = self.default_get(resp_resources.json(), 'resources', dict()) + resp = requests.get(self.get_url(schemas.ResourcesAPI.path), cookies=self.request.cookies) + check_response(resp) + res_dic = self.default_get(resp.json(), 'resources', dict()) res_ids = dict() self.flatten_tree_resource(res_dic, res_ids) return res_ids def get_services(self, cur_svc_type): try: - resp_services = requests.get('{url}/services'.format(url=self.magpie_url), cookies=self.request.cookies) - check_response(resp_services) - all_services = resp_services.json()['services'] + resp = requests.get(self.get_url(schemas.ServicesAPI.path), cookies=self.request.cookies) + check_response(resp) + all_services = resp.json()['services'] svc_types = sorted(all_services.keys()) if cur_svc_type not in svc_types: cur_svc_type = svc_types[0] @@ -108,10 +113,10 @@ def get_services(self, cur_svc_type): def get_service_data(self, service_name): try: - svc_res = requests.get('{url}/services/{svc}'.format(url=self.magpie_url, svc=service_name), - cookies=self.request.cookies) - check_response(svc_res) - return svc_res.json()[service_name] + path = schemas.ServiceAPI.path.format(service_name=service_name) + resp = requests.get(self.get_url(path), cookies=self.request.cookies) + check_response(resp) + return resp.json()[service_name] except Exception as e: raise HTTPBadRequest(detail=e.message) @@ -122,9 +127,9 @@ def update_service_name(self, old_service_name, new_service_name, service_push): svc_data['resource_name'] = new_service_name svc_data['service_push'] = service_push svc_id = str(svc_data['resource_id']) - resp_put = requests.put('{url}/resources/{svc_id}'.format(url=self.magpie_url, svc_id=svc_id), - data=svc_data, cookies=self.request.cookies) - check_response(resp_put) + path = schemas.ResourceAPI.path.format(resource_id=svc_id) + resp = requests.put(self.get_url(path), data=svc_data, cookies=self.request.cookies) + check_response(resp) except Exception as e: raise HTTPBadRequest(detail=e.message) @@ -133,21 +138,25 @@ def update_service_url(self, service_name, new_service_url, service_push): svc_data = self.get_service_data(service_name) svc_data['service_url'] = new_service_url svc_data['service_push'] = service_push - resp_put = requests.put('{url}/services/{svc}'.format(url=self.magpie_url, svc=service_name), - data=svc_data, cookies=self.request.cookies) - check_response(resp_put) + path = schemas.ServiceAPI.path.format(service_name=service_name) + resp = requests.put(self.get_url(path), data=svc_data, cookies=self.request.cookies) + check_response(resp) except Exception as e: raise HTTPBadRequest(detail=e.message) def goto_service(self, resource_id): try: - res_json = requests.get('{url}/resources/{id}'.format(url=self.magpie_url, id=resource_id), - cookies=self.request.cookies).json() - svc_name = res_json[resource_id]['resource_name'] + path = schemas.ResourceAPI.path.format(resource_id=resource_id) + resp = requests.get(self.get_url(path), cookies=self.request.cookies) + check_response(resp) + body = resp.json() + svc_name = body[resource_id]['resource_name'] # get service type instead of 'cur_svc_type' in case of 'default' ('cur_svc_type' not set yet) - res_json = requests.get('{url}/services/{svc}'.format(url=self.magpie_url, svc=svc_name), - cookies=self.request.cookies).json() - svc_type = res_json[svc_name]['service_type'] + path = schemas.ServiceAPI.path.format(service_name=svc_name) + resp = requests.get(self.get_url(path), cookies=self.request.cookies) + check_response(resp) + body = resp.json() + svc_type = body[svc_name]['service_type'] return HTTPFound(self.request.route_url('edit_service', service_name=svc_name, cur_svc_type=svc_type)) except Exception as e: raise HTTPBadRequest(detail=repr(e)) @@ -172,7 +181,8 @@ def flatten_tree_resource(resource_node, resource_dict): def view_users(self): if 'delete' in self.request.POST: user_name = self.request.POST.get('user_name') - check_response(requests.delete(self.magpie_url + '/users/' + user_name, cookies=self.request.cookies)) + path = schemas.UserAPI.path.format(user_name=user_name) + check_response(requests.delete(self.get_url(path), cookies=self.request.cookies)) if 'edit' in self.request.POST: user_name = self.request.POST.get('user_name') @@ -201,7 +211,7 @@ def add_user(self): if group_name not in groups: data = {u'group_name': group_name} - resp = requests.post('{url}/groups'.format(url=self.magpie_url), data, cookies=self.request.cookies) + resp = requests.post(self.get_url(schemas.GroupsAPI.path), data, cookies=self.request.cookies) if resp.status_code == HTTPConflict.code: return_data[u'conflict_group_name'] = True if user_email in self.get_user_emails(): @@ -225,7 +235,7 @@ def add_user(self): u'email': user_email, u'password': password, u'group_name': group_name} - check_response(requests.post(self.magpie_url + '/users', data, cookies=self.request.cookies)) + check_response(requests.post(self.get_url(schemas.UsersAPI.path), data, cookies=self.request.cookies)) return HTTPFound(self.request.route_url('view_users')) return add_template_data(self.request, return_data) @@ -234,9 +244,8 @@ def add_user(self): def edit_user(self): user_name = self.request.matchdict['user_name'] cur_svc_type = self.request.matchdict['cur_svc_type'] - inherit_groups_permissions = self.request.matchdict.get('inherit_groups_permissions', False) + inherit_grp_perms = self.request.matchdict.get('inherit_groups_permissions', False) - user_url = '{url}/users/{usr}'.format(url=self.magpie_url, usr=user_name) own_groups = self.get_user_groups(user_name) all_groups = self.get_all_groups(first_default_group=get_constant('MAGPIE_USERS_GROUP')) @@ -253,13 +262,14 @@ def edit_user(self): except Exception as e: raise HTTPBadRequest(detail=repr(e)) - user_resp = requests.get(user_url, cookies=self.request.cookies) + user_url = schemas.UsersAPI.path.format(user_name=user_name) + user_resp = requests.get(self.get_url(user_url), cookies=self.request.cookies) check_response(user_resp) user_info = user_resp.json()['user'] user_info[u'edit_mode'] = u'no_edit' user_info[u'own_groups'] = own_groups user_info[u'groups'] = all_groups - user_info[u'inherit_groups_permissions'] = inherit_groups_permissions + user_info[u'inherit_groups_permissions'] = inherit_grp_perms if self.request.method == 'POST': res_id = self.request.POST.get(u'resource_id') @@ -268,8 +278,8 @@ def edit_user(self): requires_update_name = False if u'inherit_groups_permissions' in self.request.POST: - inherit_groups_permissions = str2bool(self.request.POST[u'inherit_groups_permissions']) - user_info[u'inherit_groups_permissions'] = inherit_groups_permissions + inherit_grp_perms = str2bool(self.request.POST[u'inherit_groups_permissions']) + user_info[u'inherit_groups_permissions'] = inherit_grp_perms if u'delete' in self.request.POST: check_response(requests.delete(user_url, cookies=self.request.cookies)) @@ -306,6 +316,7 @@ def edit_user(self): elif u'force_sync' in self.request.POST: errors = [] for service_info in services.values(): + # noinspection PyBroadException try: sync_resources.fetch_single_service(service_info['resource_id'], session) except Exception: @@ -337,18 +348,19 @@ def edit_user(self): removed_groups = list(set(own_groups) - set(selected_groups)) new_groups = list(set(selected_groups) - set(own_groups)) for group in removed_groups: - url_group = '{url}/users/{usr}/groups/{grp}'.format(url=self.magpie_url, usr=user_name, grp=group) - check_response(requests.delete(url_group, cookies=self.request.cookies)) + path = schemas.UserGroupAPI.path.format(user_name=user_name, group_name=group) + check_response(requests.delete(self.get_url(path), cookies=self.request.cookies)) for group in new_groups: - url_group = '{url}/users/{usr}/groups'.format(url=self.magpie_url, usr=user_name) + path = schemas.UserGroupsAPI.path.format(user_name=user_name) data = {'group_name': group} - check_response(requests.post(url_group, data=data, cookies=self.request.cookies)) + check_response(requests.post(self.get_url(path), data=data, cookies=self.request.cookies)) user_info[u'own_groups'] = self.get_user_groups(user_name) # display resources permissions per service type tab try: res_perm_names, res_perms = self.get_user_or_group_resources_permissions_dict( - user_name, services, cur_svc_type, is_user=True, is_inherit_groups_permissions=inherit_groups_permissions) + user_name, services, cur_svc_type, is_user=True, is_inherit_groups_permissions=inherit_grp_perms + ) except Exception as e: raise HTTPBadRequest(detail=repr(e)) @@ -377,8 +389,8 @@ def edit_user(self): def view_groups(self): if 'delete' in self.request.POST: group_name = self.request.POST.get('group_name') - check_response(requests.delete('{url}/groups/{grp}'.format(url=self.magpie_url, grp=group_name), - cookies=self.request.cookies)) + path = schemas.GroupAPI.path.format(group_name=group_name) + check_response(requests.delete(self.get_url(path), cookies=self.request.cookies)) if 'edit' in self.request.POST: group_name = self.request.POST.get('group_name') @@ -402,7 +414,7 @@ def add_group(self): return add_template_data(self.request, return_data) data = {u'group_name': group_name} - resp = requests.post(self.magpie_url + '/groups', data, cookies=self.request.cookies) + resp = requests.post(self.get_url(schemas.GroupsAPI.path), data, cookies=self.request.cookies) if resp.status_code == HTTPConflict.code: return_data[u'conflict_group_name'] = True return add_template_data(self.request, return_data) @@ -445,17 +457,21 @@ def edit_group_users(self, group_name): new_members = list(set(selected_members) - set(current_members)) for user_name in removed_members: - url_group = '{url}/users/{usr}/groups/{grp}'.format(url=self.magpie_url, usr=user_name, grp=group_name) - check_response(requests.delete(url_group, cookies=self.request.cookies)) + path = schemas.UserGroupAPI.path.format(user_name=user_name, group_name=group_name) + check_response(requests.delete(self.get_url(path), cookies=self.request.cookies)) for user_name in new_members: - url_group = '{url}/users/{usr}/groups'.format(url=self.magpie_url, usr=user_name) + path = schemas.UserGroupsAPI.path.format(user_name=user_name) data = {'group_name': group_name} - check_response(requests.post(url_group, data=data, cookies=self.request.cookies)) + check_response(requests.post(self.get_url(path), data=data, cookies=self.request.cookies)) def edit_user_or_group_resource_permissions(self, user_or_group_name, resource_id, is_user=False): - usr_grp_type = 'users' if is_user else 'groups' - res_perms_url = '{url}/{grp_type}/{grp}/resources/{res_id}/permissions' \ - .format(url=self.magpie_url, grp_type=usr_grp_type, grp=user_or_group_name, res_id=resource_id) + if is_user: + path = schemas.UserResourcePermissionsAPI.path\ + .format(user_name=user_or_group_name, resource_id=resource_id) + else: + path = schemas.GroupResourcePermissionsAPI.path\ + .format(group_name=user_or_group_name, resource_id=resource_id) + res_perms_url = self.get_url(path) try: res_perms_resp = requests.get(res_perms_url, cookies=self.request.cookies) res_perms = res_perms_resp.json()['permission_names'] @@ -476,17 +492,17 @@ def edit_user_or_group_resource_permissions(self, user_or_group_name, resource_i def get_user_or_group_resources_permissions_dict(self, user_or_group_name, services, service_type, is_user=False, is_inherit_groups_permissions=False): - user_or_group_type = 'users' if is_user else 'groups' - inherit_type = 'inherited_' if is_inherit_groups_permissions and is_user else '' + if is_user: + query = '?inherit=true' if is_inherit_groups_permissions else '' + path = schemas.UserResourcesAPI.path.format(user_name=user_or_group_name) + query + else: + path = schemas.GroupResourcesAPI.path.format(group_name=user_or_group_name) - group_perms_url = '{url}/{usr_grp_type}/{usr_grp}/{inherit}resources' \ - .format(url=self.magpie_url, usr_grp_type=user_or_group_type, - usr_grp=user_or_group_name, inherit=inherit_type) - resp_group_perms = check_response(requests.get(group_perms_url, cookies=self.request.cookies)) + resp_group_perms = check_response(requests.get(self.get_url(path), cookies=self.request.cookies)) resp_group_perms_json = resp_group_perms.json() - svc_perm_url = '{url}/services/types/{svc_type}'.format(url=self.magpie_url, svc_type=service_type) - resp_svc_type = check_response(requests.get(svc_perm_url, cookies=self.request.cookies)) + path = schemas.ServiceTypeAPI.path.format(service_type=service_type) + resp_svc_type = check_response(requests.get(self.get_url(path), cookies=self.request.cookies)) resp_available_svc_types = resp_svc_type.json()['services'][service_type] # remove possible duplicate permissions from different services @@ -509,10 +525,9 @@ def get_user_or_group_resources_permissions_dict(self, user_or_group_name, servi except KeyError: pass - resp_resources = check_response(requests.get('{url}/services/{svc}/resources' - .format(url=self.magpie_url, svc=service), - cookies=self.request.cookies)) - raw_resources = resp_resources.json()[service] + path = schemas.ServiceResourcesAPI.path.format(service_name=service) + resp = check_response(requests.get(self.get_url(path), cookies=self.request.cookies)) + raw_resources = resp.json()[service] resources[service] = OrderedDict( id=raw_resources['resource_id'], permission_names=self.default_get(permission, raw_resources['resource_id'], []), @@ -527,9 +542,9 @@ def edit_group(self): error_message = "" - # Todo: - # Until the api is modified to make it possible to request from the RemoteResource table, - # we have to access the database directly here + # TODO: + # Until the api is modified to make it possible to request from the RemoteResource table, + # we have to access the database directly here session = self.request.db try: @@ -541,7 +556,7 @@ def edit_group(self): # move to service or edit requested group/permission changes if self.request.method == 'POST': res_id = self.request.POST.get('resource_id') - group_url = '{url}/groups/{grp}'.format(url=self.magpie_url, grp=group_name) + group_url = self.get_url(schemas.GroupAPI.path.format(group_name=group_name)) if u'delete' in self.request.POST: check_response(requests.delete(group_url, cookies=self.request.cookies)) @@ -562,13 +577,15 @@ def edit_group(self): if not res_id or res_id == 'None': remote_id = int(self.request.POST.get('remote_id')) services_names = [s['service_name'] for s in services.values()] - res_id = self.add_remote_resource(cur_svc_type, services_names, group_name, remote_id, is_user=False) + res_id = self.add_remote_resource(cur_svc_type, services_names, group_name, + remote_id, is_user=False) self.edit_user_or_group_resource_permissions(group_name, res_id, is_user=False) elif u'member' in self.request.POST: self.edit_group_users(group_name) elif u'force_sync' in self.request.POST: errors = [] for service_info in services.values(): + # noinspection PyBroadException try: sync_resources.fetch_single_service(service_info['resource_id'], session) except Exception: @@ -585,8 +602,9 @@ def edit_group(self): # display resources permissions per service type tab try: - res_perm_names, res_perms = self.get_user_or_group_resources_permissions_dict(group_name, services, - cur_svc_type, is_user=False) + res_perm_names, res_perms = self.get_user_or_group_resources_permissions_dict( + group_name, services, cur_svc_type, is_user=False + ) except Exception as e: raise HTTPBadRequest(detail=repr(e)) @@ -614,7 +632,8 @@ def edit_group(self): group_info[u'permissions'] = res_perm_names return add_template_data(self.request, data=group_info) - def make_sync_error_message(self, service_names): + @staticmethod + def make_sync_error_message(service_names): this = "this service" if len(service_names) == 1 else "these services" error_message = ("There seems to be an issue synchronizing resources from " "{}: {}".format(this, ", ".join(service_names))) @@ -629,7 +648,7 @@ def get_remote_resources_info(self, res_perms, services, session): last_sync_datetimes = self.get_last_sync_datetimes(service_ids, session) if any(last_sync_datetimes): - last_sync_datetime = min(filter(bool, last_sync_datetimes)) + last_sync_datetime = min(filter(bool, last_sync_datetimes)) # type: datetime.datetime last_sync_humanized = humanize.naturaltime(now - last_sync_datetime) res_perms = self.merge_remote_resources(res_perms, services, session) @@ -640,7 +659,8 @@ def get_remote_resources_info(self, res_perms, services, session): out_of_sync.append(service_name) return res_perms, ids_to_clean, last_sync_humanized, out_of_sync - def merge_remote_resources(self, res_perms, services, session): + @staticmethod + def merge_remote_resources(res_perms, services, session): merged_resources = {} for service_name, service_values in services.items(): service_id = service_values["resource_id"] @@ -649,13 +669,14 @@ def merge_remote_resources(self, res_perms, services, session): merged_resources[service_name] = resources_for_service[service_name] return merged_resources - def get_last_sync_datetimes(self, service_ids, session): + @staticmethod + def get_last_sync_datetimes(service_ids, session): return [sync_resources.get_last_sync(s, session) for s in service_ids] def delete_resource(self, res_id): - url = '{url}/resources/{resource_id}'.format(url=self.magpie_url, resource_id=res_id) try: - check_response(requests.delete(url, cookies=self.request.cookies)) + path = schemas.ResourceAPI.path.format(resource_id=res_id) + check_response(requests.delete(self.get_url(path), cookies=self.request.cookies)) except HTTPNotFound: # Some resource ids are already deleted because they were a child # of another just deleted parent resource. @@ -672,10 +693,9 @@ def get_ids_to_clean(self, resources): def add_remote_resource(self, service_type, services_names, user_or_group, remote_id, is_user=False): try: - res_perm_names, res_perms = self.get_user_or_group_resources_permissions_dict(user_or_group, - services=services_names, - service_type=service_type, - is_user=is_user) + res_perm_names, res_perms = self.get_user_or_group_resources_permissions_dict( + user_or_group, services=services_names, service_type=service_type, is_user=is_user + ) except Exception as e: raise HTTPBadRequest(detail=repr(e)) @@ -702,9 +722,9 @@ def add_remote_resource(self, service_type, services_names, user_or_group, remot 'resource_type': remote_resource.resource_type, 'parent_id': parent_id, } - resources_url = '{url}/resources'.format(url=self.magpie_url) - response = check_response(requests.post(resources_url, data=data, cookies=self.request.cookies)) - parent_id = response.json()['resource']['resource_id'] + resp = requests.post(self.get_url(schemas.ResourcesAPI.path), data=data, cookies=self.request.cookies) + check_response(resp) + parent_id = resp.json()['resource']['resource_id'] return parent_id @@ -713,8 +733,9 @@ def view_services(self): if 'delete' in self.request.POST: service_name = self.request.POST.get('service_name') service_data = {u'service_push': self.request.POST.get('service_push')} - check_response(requests.delete(self.magpie_url + '/services/' + service_name, - data=json.dumps(service_data), cookies=self.request.cookies)) + path = schemas.ServiceAPI.path.format(service_name=service_name) + resp = requests.delete(self.get_url(path), data=json.dumps(service_data), cookies=self.request.cookies) + check_response(resp) cur_svc_type = self.request.matchdict['cur_svc_type'] svc_types, cur_svc_type, services = self.get_services(cur_svc_type) @@ -752,7 +773,8 @@ def add_service(self): u'service_url': service_url, u'service_type': service_type, u'service_push': service_push} - check_response(requests.post(self.magpie_url + '/services', data=data, cookies=self.request.cookies)) + resp = requests.post(self.get_url(schemas.ServicesAPI.path), data=data, cookies=self.request.cookies) + check_response(resp) return HTTPFound(self.request.route_url('view_services', cur_svc_type=service_type)) services_keys_sorted = sorted(service_type_dict) @@ -807,14 +829,14 @@ def edit_service(self): if 'delete' in self.request.POST: service_data = json.dumps({u'service_push': service_push}) - check_response(requests.delete('{url}/services/{svc}'.format(url=self.magpie_url, svc=service_name), - data=service_data, cookies=self.request.cookies)) + path = schemas.ServiceAPI.path.format(service_name=service_name) + check_response(requests.delete(self.get_url(path), data=service_data, cookies=self.request.cookies)) return HTTPFound(self.request.route_url('view_services', **service_info)) if 'delete_child' in self.request.POST: resource_id = self.request.POST.get('resource_id') - check_response(requests.delete('{url}/resources/{res_id}'.format(url=self.magpie_url, res_id=resource_id), - cookies=self.request.cookies)) + path = schemas.ResourceAPI.format(resource_id=resource_id) + check_response(requests.delete(self.get_url(path), cookies=self.request.cookies)) if 'add_child' in self.request.POST: service_info['resource_id'] = self.request.POST.get('resource_id') @@ -822,9 +844,9 @@ def edit_service(self): try: resources = {} - url_resources = '{url}/services/{svc}/resources'.format(url=self.magpie_url, svc=service_name) - res_resources = check_response(requests.get(url_resources, cookies=self.request.cookies)) - raw_resources = res_resources.json()[service_name] + path = schemas.ServiceResourcesAPI.path.format(service_name=service_name) + resp = check_response(requests.get(self.get_url(path), cookies=self.request.cookies)) + raw_resources = resp.json()[service_name] resources[service_name] = dict( id=raw_resources['resource_id'], permission_names=[], @@ -853,15 +875,14 @@ def add_resource(self): data = {u'resource_name': resource_name, u'resource_type': resource_type, u'parent_id': resource_id} - - check_response(requests.post('{url}/resources'.format(url=self.magpie_url), - data=data, cookies=self.request.cookies)) + resp = requests.post(self.get_url(schemas.ResourcesAPI.path), data=data, cookies=self.request.cookies) + check_response(resp) return HTTPFound(self.request.route_url('edit_service', service_name=service_name, cur_svc_type=cur_svc_type)) - res_types_url = '{url}/services/types/{type}/resources/types'.format(url=self.magpie_url, type=cur_svc_type) - cur_svc_res = check_response(requests.get(res_types_url, cookies=self.request.cookies)) + path = schemas.ServiceResourceTypesAPI.path.format(service_type=cur_svc_type) + cur_svc_res = check_response(requests.get(self.get_url(path), cookies=self.request.cookies)) raw_svc_res = cur_svc_res.json()['resource_types'] return add_template_data(self.request, diff --git a/magpie/ui/utils.py b/magpie/ui/utils.py new file mode 100644 index 000000000..5d21d19f0 --- /dev/null +++ b/magpie/ui/utils.py @@ -0,0 +1,7 @@ +from pyramid.httpexceptions import exception_response + + +def check_response(response): + if response.status_code >= 400: + raise exception_response(response.status_code, body=response.text) + return response From a0898d77e4c10815c87ca62ee50c2d58a87baf62 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Fri, 1 Feb 2019 17:24:30 -0500 Subject: [PATCH 67/76] many more py3 compatibility fixes --- magpie/adapter/magpieowssecurity.py | 2 +- magpie/adapter/magpieprocess.py | 2 +- .../api/management/service/service_views.py | 4 +- magpie/helpers/sync_resources.py | 39 +++++++++---------- magpie/register.py | 2 +- magpie/services.py | 29 +++++++------- magpie/ui/management/views.py | 23 ++++++----- tests/interfaces.py | 28 +++++++++---- tests/test_magpie_api.py | 12 +++--- tests/test_magpie_ui.py | 8 ++-- tests/utils.py | 6 +-- 11 files changed, 82 insertions(+), 73 deletions(-) diff --git a/magpie/adapter/magpieowssecurity.py b/magpie/adapter/magpieowssecurity.py index f3c235437..8e0f07ca4 100644 --- a/magpie/adapter/magpieowssecurity.py +++ b/magpie/adapter/magpieowssecurity.py @@ -79,7 +79,7 @@ def update_request_cookies(self, request): # use specific domain to differentiate between `.{hostname}` and `{hostname}` variations if applicable # noinspection PyProtectedMember request_cookies = session_resp.request._cookies - magpie_cookies = filter(lambda cookie: cookie.name == token_name, request_cookies) + magpie_cookies = list(filter(lambda cookie: cookie.name == token_name, request_cookies)) magpie_domain = urlparse(self.magpie_url).hostname if len(magpie_cookies) > 1 else None session_cookies = RequestsCookieJar.get(request_cookies, token_name, domain=magpie_domain) if not session_resp.json().get('authenticated') or not session_cookies: diff --git a/magpie/adapter/magpieprocess.py b/magpie/adapter/magpieprocess.py index 67bc6dd51..33ae8f018 100644 --- a/magpie/adapter/magpieprocess.py +++ b/magpie/adapter/magpieprocess.py @@ -358,7 +358,7 @@ def list_processes(self, visibility=None, request=None): if self.twitcher_config == TWITCHER_CONFIGURATION_EMS: try: ems_processes_id = self._get_service_processes_resource() - processes = filter(lambda p: self.is_visible_by_user(ems_processes_id, p.id, request), processes) + processes = list(filter(lambda p: self.is_visible_by_user(ems_processes_id, p.id, request), processes)) except KeyError: raise ProcessNotFound("Failed retrieving processes read permissions for listing.") except Exception as ex: diff --git a/magpie/api/management/service/service_views.py b/magpie/api/management/service/service_views.py index 1be910f5a..92f5ef65d 100644 --- a/magpie/api/management/service/service_views.py +++ b/magpie/api/management/service/service_views.py @@ -164,8 +164,8 @@ def get_service_resources_view(request): """List all resources registered under a service.""" service = ar.get_service_matchdict_checked(request) svc_res_json = sf.format_service_resources(service, db_session=request.db, display_all=True, show_private_url=True) - return ax.valid_http(httpSuccess=HTTPOk, detail=ServiceResources_GET_OkResponseSchema.description, - content={str(service.resource_name): svc_res_json}) + return ax.valid_http(httpSuccess=HTTPOk, content={svc_res_json['service_name']: svc_res_json}, + detail=ServiceResources_GET_OkResponseSchema.description) @ServiceResourcesAPI.post(schema=ServiceResources_POST_RequestSchema, tags=[ServicesTag], diff --git a/magpie/helpers/sync_resources.py b/magpie/helpers/sync_resources.py index 108a30dce..a88b31ed0 100644 --- a/magpie/helpers/sync_resources.py +++ b/magpie/helpers/sync_resources.py @@ -1,5 +1,5 @@ """ -Sychronize local and remote resources. +Synchronize local and remote resources. To implement a new service, see the _SyncServiceInterface class. """ @@ -21,9 +21,9 @@ CRON_SERVICE = False OUT_OF_SYNC = datetime.timedelta(hours=3) - +# noinspection PyProtectedMember SYNC_SERVICES_TYPES = defaultdict(lambda: sync_services._SyncServiceDefault) -# noinspection PyTypeChecker +# noinspection PyTypeChecker, PyProtectedMember SYNC_SERVICES_TYPES.update({ "thredds": sync_services._SyncServiceThreads, "geoserver-api": sync_services._SyncServiceGeoserver, @@ -41,7 +41,7 @@ def merge_local_and_remote_resources(resources_local, service_sync_type, service if not get_last_sync(service_id, session): return resources_local remote_resources = _query_remote_resources_in_database(service_id, session=session) - max_depth = _get_max_depth(service_sync_type) + max_depth = SYNC_SERVICES_TYPES[service_sync_type]("", "").max_depth merged_resources = _merge_resources(resources_local, remote_resources, max_depth) _sort_resources(merged_resources) return merged_resources @@ -120,6 +120,7 @@ def _ensure_sync_info_exists(service_resource_id, session): """ service_sync_info = models.RemoteResourcesSyncInfo.by_service_id(service_resource_id, session) if not service_sync_info: + # noinspection PyArgumentList sync_info = models.RemoteResourcesSyncInfo(service_id=service_resource_id) session.add(sync_info) session.flush() @@ -136,8 +137,9 @@ def _get_remote_resources(service): if service_url.endswith("/"): # remove trailing slash service_url = service_url[:-1] - sync_service_class = SYNC_SERVICES_TYPES.get(service.sync_type.lower(), sync_services._SyncServiceDefault) - sync_service = sync_service_class(service.resource_name, service_url) + # noinspection PyProtectedMember + sync_svc_cls = SYNC_SERVICES_TYPES.get(service.sync_type.lower(), sync_services._SyncServiceDefault) + sync_service = sync_svc_cls(service.resource_name, service_url) return sync_service.get_resources() @@ -161,8 +163,9 @@ def _create_main_resource(service_id, session): :param session: """ sync_info = models.RemoteResourcesSyncInfo.by_service_id(service_id, session) + # noinspection PyArgumentList main_resource = models.RemoteResource(service_id=service_id, - resource_name=unicode(sync_info.service.resource_name), + resource_name=str(sync_info.service.resource_name), resource_type=u"directory") session.add(main_resource) session.flush() @@ -181,9 +184,10 @@ def _update_db(remote_resources, service_id, session): def add_children(resources, parent_id, position=0): for resource_name, values in resources.items(): - resource_display_name = unicode(values.get('resource_display_name', resource_name)) + resource_display_name = str(values.get('resource_display_name', resource_name)) + # noinspection PyArgumentList new_resource = models.RemoteResource(service_id=sync_info.service_id, - resource_name=unicode(resource_name), + resource_name=str(resource_name), resource_display_name=resource_display_name, resource_type=values['resource_type'], parent_id=parent_id, @@ -246,16 +250,9 @@ def _format_resource_tree(children): return fmt_res_tree -def _get_max_depth(service_type): - name, url = "", "" - return SYNC_SERVICES_TYPES[service_type](name, url).max_depth - - def _query_remote_resources_in_database(service_id, session): """ Reads remote resources from the RemoteResources table. No external request is made. - :param service_type: - :param session: :return: a dictionary of the form defined in 'sync_services.is_valid_resource_schema' """ service = session.query(models.Service).filter_by(resource_id=service_id).first() @@ -289,7 +286,7 @@ def fetch_all_services_by_type(service_type, session): # noinspection PyBroadException try: fetch_single_service(service, session) - except: + except Exception: if CRON_SERVICE: LOGGER.exception("There was an error when fetching data from the url: %s" % service.url) pass @@ -320,10 +317,10 @@ def fetch(): Main function to get all remote resources for each service and write to database. """ LOGGER.info("Getting database url") - url = db.get_db_url() + db_url = db.get_db_url() - LOGGER.debug("Database url: %s" % url) - engine = create_engine(url) + LOGGER.debug("Database url: %s" % db_url) + engine = create_engine(db_url) session = Session(bind=engine) @@ -366,7 +363,7 @@ def main(): LOGGER.info("Starting to fetch data for all service types") fetch() except Exception: - LOGGER.exception("An error occured") + LOGGER.exception("An error occurred") raise LOGGER.info("Success, exiting.") diff --git a/magpie/register.py b/magpie/register.py index 0f37fd783..96046ff04 100644 --- a/magpie/register.py +++ b/magpie/register.py @@ -539,7 +539,7 @@ def parse_resource_path(permission_config_entry, # type: ConfigItem for res in resource_path.split('/'): if len(res_dict): # resource is specified by id/name - res_id = filter(lambda r: res in [r, res_dict[r]["resource_name"]], res_dict) + res_id = list(filter(lambda r: res in [r, res_dict[r]["resource_name"]], res_dict)) if res_id: res_dict = res_dict[res_id[0]]['children'] # update for upcoming sub-resource iteration parent = int(res[0]) diff --git a/magpie/services.py b/magpie/services.py index db76a7368..170748689 100644 --- a/magpie/services.py +++ b/magpie/services.py @@ -6,11 +6,25 @@ from magpie.owsrequest import * from magpie import models from typing import AnyStr, List, Dict, Union +from six import with_metaclass ResourcePermissionType = Union[models.GroupPermission, models.UserPermission] -class ServiceI(object): +class ServiceMeta(type): + @property + def resource_types(cls): + # type: (...) -> List[AnyStr] + """Allowed resources types under the service.""" + return list(cls.resource_types_permissions.keys()) + + @property + def child_resource_allowed(cls): + # type: (...) -> bool + return len(cls.resource_types) > 0 + + +class ServiceI(with_metaclass(ServiceMeta)): # required request parameters for the service params_expected = [] # type: List[str] # global permissions allowed for the service (top-level resource) @@ -18,19 +32,6 @@ class ServiceI(object): # dict of list for each corresponding allowed resource permissions (children resources) resource_types_permissions = {} # type: Dict[str,List[str]] - # make 'property' getter from derived classes - class __metaclass__(type): - @property - def resource_types(cls): - # type: (...) -> List[AnyStr] - """Allowed resources types under the service.""" - return cls.resource_types_permissions.keys() - - @property - def child_resource_allowed(cls): - # type: (...) -> bool - return len(cls.resource_types) > 0 - def __init__(self, service, request): self.service = service self.request = request diff --git a/magpie/ui/management/views.py b/magpie/ui/management/views.py index 9e652fec1..29ab1a04b 100644 --- a/magpie/ui/management/views.py +++ b/magpie/ui/management/views.py @@ -1,8 +1,3 @@ -import datetime -from collections import OrderedDict - -import humanize - from magpie.api import api_rest_schemas as schemas from magpie.definitions.pyramid_definitions import * from magpie.constants import get_constant @@ -14,9 +9,13 @@ from magpie.helpers import sync_resources from magpie.ui.home import add_template_data from magpie import register, __meta__ +from collections import OrderedDict from distutils.version import LooseVersion +import datetime +import humanize import requests import json +import six class ManagementViews(object): @@ -36,7 +35,7 @@ def get_all_groups(self, first_default_group=None): check_response(resp) try: groups = list(resp.json()['group_names']) - if type(first_default_group) is str and first_default_group in groups: + if isinstance(first_default_group, six.string_types) and first_default_group in groups: groups.remove(first_default_group) groups.insert(0, first_default_group) return groups @@ -45,7 +44,7 @@ def get_all_groups(self, first_default_group=None): def get_group_users(self, group_name): try: - path = schemas.GroupAPI.path.format(group_name=group_name) + path = schemas.GroupUsersAPI.path.format(group_name=group_name) resp = requests.get(self.get_url(path), cookies=self.request.cookies) check_response(resp) return resp.json()['user_names'] @@ -169,7 +168,7 @@ def flatten_tree_resource(resource_node, resource_dict): :return: flattened dictionary `resource_dict` of all {id: 'resource_type'} :rtype: dict """ - if type(resource_node) is not dict: + if not isinstance(resource_node, dict): return if not len(resource_node) > 0: return @@ -251,9 +250,9 @@ def edit_user(self): error_message = "" - # Todo: - # Until the api is modified to make it possible to request from the RemoteResource table, - # we have to access the database directly here + # TODO: + # Until the api is modified to make it possible to request from the RemoteResource table, + # we have to access the database directly here session = self.request.db try: @@ -262,7 +261,7 @@ def edit_user(self): except Exception as e: raise HTTPBadRequest(detail=repr(e)) - user_url = schemas.UsersAPI.path.format(user_name=user_name) + user_url = schemas.UserAPI.path.format(user_name=user_name) user_resp = requests.get(self.get_url(user_url), cookies=self.request.cookies) check_response(user_resp) user_info = user_resp.json()['user'] diff --git a/tests/interfaces.py b/tests/interfaces.py index 3605c0b52..fdfcedfb3 100644 --- a/tests/interfaces.py +++ b/tests/interfaces.py @@ -3,6 +3,7 @@ import pyramid.testing import yaml import six +from six.moves.urllib.parse import urlparse from distutils.version import LooseVersion from magpie.api.api_rest_schemas import SwaggerGenerator from magpie.constants import get_constant @@ -11,9 +12,10 @@ from tests import utils, runner +# noinspection PyPep8Naming @pytest.mark.api @unittest.skipUnless(runner.MAGPIE_TEST_API, reason=runner.MAGPIE_TEST_DISABLED_MESSAGE('api')) -class TestMagpieAPI_NoAuth_Interface(unittest.TestCase): +class Interface_MagpieAPI_NoAuth(object): """ Interface class for unittests of Magpie API. Test any operation that do not require user AuthN/AuthZ. @@ -67,11 +69,12 @@ def test_GetCurrentUser(self): utils.check_val_equal(json_body['user_name'], self.usr) +# noinspection PyPep8Naming @unittest.skip("Not implemented.") @pytest.mark.skip(reason="Not implemented.") @pytest.mark.api @unittest.skipUnless(runner.MAGPIE_TEST_API, reason=runner.MAGPIE_TEST_DISABLED_MESSAGE('api')) -class TestMagpieAPI_UsersAuth_Interface(unittest.TestCase): +class Interface_MagpieAPI_UsersAuth(unittest.TestCase): """ Interface class for unittests of Magpie API. Test any operation that require at least 'Users' group AuthN/AuthZ. @@ -90,9 +93,10 @@ def tearDownClass(cls): pyramid.testing.tearDown() +# noinspection PyPep8Naming @pytest.mark.api @unittest.skipUnless(runner.MAGPIE_TEST_API, reason=runner.MAGPIE_TEST_DISABLED_MESSAGE('api')) -class TestMagpieAPI_AdminAuth_Interface(unittest.TestCase): +class Interface_MagpieAPI_AdminAuth(object): """ Interface class for unittests of Magpie API. Test any operation that require at least 'administrator' group AuthN/AuthZ. @@ -140,7 +144,7 @@ def setup_test_values(cls): cls.test_resource_name = u'magpie-unittest-resource' test_service_res_perm_dict = service_type_dict[cls.test_service_type].resource_types_permissions - test_service_resource_types = test_service_res_perm_dict.keys() + test_service_resource_types = list(test_service_res_perm_dict.keys()) assert len(test_service_resource_types), "test service must allow at least 1 sub-resource for test execution" cls.test_resource_type = test_service_resource_types[0] test_service_resource_perms = test_service_res_perm_dict[cls.test_resource_type] @@ -952,10 +956,16 @@ def test_ValidateDefaultServiceProviders(self): if svc_name in self.test_services_info: utils.check_val_equal(svc['service_type'], self.test_services_info[svc_name]['type']) hostname = utils.get_hostname(self.url) - twitcher_svc_url = get_twitcher_protected_service_url(svc_name, hostname=hostname) - utils.check_val_equal(svc['public_url'], twitcher_svc_url) + # private service URL should match format of Magpie (schema/host) svc_url = self.test_services_info[svc_name]['url'].replace('${HOSTNAME}', hostname) utils.check_val_equal(svc['service_url'], svc_url) + # public service URL should match Twitcher config, but ignore schema that depends on each server config + twitcher_svc_url = get_twitcher_protected_service_url(svc_name, hostname=hostname) + twitcher_parsed_url = urlparse(twitcher_svc_url) + twitcher_test_url = twitcher_parsed_url.netloc + twitcher_parsed_url.path + svc_parsed_url = urlparse(svc['public_url']) + svc_test_public_url = svc_parsed_url.netloc + svc_parsed_url.path + utils.check_val_equal(svc_test_public_url, twitcher_test_url) # ensure that no providers are missing from registered services registered_svc_names = [svc['service_name'] for svc in services_list] @@ -1064,9 +1074,10 @@ def test_DeleteResource(self): utils.TestSetup.check_NonExistingTestServiceResource(self) +# noinspection PyPep8Naming @pytest.mark.ui @unittest.skipUnless(runner.MAGPIE_TEST_UI, reason=runner.MAGPIE_TEST_DISABLED_MESSAGE('ui')) -class TestMagpieUI_NoAuth_Interface(unittest.TestCase): +class Interface_MagpieUI_NoAuth(object): """ Interface class for unittests of Magpie UI. Test any operation that do not require user AuthN/AuthZ. @@ -1155,9 +1166,10 @@ def test_AddService(self): utils.TestSetup.check_Unauthorized(self, method='POST', path=path) +# noinspection PyPep8Naming @pytest.mark.ui @unittest.skipUnless(runner.MAGPIE_TEST_UI, reason=runner.MAGPIE_TEST_DISABLED_MESSAGE('ui')) -class TestMagpieUI_AdminAuth_Interface(unittest.TestCase): +class Interface_MagpieUI_AdminAuth(object): """ Interface class for unittests of Magpie UI. Test any operation that require at least 'administrator' group AuthN/AuthZ. diff --git a/tests/test_magpie_api.py b/tests/test_magpie_api.py index ef656874c..8adf51644 100644 --- a/tests/test_magpie_api.py +++ b/tests/test_magpie_api.py @@ -22,7 +22,7 @@ @pytest.mark.local @unittest.skipUnless(runner.MAGPIE_TEST_API, reason=runner.MAGPIE_TEST_DISABLED_MESSAGE('api')) @unittest.skipUnless(runner.MAGPIE_TEST_LOCAL, reason=runner.MAGPIE_TEST_DISABLED_MESSAGE('local')) -class TestMagpieAPI_NoAuth_Local(ti.TestMagpieAPI_NoAuth_Interface, unittest.TestCase): +class TestCase_MagpieAPI_NoAuth_Local(ti.Interface_MagpieAPI_NoAuth, unittest.TestCase): """ Test any operation that do not require user AuthN/AuthZ. Use a local Magpie test application. @@ -46,7 +46,7 @@ def setUpClass(cls): @pytest.mark.local @unittest.skipUnless(runner.MAGPIE_TEST_API, reason=runner.MAGPIE_TEST_DISABLED_MESSAGE('api')) @unittest.skipUnless(runner.MAGPIE_TEST_LOCAL, reason=runner.MAGPIE_TEST_DISABLED_MESSAGE('local')) -class TestMagpieAPI_UsersAuth_Local(ti.TestMagpieAPI_UsersAuth_Interface): +class TestCase_MagpieAPI_UsersAuth_Local(ti.Interface_MagpieAPI_UsersAuth, unittest.TestCase): """ Test any operation that require at least 'Users' group AuthN/AuthZ. Use a local Magpie test application. @@ -63,7 +63,7 @@ def setUpClass(cls): @pytest.mark.local @unittest.skipUnless(runner.MAGPIE_TEST_API, reason=runner.MAGPIE_TEST_DISABLED_MESSAGE('api')) @unittest.skipUnless(runner.MAGPIE_TEST_LOCAL, reason=runner.MAGPIE_TEST_DISABLED_MESSAGE('local')) -class TestMagpieAPI_AdminAuth_Local(ti.TestMagpieAPI_AdminAuth_Interface): +class TestCase_MagpieAPI_AdminAuth_Local(ti.Interface_MagpieAPI_AdminAuth, unittest.TestCase): """ Test any operation that require at least 'Administrator' group AuthN/AuthZ. Use a local Magpie test application. @@ -95,7 +95,7 @@ def setUpClass(cls): @pytest.mark.remote @unittest.skipUnless(runner.MAGPIE_TEST_API, reason=runner.MAGPIE_TEST_DISABLED_MESSAGE('api')) @unittest.skipUnless(runner.MAGPIE_TEST_REMOTE, reason=runner.MAGPIE_TEST_DISABLED_MESSAGE('remote')) -class TestMagpieAPI_NoAuth_Remote(ti.TestMagpieAPI_NoAuth_Interface, unittest.TestCase): +class TestCase_MagpieAPI_NoAuth_Remote(ti.Interface_MagpieAPI_NoAuth, unittest.TestCase): """ Test any operation that do not require user AuthN/AuthZ. Use an already running remote bird server. @@ -118,7 +118,7 @@ def setUpClass(cls): @pytest.mark.remote @unittest.skipUnless(runner.MAGPIE_TEST_API, reason=runner.MAGPIE_TEST_DISABLED_MESSAGE('api')) @unittest.skipUnless(runner.MAGPIE_TEST_LOCAL, reason=runner.MAGPIE_TEST_DISABLED_MESSAGE('remote')) -class TestMagpieAPI_UsersAuth_Remote(ti.TestMagpieAPI_UsersAuth_Interface, unittest.TestCase): +class TestCase_MagpieAPI_UsersAuth_Remote(ti.Interface_MagpieAPI_UsersAuth, unittest.TestCase): """ Test any operation that require at least 'Users' group AuthN/AuthZ. Use an already running remote bird server. @@ -135,7 +135,7 @@ def setUpClass(cls): @pytest.mark.remote @unittest.skipUnless(runner.MAGPIE_TEST_API, reason=runner.MAGPIE_TEST_DISABLED_MESSAGE('api')) @unittest.skipUnless(runner.MAGPIE_TEST_REMOTE, reason=runner.MAGPIE_TEST_DISABLED_MESSAGE('remote')) -class TestMagpieAPI_AdminAuth_Remote(ti.TestMagpieAPI_AdminAuth_Interface, unittest.TestCase): +class TestCase_MagpieAPI_AdminAuth_Remote(ti.Interface_MagpieAPI_AdminAuth, unittest.TestCase): """ Test any operation that require at least 'Administrator' group AuthN/AuthZ. Use an already running remote bird server. diff --git a/tests/test_magpie_ui.py b/tests/test_magpie_ui.py index 9c561829a..fb4f79933 100644 --- a/tests/test_magpie_ui.py +++ b/tests/test_magpie_ui.py @@ -21,7 +21,7 @@ @pytest.mark.local @unittest.skipUnless(runner.MAGPIE_TEST_UI, reason=runner.MAGPIE_TEST_DISABLED_MESSAGE('ui')) @unittest.skipUnless(runner.MAGPIE_TEST_LOCAL, reason=runner.MAGPIE_TEST_DISABLED_MESSAGE('local')) -class TestMagpieUI_NoAuth_Local(ti.TestMagpieUI_NoAuth_Interface): +class TestCase_MagpieUI_NoAuth_Local(ti.Interface_MagpieUI_NoAuth, unittest.TestCase): """ Test any operation that do not require user AuthN/AuthZ. Use a local Magpie test application. @@ -46,7 +46,7 @@ def setUpClass(cls): @pytest.mark.local @unittest.skipUnless(runner.MAGPIE_TEST_UI, reason=runner.MAGPIE_TEST_DISABLED_MESSAGE('ui')) @unittest.skipUnless(runner.MAGPIE_TEST_LOCAL, reason=runner.MAGPIE_TEST_DISABLED_MESSAGE('local')) -class TestMagpieUI_AdminAuth_Local(ti.TestMagpieUI_AdminAuth_Interface): +class TestCase_MagpieUI_AdminAuth_Local(ti.Interface_MagpieUI_AdminAuth, unittest.TestCase): """ Test any operation that require at least 'administrator' group AuthN/AuthZ. Use a local Magpie test application. @@ -81,7 +81,7 @@ def setUpClass(cls): @pytest.mark.remote @unittest.skipUnless(runner.MAGPIE_TEST_UI, reason=runner.MAGPIE_TEST_DISABLED_MESSAGE('ui')) @unittest.skipUnless(runner.MAGPIE_TEST_REMOTE, reason=runner.MAGPIE_TEST_DISABLED_MESSAGE('remote')) -class TestMagpieUI_NoAuth_Remote(ti.TestMagpieUI_NoAuth_Interface): +class TestCase_MagpieUI_NoAuth_Remote(ti.Interface_MagpieUI_NoAuth, unittest.TestCase): """ Test any operation that do not require user AuthN/AuthZ. Use an already running remote bird server. @@ -105,7 +105,7 @@ def setUpClass(cls): @pytest.mark.remote @unittest.skipUnless(runner.MAGPIE_TEST_UI, reason=runner.MAGPIE_TEST_DISABLED_MESSAGE('ui')) @unittest.skipUnless(runner.MAGPIE_TEST_REMOTE, reason=runner.MAGPIE_TEST_DISABLED_MESSAGE('remote')) -class TestMagpieUI_AdminAuth_Remote(ti.TestMagpieUI_AdminAuth_Interface): +class TestCase_MagpieUI_AdminAuth_Remote(ti.Interface_MagpieUI_AdminAuth, unittest.TestCase): """ Test any operation that require at least 'Administrator' group AuthN/AuthZ. Use an already running remote bird server. diff --git a/tests/utils.py b/tests/utils.py index 52aeec710..1a7e693c8 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -476,7 +476,7 @@ def get_AnyServiceOfTestServiceType(test_class): check_val_not_equal(len(json_body['services'][test_class.test_service_type]), 0, msg="Missing any required service of type: `{}`".format(test_class.test_service_type)) services_dict = json_body['services'][test_class.test_service_type] - return services_dict.values()[0] + return list(services_dict.values())[0] @staticmethod def create_TestServiceResource(test_class, data_override=None): @@ -523,7 +523,7 @@ def check_NonExistingTestServiceResource(test_class): def delete_TestServiceResource(test_class, override_resource_name=None): resource_name = override_resource_name or test_class.test_resource_name resources = TestSetup.get_TestServiceDirectResources(test_class, ignore_missing_service=True) - test_resource = filter(lambda r: r['resource_name'] == resource_name, resources) + test_resource = list(filter(lambda r: r['resource_name'] == resource_name, resources)) # delete as required, skip if non-existing if len(test_resource) > 0: resource_id = test_resource[0]['resource_id'] @@ -561,7 +561,7 @@ def check_NonExistingTestService(test_class): def delete_TestService(test_class, override_service_name=None): service_name = override_service_name or test_class.test_service_name services_info = TestSetup.get_RegisteredServicesList(test_class) - test_service = filter(lambda r: r['service_name'] == service_name, services_info) + test_service = list(filter(lambda r: r['service_name'] == service_name, services_info)) # delete as required, skip if non-existing if len(test_service) > 0: route = '/services/{svc_name}'.format(svc_name=test_class.test_service_name) From b0968f585cefd5ed28db4df783a26df2b13e0bfd Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Fri, 1 Feb 2019 17:46:39 -0500 Subject: [PATCH 68/76] history update --- HISTORY.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index 9351fffac..88b9ec560 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -9,7 +9,8 @@ History * add permissions config to auto-generate user/group rules on startup * fix many invalid or erroneous swagger specifications * attempt db creation on first migration if not existing -* add continuous integration testing and deployment +* add continuous integration testing and deployment (with python 2/3 tests) +* ensure python compatibility for Python 2.7, 3.5, 3.6 * reduce excessive sqlalchemy logging using `MAGPIE_LOG_LEVEL >= INFO` * use schema API route definitions for UI calls From fd860a86592a261830a1aa5f67c410d3c30acd0b Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Fri, 1 Feb 2019 18:04:57 -0500 Subject: [PATCH 69/76] fix test runner discovery --- setup.py | 1 + tests/runner.py | 14 ++++++++------ 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/setup.py b/setup.py index e98d9548e..56ecfcc0d 100644 --- a/setup.py +++ b/setup.py @@ -94,6 +94,7 @@ def _parse_requirements(file_path, requirements, links): # test_suite='nose.collector', # test_suite='tests.test_runner', # test_loader='tests.test_runner:run_suite', + test_suite='tests', tests_require=TEST_REQUIREMENTS, # -- script entry points ----------------------------------------------- diff --git a/tests/runner.py b/tests/runner.py index 8753c214d..19da5bc1d 100644 --- a/tests/runner.py +++ b/tests/runner.py @@ -8,12 +8,14 @@ import os +def filter_test_files(root, filename): + return os.path.isfile(os.path.join(root, filename)) and filename.startswith('test') and filename.endswith('.py') + + test_root_path = os.path.abspath(os.path.dirname(__file__)) test_root_name = os.path.split(test_root_path)[1] -test_modules = [ - '{}.test_magpie_api'.format(test_root_name), - '{}.test_magpie_ui'.format(test_root_name), -] +test_files = os.listdir(test_root_path) +test_modules = [os.path.splitext(f)[0] for f in filter(lambda i: filter_test_files(test_root_path, i), test_files)] def default_run(option): @@ -54,8 +56,8 @@ def test_suite(): suite.addTest(unittest.defaultTestLoader.loadTestsFromName(t)) except AttributeError: # if still not found, try discovery from root directory - #tests = unittest.defaultTestLoader.loadTestsFromModule(t) - #suite.addTests(tests) + # tests = unittest.defaultTestLoader.loadTestsFromModule(t) + # suite.addTests(tests) suite.addTest(unittest.defaultTestLoader.discover(test_root_path)) return suite From aa92ede938ab7f3134db27c433518980a6c31cdf Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Fri, 1 Feb 2019 18:15:33 -0500 Subject: [PATCH 70/76] reduce logging of sqlalchemy --- ci/magpie.env | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/magpie.env b/ci/magpie.env index c864e7305..18f5e9efc 100644 --- a/ci/magpie.env +++ b/ci/magpie.env @@ -9,3 +9,4 @@ HOSTNAME=localhost MAGPIE_TEST_REMOTE_SERVER_URL=http://localhost:2001 MAGPIE_TEST_ADMIN_USERNAME=admin MAGPIE_TEST_ADMIN_PASSWORD=qwerty +MAGPIE_LOG_LEVEL=INFO From 037bc2b1798804c0770fd544ca50dbae20c6cd9d Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Fri, 1 Feb 2019 18:24:23 -0500 Subject: [PATCH 71/76] all sqlalchemy logging to warn --- ci/magpie.env | 6 +++++- magpie/magpiectl.py | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/ci/magpie.env b/ci/magpie.env index 18f5e9efc..90453f002 100644 --- a/ci/magpie.env +++ b/ci/magpie.env @@ -6,7 +6,11 @@ MAGPIE_POSTGRES_PORT=5432 MAGPIE_POSTGRES_DB=magpie PHOENIX_PUSH=false HOSTNAME=localhost +MAGPIE_SECRET=magpie +MAGPIE_ADMIN_GROUP=administrators +MAGPIE_ADMIN_USER=admin +MAGPIE_ADMIN_PASSWORD=qwerty +MAGPIE_LOG_LEVEL=INFO MAGPIE_TEST_REMOTE_SERVER_URL=http://localhost:2001 MAGPIE_TEST_ADMIN_USERNAME=admin MAGPIE_TEST_ADMIN_PASSWORD=qwerty -MAGPIE_LOG_LEVEL=INFO diff --git a/magpie/magpiectl.py b/magpie/magpiectl.py index e8a3cea69..d796fefed 100644 --- a/magpie/magpiectl.py +++ b/magpie/magpiectl.py @@ -39,7 +39,7 @@ def main(global_config=None, **settings): raise_missing=False, print_missing=False, raise_not_set=False) log_lvl = logging.getLevelName(log_lvl) if isinstance(log_lvl, int) else log_lvl if log_lvl.upper() != 'DEBUG': - sa_log = logging.getLogger('sqlalchemy.engine.base.Engine') + sa_log = logging.getLogger('sqlalchemy') sa_log.setLevel(logging.WARN) # WARN to avoid INFO logs LOGGER.setLevel(log_lvl) From 1cbfa752f62cd39f0edb770135b6b6a690fce1f5 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Mon, 4 Feb 2019 10:38:49 -0500 Subject: [PATCH 72/76] try direct export log level before tests --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index ac3aafb79..5815ee79d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -68,6 +68,7 @@ before_script: - echo $CONDA_PREFIX - echo $CONDA_ENV script: + - export MAGPIE_LOG_LEVEL=INFO - make $START_TARGET $TEST_TARGET notifications: email: false From 32362ac905384c7dc71e9257f6e9f3535a497a03 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Mon, 4 Feb 2019 15:36:17 -0500 Subject: [PATCH 73/76] bye-bye sqlalchemy logging --- magpie/alembic/env.py | 36 ++++++--- magpie/constants.py | 2 +- magpie/db.py | 97 +++++++++++++++++++++-- magpie/definitions/alembic_definitions.py | 1 + magpie/definitions/typedefs.py | 5 ++ magpie/helpers/register_default_users.py | 19 ++++- magpie/magpie.ini | 2 +- magpie/magpiectl.py | 39 +++------ 8 files changed, 150 insertions(+), 51 deletions(-) create mode 100644 magpie/definitions/typedefs.py diff --git a/magpie/alembic/env.py b/magpie/alembic/env.py index a76bbfe82..e88dbc43c 100644 --- a/magpie/alembic/env.py +++ b/magpie/alembic/env.py @@ -2,7 +2,8 @@ from alembic import context from logging.config import fileConfig from sqlalchemy.schema import MetaData -from sqlalchemy.engine import create_engine +# noinspection PyProtectedMember +from sqlalchemy.engine import create_engine, Connection from sqlalchemy.exc import OperationalError from magpie.db import get_db_url from magpie.constants import get_constant @@ -12,6 +13,11 @@ # access to the values within the .ini file in use. config = context.config +# verify if a connection is already provided +config_connection = None +if 'connection' in config.attributes and isinstance(config.attributes['connection'], Connection): + config_connection = context.config.attributes['connection'] + # Interpret the config file for Python logging. # This line sets up loggers basically. fileConfig(config.config_file_name) @@ -52,6 +58,20 @@ def run_migrations_offline(): context.run_migrations() +def run_migrations_connection(connection): + # type: (Connection) -> None + """Run migrations in 'online' mode with provided connection.""" + context.configure( + connection=connection, + target_metadata=target_metadata, + version_table='alembic_version', + transaction_per_migration=True, + render_as_batch=True + ) + with context.begin_transaction(): + context.run_migrations() + + def run_migrations_online(): """Run migrations in 'online' mode. @@ -85,21 +105,15 @@ def run_migrations_online(): # retry connection and run migration with connectable.connect() as connection: - context.configure( - connection=connection, - target_metadata=target_metadata, - version_table='alembic_version', - transaction_per_migration=True, - render_as_batch=True - ) try: - with context.begin_transaction(): - context.run_migrations() + run_migrations_connection(connection) finally: connection.close() if context.is_offline_mode(): run_migrations_offline() -else: +elif config_connection is None: run_migrations_online() +else: + run_migrations_connection(config_connection) diff --git a/magpie/constants.py b/magpie/constants.py index cf3ed503d..5c70d1564 100644 --- a/magpie/constants.py +++ b/magpie/constants.py @@ -147,5 +147,5 @@ def get_constant(name, settings=None, settings_name=None, default_value=None, if missing and raise_missing: raise_log("Constant could not be found: {}".format(name), level=logging.ERROR) if missing and print_missing: - print_log("Constant could not be found: {}".format(name), level=logging.WARN) + print_log("Constant could not be found: {} (using default: {})".format(name, default_value), level=logging.WARN) return magpie_value or default_value diff --git a/magpie/db.py b/magpie/db.py index 1042f2031..1ee60c751 100644 --- a/magpie/db.py +++ b/magpie/db.py @@ -1,13 +1,16 @@ #!/usr/bin/python # -*- coding: utf-8 -*- from magpie import constants -from magpie.common import get_settings_from_config_ini +from magpie.common import get_settings_from_config_ini, print_log, raise_log from magpie.definitions.alembic_definitions import * from magpie.definitions.sqlalchemy_definitions import * +from magpie.definitions.typedefs import AnyStr, SettingsDict, Optional, Union import transaction import inspect import zope.sqlalchemy +import warnings import logging +import time # import or define all models here to ensure they are attached to the # Base.metadata prior to any initialization routines @@ -74,25 +77,42 @@ def get_db_session_from_settings(settings): return db_session -def get_db_session_from_config_ini(config_ini_path, ini_main_section_name='app:magpie_app'): +def get_db_session_from_config_ini(config_ini_path, ini_main_section_name='app:magpie_app', settings_override=None): settings = get_settings_from_config_ini(config_ini_path, ini_main_section_name) + if isinstance(settings_override, dict): + settings.update(settings_override) return get_db_session_from_settings(settings) -def run_database_migration(): +def run_database_migration(db_session=None): + # type: (Optional[Session]) -> None + """Runs db migration operations with alembic, using db session or a new engine connection.""" logger.info("Using file '{}' for migration.".format(constants.MAGPIE_ALEMBIC_INI_FILE_PATH)) alembic_args = ['-c', constants.MAGPIE_ALEMBIC_INI_FILE_PATH, 'upgrade', 'heads'] - alembic.config.main(argv=alembic_args) + if not isinstance(db_session, Session): + alembic.config.main(argv=alembic_args) + else: + engine = db_session.bind + with engine.begin() as connection: + alembic_cfg = alembic.config.Config(file_=constants.MAGPIE_ALEMBIC_INI_FILE_PATH) + alembic_cfg.attributes['connection'] = connection + alembic.command.upgrade(alembic_cfg, "head") def get_database_revision(db_session): + # type: (Session) -> AnyStr s = select(['version_num'], from_obj='alembic_version') result = db_session.execute(s).fetchone() return result['version_num'] -def is_database_ready(): - inspector = Inspector.from_engine(get_engine(dict())) +def is_database_ready(db_session=None): + # type: (Optional[Session]) -> bool + if isinstance(db_session, Session): + engine = db_session.bind + else: + engine = get_engine(dict()) + inspector = Inspector.from_engine(engine) table_names = inspector.get_table_names() for name, obj in inspect.getmembers(models): @@ -107,6 +127,71 @@ def is_database_ready(): return True +def run_database_migration_when_ready(settings, db_session=None): + # type: (SettingsDict, Optional[Session]) -> None + """Runs db migration if requested, """ + db_ready = False + if constants.get_constant('MAGPIE_DB_MIGRATION', settings, 'magpie.db_migration', True, + raise_missing=False, raise_not_set=False, print_missing=True): + attempts = constants.get_constant('MAGPIE_DB_MIGRATION_ATTEMPTS', settings, 'magpie.db_migration_attempts', + default_value=5, raise_missing=False, raise_not_set=False, print_missing=True) + + print_log('Running database migration (as required) ...') + attempts = max(attempts, 1) + for i in range(1, attempts + 1): + try: + with warnings.catch_warnings(): + warnings.simplefilter("ignore", category=sa_exc.SAWarning) + run_database_migration(db_session) + except ImportError: + pass + except Exception as e: + if i < attempts: + print_log('Database migration failed [{!s}]. Retrying... ({}/{})'.format(e, i, attempts)) + time.sleep(2) + continue + else: + raise_log('Database migration failed [{!s}]'.format(e), exception=RuntimeError) + + # HACK: + # migration can cause sqlalchemy engine to reset its internal logger level, although it is properly set + # to 'echo=False'... apply configs to re-enforce the logging level of `sqlalchemy.engine.base.Engine` + log_lvl = constants.get_constant( + 'MAGPIE_LOG_LEVEL', settings, 'magpie.log_level', default_value=logging.INFO, + raise_missing=False, print_missing=False, raise_not_set=False + ) + set_sqlalchemy_log_level(log_lvl) + + db_ready = is_database_ready(db_session) + if not db_ready: + print_log('Database not ready. Retrying... ({}/{})'.format(i, attempts)) + time.sleep(2) + break + else: + db_ready = is_database_ready(db_session) + if not db_ready: + time.sleep(2) + raise_log('Database not ready') + + +def set_sqlalchemy_log_level(magpie_log_level): + # type: (Union[AnyStr, int]) -> SettingsDict + """Suppresses sqlalchemy logging if not in debug for magpie.""" + log_lvl = logging.getLevelName(magpie_log_level) if isinstance(magpie_log_level, int) else magpie_log_level + sa_settings = {'sqlalchemy.echo': True} + if log_lvl.upper() != 'DEBUG': + sa_settings['sqlalchemy.echo'] = False + sa_loggers = 'sqlalchemy.engine.base.Engine'.split('.') + sa_log = logging.getLogger(sa_loggers[0]) + sa_log.setLevel(logging.WARN) # WARN to avoid INFO logs + for h in sa_log.handlers: + sa_log.removeHandler(h) + for sa_mod in sa_loggers[1:]: + sa_log = sa_log.getChild(sa_mod) + sa_log.setLevel(logging.WARN) + return sa_settings + + def includeme(config): """ Initialize the model for a Pyramid app. diff --git a/magpie/definitions/alembic_definitions.py b/magpie/definitions/alembic_definitions.py index 7b57ca142..410382adc 100644 --- a/magpie/definitions/alembic_definitions.py +++ b/magpie/definitions/alembic_definitions.py @@ -1,3 +1,4 @@ from alembic import op from alembic.context import get_context import alembic.config +import alembic.command diff --git a/magpie/definitions/typedefs.py b/magpie/definitions/typedefs.py new file mode 100644 index 000000000..873e900ca --- /dev/null +++ b/magpie/definitions/typedefs.py @@ -0,0 +1,5 @@ +# noinspection PyUnresolvedReferences +from typing import AnyStr, Dict, Optional, Union # noqa: F401 + +SettingField = Union[AnyStr, int, float, bool] +SettingsDict = Dict[AnyStr, SettingField] diff --git a/magpie/helpers/register_default_users.py b/magpie/helpers/register_default_users.py index 627db30f2..4e57f74ae 100644 --- a/magpie/helpers/register_default_users.py +++ b/magpie/helpers/register_default_users.py @@ -1,6 +1,8 @@ from magpie import constants, db, models from magpie.common import print_log, raise_log +from magpie.definitions.sqlalchemy_definitions import Session from magpie.definitions.ziggurat_definitions import * +from magpie.definitions.typedefs import Optional import transaction import logging import time @@ -10,12 +12,14 @@ def register_user_with_group(user_name, group_name, email, password, db_session): if not GroupService.by_group_name(group_name, db_session=db_session): + # noinspection PyArgumentList new_group = models.Group(group_name=group_name) db_session.add(new_group) registered_group = GroupService.by_group_name(group_name=group_name, db_session=db_session) registered_user = UserService.by_user_name(user_name, db_session=db_session) if not registered_user: + # noinspection PyArgumentList new_user = models.User(user_name=user_name, email=email) UserService.set_password(new_user, password) UserService.regenerate_security_code(new_user) @@ -24,14 +28,16 @@ def register_user_with_group(user_name, group_name, email, password, db_session) else: print_log('User `{}` already exist'.format(user_name), level=logging.DEBUG) + # noinspection PyBroadException try: # ensure the reference between user/group exists (user joined the group) user_group_refs = models.UserGroup.all(db_session=db_session) user_group_refs_tup = [(ref.group_id, ref.user_id) for ref in user_group_refs] if (registered_group.id, registered_user.id) not in user_group_refs_tup: + # noinspection PyArgumentList group_entry = models.UserGroup(group_id=registered_group.id, user_id=registered_user.id) db_session.add(group_entry) - except: # in case reference already exists, avoid duplicate error + except Exception: # in case reference already exists, avoid duplicate error db_session.rollback() @@ -56,6 +62,7 @@ def init_admin(db_session): magpie_admin_group = GroupService.by_group_name(constants.MAGPIE_ADMIN_GROUP, db_session=db_session) permission_names = [permission.perm_name for permission in magpie_admin_group.permissions] if constants.MAGPIE_ADMIN_PERMISSION not in permission_names: + # noinspection PyArgumentList new_group_permission = models.GroupPermission(perm_name=constants.MAGPIE_ADMIN_PERMISSION, group_id=magpie_admin_group.id) try: @@ -67,18 +74,22 @@ def init_admin(db_session): def init_user_group(db_session): if not GroupService.by_group_name(constants.MAGPIE_USERS_GROUP, db_session=db_session): + # noinspection PyArgumentList user_group = models.Group(group_name=constants.MAGPIE_USERS_GROUP) db_session.add(user_group) else: print_log('MAGPIE_USERS_GROUP already initialized', level=logging.DEBUG) -def register_default_users(): - if not db.is_database_ready(): +def register_default_users(db_session=None): + # type: (Optional[Session]) -> None + if not isinstance(db_session, Session): + db_session = db.get_db_session_from_config_ini(constants.MAGPIE_INI_FILE_PATH) + db_close = False + if not db.is_database_ready(db_session): time.sleep(2) raise_log('Database not ready') - db_session = db.get_db_session_from_config_ini(constants.MAGPIE_INI_FILE_PATH) init_admin(db_session) init_anonymous(db_session) init_user_group(db_session) diff --git a/magpie/magpie.ini b/magpie/magpie.ini index d65575bf7..dfc6d54fb 100644 --- a/magpie/magpie.ini +++ b/magpie/magpie.ini @@ -87,4 +87,4 @@ level = INFO formatter = generic [formatter_generic] -format = %(asctime)s %(levelname)-5.5s [%(name)s][%(threadName)s] %(message)s +format = %(asctime)s %(levelname)-7.7s [%(name)s][%(threadName)s] %(message)s diff --git a/magpie/magpiectl.py b/magpie/magpiectl.py index d796fefed..c38542d30 100644 --- a/magpie/magpiectl.py +++ b/magpie/magpiectl.py @@ -6,9 +6,8 @@ """ # -- Project specific -------------------------------------------------------- -from magpie.common import print_log, raise_log, str2bool +from magpie.common import print_log, str2bool from magpie.constants import get_constant -from magpie.definitions.sqlalchemy_definitions import * from magpie.helpers.register_default_users import register_default_users from magpie.register import ( magpie_register_services_from_config, @@ -18,8 +17,6 @@ from magpie import db, constants # -- Standard library -------------------------------------------------------- -import time -import warnings # noinspection PyUnresolvedReferences import logging import logging.config # find config in 'logging.ini' @@ -34,36 +31,22 @@ def main(global_config=None, **settings): settings['magpie.root'] = constants.MAGPIE_ROOT settings['magpie.module'] = constants.MAGPIE_MODULE_DIR - # suppress sqlalchemy logging if not in debug for magpie - log_lvl = get_constant('MAGPIE_LOG_LEVEL', settings, 'magpie.log_level', default_value=logging.INFO, - raise_missing=False, print_missing=False, raise_not_set=False) - log_lvl = logging.getLevelName(log_lvl) if isinstance(log_lvl, int) else log_lvl - if log_lvl.upper() != 'DEBUG': - sa_log = logging.getLogger('sqlalchemy') - sa_log.setLevel(logging.WARN) # WARN to avoid INFO logs + print_log('Setting up loggers...') + log_lvl = get_constant('MAGPIE_LOG_LEVEL', settings, 'magpie.log_level', default_value='INFO', + raise_missing=False, raise_not_set=False, print_missing=True) LOGGER.setLevel(log_lvl) + sa_settings = db.set_sqlalchemy_log_level(log_lvl) + # fetch db session here, otherwise, any other db engine connection (ie: during migration) will re-initialize + # with a new engine class and logging settings don't get re-evaluated/applied + db_session = db.get_db_session_from_config_ini(constants.MAGPIE_INI_FILE_PATH, settings_override=sa_settings) - # migrate db as required and check if database is ready - if get_constant('MAGPIE_DB_MIGRATION', settings, 'magpie.db_migration', True, - raise_missing=False, raise_not_set=False, print_missing=True): - print_log('Running database migration (as required) ...') - try: - with warnings.catch_warnings(): - warnings.simplefilter("ignore", category=sa_exc.SAWarning) - db.run_database_migration() - except ImportError: - pass - except Exception as e: - raise_log('Database migration failed [{}]'.format(str(e)), exception=RuntimeError) - if not db.is_database_ready(): - time.sleep(2) - raise_log('Database not ready') + print_log('Looking for db migration requirement...') + db.run_database_migration_when_ready(settings, db_session=db_session) print_log('Register default users...') - register_default_users() + register_default_users(db_session=db_session) print_log('Register configuration providers...', LOGGER) - db_session = db.get_db_session_from_config_ini(constants.MAGPIE_INI_FILE_PATH) push_phoenix = str2bool(get_constant('PHOENIX_PUSH', settings=settings, settings_name='magpie.phoenix_push', raise_missing=False, raise_not_set=False, print_missing=True)) magpie_register_services_from_config(constants.MAGPIE_PROVIDERS_CONFIG_PATH, push_to_phoenix=push_phoenix, From eeb5bf86e3b8b1835e80f3715c4728f565a91ba7 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Mon, 4 Feb 2019 18:40:19 -0500 Subject: [PATCH 74/76] reduce to minimum logging sqlalchemy engine with db-create+migration support --- magpie/alembic/env.py | 95 +++++++++++-------- .../20671b28c538_change_all_linking_k.py | 3 +- magpie/db.py | 27 +++--- magpie/magpiectl.py | 14 ++- 4 files changed, 79 insertions(+), 60 deletions(-) diff --git a/magpie/alembic/env.py b/magpie/alembic/env.py index e88dbc43c..3b8b1a9e6 100644 --- a/magpie/alembic/env.py +++ b/magpie/alembic/env.py @@ -3,10 +3,14 @@ from logging.config import fileConfig from sqlalchemy.schema import MetaData # noinspection PyProtectedMember -from sqlalchemy.engine import create_engine, Connection +from sqlalchemy.engine import create_engine, Connection, Connectable from sqlalchemy.exc import OperationalError from magpie.db import get_db_url from magpie.constants import get_constant +import logging +import os + +LOGGER = logging.getLogger(__name__) # this is the Alembic Config object, which provides @@ -38,6 +42,25 @@ # ... etc. +def create_database(db_name, db_host, db_port): + db_default_postgres_url = get_db_url( + # only postgres user can connect to default 'postgres' db + # credentials correspond to postgres running instance, not magpie-specific credentials + # see for details: + # https://stackoverflow.com/questions/6506578 + username=os.getenv('POSTGRES_USER', 'postgres'), + password=os.getenv('POSTGRES_PASSWORD', ''), + db_host=db_host, + db_port=db_port, + db_name='postgres' + ) + connectable = create_engine(db_default_postgres_url) + with connectable.connect() as connection: + connection.execute("commit") # end initial transaction + connection.execute("create database {}".format(db_name)) + return connectable + + def run_migrations_offline(): """Run migrations in 'offline' mode. @@ -58,21 +81,7 @@ def run_migrations_offline(): context.run_migrations() -def run_migrations_connection(connection): - # type: (Connection) -> None - """Run migrations in 'online' mode with provided connection.""" - context.configure( - connection=connection, - target_metadata=target_metadata, - version_table='alembic_version', - transaction_per_migration=True, - render_as_batch=True - ) - with context.begin_transaction(): - context.run_migrations() - - -def run_migrations_online(): +def run_migrations_online(connection=None): """Run migrations in 'online' mode. In this scenario we need to create an Engine @@ -81,39 +90,49 @@ def run_migrations_online(): """ # test the connection, if database is missing try creating it url = get_db_url() + + def connect(c=None): + if isinstance(c, Connection) and not c.closed: + return c + if not isinstance(c, Connectable): + c = create_engine(url) + return c.connect() + try: - connectable = create_engine(url) - with connectable.connect(): + conn = connect() # use new connection to not close arg one + with conn: pass + conn = connection except OperationalError as ex: - # see for details: - # https://stackoverflow.com/questions/6506578 db_name = get_constant('MAGPIE_POSTGRES_DB') + # message format is important, match error type with it + # any error is `OperationalError`, validate only missing db error if 'database "{}" does not exist'.format(db_name) not in str(ex): - raise # any error is OperationalError, so validate only missing db error - db_default_postgres_url = get_db_url( - username='postgres', # only postgres user can connect to default 'postgres' db - password='', - db_host=get_constant('MAGPIE_POSTGRES_HOST'), - db_port=get_constant('MAGPIE_POSTGRES_PORT'), - db_name='postgres' - ) - connectable = create_engine(db_default_postgres_url) - with connectable.connect() as connection: - connection.execute("commit") # end initial transaction - connection.execute("create database {}".format(db_name)) + raise + LOGGER.info('database [{}] not found, attempting creation...'.format(db_name)) + db_host = get_constant('MAGPIE_POSTGRES_HOST') + db_port = get_constant('MAGPIE_POSTGRES_PORT') + conn = create_database(db_name, db_host, db_port) # retry connection and run migration - with connectable.connect() as connection: + with connect(conn) as migrate_conn: try: - run_migrations_connection(connection) + context.configure( + connection=migrate_conn, + target_metadata=target_metadata, + version_table='alembic_version', + transaction_per_migration=True, + render_as_batch=True + ) + with context.begin_transaction(): + context.run_migrations() finally: - connection.close() + # close the connection only if not coming from upper call + if migrate_conn is not connection: + migrate_conn.close() if context.is_offline_mode(): run_migrations_offline() -elif config_connection is None: - run_migrations_online() else: - run_migrations_connection(config_connection) + run_migrations_online(config_connection) diff --git a/magpie/alembic/versions/20671b28c538_change_all_linking_k.py b/magpie/alembic/versions/20671b28c538_change_all_linking_k.py index 2f08716c3..1e52c38e1 100644 --- a/magpie/alembic/versions/20671b28c538_change_all_linking_k.py +++ b/magpie/alembic/versions/20671b28c538_change_all_linking_k.py @@ -30,8 +30,7 @@ def upgrade(): users_resources_permissions_pkey = 'users_resources_permissions_pkey' # inspected keys - groups_permissions_pkey = insp.get_pk_constraint('groups_permissions')[ - 'name'] + groups_permissions_pkey = insp.get_pk_constraint('groups_permissions')['name'] groups_pkey = insp.get_pk_constraint('groups')['name'] groups_resources_permissions_pkey = \ insp.get_pk_constraint('groups_resources_permissions')['name'] diff --git a/magpie/db.py b/magpie/db.py index 1ee60c751..02adca21f 100644 --- a/magpie/db.py +++ b/magpie/db.py @@ -129,7 +129,10 @@ def is_database_ready(db_session=None): def run_database_migration_when_ready(settings, db_session=None): # type: (SettingsDict, Optional[Session]) -> None - """Runs db migration if requested, """ + """ + Runs db migration if requested by config and need from revisions. + """ + db_ready = False if constants.get_constant('MAGPIE_DB_MIGRATION', settings, 'magpie.db_migration', True, raise_missing=False, raise_not_set=False, print_missing=True): @@ -137,41 +140,33 @@ def run_database_migration_when_ready(settings, db_session=None): default_value=5, raise_missing=False, raise_not_set=False, print_missing=True) print_log('Running database migration (as required) ...') - attempts = max(attempts, 1) + attempts = max(attempts, 2) # enforce at least 2 attempts, 1 for db creation and one for actual migration for i in range(1, attempts + 1): try: with warnings.catch_warnings(): warnings.simplefilter("ignore", category=sa_exc.SAWarning) run_database_migration(db_session) - except ImportError: + except ImportError as ex: pass except Exception as e: - if i < attempts: - print_log('Database migration failed [{!s}]. Retrying... ({}/{})'.format(e, i, attempts)) + if i <= attempts: + print_log('Database migration failed [{!r}]. Retrying... ({}/{})'.format(e, i, attempts)) time.sleep(2) continue else: - raise_log('Database migration failed [{!s}]'.format(e), exception=RuntimeError) - - # HACK: - # migration can cause sqlalchemy engine to reset its internal logger level, although it is properly set - # to 'echo=False'... apply configs to re-enforce the logging level of `sqlalchemy.engine.base.Engine` - log_lvl = constants.get_constant( - 'MAGPIE_LOG_LEVEL', settings, 'magpie.log_level', default_value=logging.INFO, - raise_missing=False, print_missing=False, raise_not_set=False - ) - set_sqlalchemy_log_level(log_lvl) + raise_log('Database migration failed [{!r}]'.format(e), exception=RuntimeError) db_ready = is_database_ready(db_session) if not db_ready: print_log('Database not ready. Retrying... ({}/{})'.format(i, attempts)) time.sleep(2) + continue break else: db_ready = is_database_ready(db_session) if not db_ready: time.sleep(2) - raise_log('Database not ready') + raise_log('Database not ready', exception=RuntimeError) def set_sqlalchemy_log_level(magpie_log_level): diff --git a/magpie/magpiectl.py b/magpie/magpiectl.py index c38542d30..8c7d09f36 100644 --- a/magpie/magpiectl.py +++ b/magpie/magpiectl.py @@ -36,12 +36,18 @@ def main(global_config=None, **settings): raise_missing=False, raise_not_set=False, print_missing=True) LOGGER.setLevel(log_lvl) sa_settings = db.set_sqlalchemy_log_level(log_lvl) - # fetch db session here, otherwise, any other db engine connection (ie: during migration) will re-initialize - # with a new engine class and logging settings don't get re-evaluated/applied - db_session = db.get_db_session_from_config_ini(constants.MAGPIE_INI_FILE_PATH, settings_override=sa_settings) print_log('Looking for db migration requirement...') - db.run_database_migration_when_ready(settings, db_session=db_session) + db.run_database_migration_when_ready(settings) # cannot pass db session as it might not even exist yet! + + # HACK: + # migration can cause sqlalchemy engine to reset its internal logger level, although it is properly set + # to 'echo=False' because engines are re-created as needed... (ie: missing db) + # apply configs to re-enforce the logging level of `sqlalchemy.engine.base.Engine`""" + db.set_sqlalchemy_log_level(log_lvl) + # fetch db session here, otherwise, any following db engine connection will re-initialize + # with a new engine class and logging settings don't get re-evaluated/applied + db_session = db.get_db_session_from_config_ini(constants.MAGPIE_INI_FILE_PATH, settings_override=sa_settings) print_log('Register default users...') register_default_users(db_session=db_session) From 172bd0dba42ae186ff1df4a2373e58be16857e36 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Mon, 4 Feb 2019 18:48:33 -0500 Subject: [PATCH 75/76] temp disable local tests, only use 'remote' instance (ref issue #114) --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 5815ee79d..2b3573997 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,7 +16,8 @@ env: - CONDA_HOME=$HOME/conda - DOWNLOAD_CACHE=$HOME/downloads matrix: - - TEST_TARGET=test-local START_TARGET= + # FIXME: local login not functional, cannot run local tests on travis + #- TEST_TARGET=test-local START_TARGET= - TEST_TARGET=test-remote START_TARGET=start - TEST_TARGET=coverage START_TARGET= addons: From b1e36184328b8473a653efc07ad1f7f0454edec8 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Mon, 4 Feb 2019 19:23:52 -0500 Subject: [PATCH 76/76] add xml coverage report for travis upload to codecov --- .gitignore | 1 + Makefile | 2 ++ 2 files changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 3cf16b840..8c11471cf 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ __pycache__/ .idea/ .eggs/ .pytest_cache/ +coverage.xml build/ coverage/ dist/ diff --git a/Makefile b/Makefile index a6216a276..4f16b9a9a 100644 --- a/Makefile +++ b/Makefile @@ -101,6 +101,7 @@ clean-test: @echo "Cleaning tests artifacts..." rm -fr .tox/ rm -f .coverage + rm -f coverage.xml rm -fr "$(CUR_DIR)/coverage/" .PHONY: lint @@ -132,6 +133,7 @@ test-tox: install-dev install coverage: install-dev install @echo "Running coverage analysis..." @bash -c 'source "$(CONDA_HOME)/bin/activate" "$(CONDA_ENV)"; coverage run --source magpie setup.py test || true' + @bash -c 'source "$(CONDA_HOME)/bin/activate" "$(CONDA_ENV)"; coverage xml -i' @bash -c 'source "$(CONDA_HOME)/bin/activate" "$(CONDA_ENV)"; coverage report -m' .PHONY: coverage-show