Merge pull request #120 from Ouranosinc/fix-tests-upgrade-ziggurat

Fix tests + Upgrade ziggurat

magpie/__meta__.py

@@ -2,7 +2,7 @@
 General meta information on the magpie package.
 """
 
-__version__ = '0.7.2'
+__version__ = '0.7.3'
 __author__ = "Francois-Xavier Derue, Francis Charette-Migneault"
 __maintainer__ = "Francis Charette-Migneault"
 __email__ = '[email protected]'

magpie/adapter/__init__.py

@@ -8,7 +8,7 @@
 from magpie.db import *
 from magpie import __meta__
 import logging
-logger = logging.getLogger("TWITCHER")
+LOGGER = logging.getLogger("TWITCHER")
 
 
 class MagpieAdapter(AdapterInterface):
@@ -29,20 +29,20 @@ def jobstore_factory(self, registry):
         return DefaultAdapter().jobstore_factory(registry)
 
     def owssecurity_factory(self, registry):
-        return MagpieOWSSecurity()
+        return MagpieOWSSecurity(registry=registry)
 
     def configurator_factory(self, settings):
         # Disable rpcinterface which is conflicting with postgres db
         settings['twitcher.rpcinterface'] = False
 
-        logger.info('Loading MagpieAdapter config')
+        LOGGER.info('Loading MagpieAdapter config')
         config = auth_config_from_settings(settings)
         config.set_request_property(get_user, 'user', reify=True)
         self.owsproxy_config(settings, config)
         return config
 
     def owsproxy_config(self, settings, config):
-        logger.info('Loading MagpieAdapter owsproxy config')
+        LOGGER.info('Loading MagpieAdapter owsproxy config')
 
         # use pyramid_tm to hook the transaction lifecycle to the request
         config.include('pyramid_tm')
@@ -58,7 +58,7 @@ def owsproxy_config(self, settings, config):
             reify=True
         )
 
-        logger.info('Adding MagpieAdapter owsproxy routes and views')
+        LOGGER.info('Adding MagpieAdapter owsproxy routes and views')
         protected_path = settings.get('twitcher.ows_proxy_protected_path', '/ows')
         config.add_route('owsproxy', protected_path + '/{service_name}')
         config.add_route('owsproxy_extra', protected_path + '/{service_name}/{extra_path:.*}')

magpie/adapter/magpieowssecurity.py

@@ -3,17 +3,23 @@
 from magpie.services import service_factory
 from magpie.models import Service
 from magpie.api.api_except import evaluate_call, verify_param
+from magpie.adapter.utils import get_magpie_url
 import requests
 import logging
 LOGGER = logging.getLogger("TWITCHER")
 
 
 class MagpieOWSSecurity(OWSSecurityInterface):
 
+    def __init__(self, registry):
+        super(MagpieOWSSecurity, self).__init__()
+        self.magpie_url = get_magpie_url(registry)
+        self.twitcher_ssl_verify = asbool(registry.settings.get('twitcher.ows_proxy_ssl_verify', True))
+        self.twitcher_protected_path = registry.settings.get('twitcher.ows_proxy_protected_path', '/ows')
+
     def check_request(self, request):
-        twitcher_protected_path = request.registry.settings.get('twitcher.ows_proxy_protected_path', '/ows')
-        if request.path.startswith(twitcher_protected_path):
-            service_name = parse_service_name(request.path, twitcher_protected_path)
+        if request.path.startswith(self.twitcher_protected_path):
+            service_name = parse_service_name(request.path, self.twitcher_protected_path)
             service = evaluate_call(lambda: Service.by_service_name(service_name, db_session=request.db),
                                     fallback=lambda: request.db.rollback(),
                                     httpError=HTTPForbidden, msgOnFail="Service query by name refused by db")
@@ -34,23 +40,21 @@ def check_request(self, request):
                 if not has_permission:
                     raise OWSAccessForbidden("Not authorized to access this resource.")
 
-    @staticmethod
-    def update_request_cookies(request):
+    def update_request_cookies(self, request):
         """
         Ensure login of the user and update the request cookies if twitcher is in a special configuration.
         Only update if Magpie `auth_tkt` is missing and can be retrieved from `access_token` in `Authorization` header.
         Counter-validate the login procedure by calling Magpie's `/session` which should indicated a logged user.
         """
         not_default = get_twitcher_configuration(request.registry.settings) != TWITCHER_CONFIGURATION_DEFAULT
         if not_default and 'Authorization' in request.headers and 'auth_tkt' not in request.cookies:
-            ssl_verify = asbool(request.registry.settings.get('twitcher.ows_proxy_ssl_verify', True))
             magpie_url = request.registry.settings.get('magpie.url')
             magpie_prov = request.params.get('provider', 'WSO2')
             magpie_auth = '{host}/providers/{provider}/signin'.format(host=magpie_url, provider=magpie_prov)
             headers = request.headers
             headers['Homepage-Route'] = '/session'
             headers['Accept'] = 'application/json'
-            auth_resp = requests.get(magpie_auth, headers=headers, verify=ssl_verify)
+            auth_resp = requests.get(magpie_auth, headers=headers, verify=self.twitcher_ssl_verify)
             if auth_resp.status_code != HTTPOk.code:
                 raise auth_resp.raise_for_status()
             if not auth_resp.json().get('authenticated') or 'auth_tkt' not in auth_resp.request._cookies:

magpie/adapter/magpieservice.py

@@ -2,34 +2,22 @@
 Store adapters to read data from magpie.
 """
 
-from six.moves.urllib.parse import urlparse
-import logging
+from magpie.definitions.twitcher_definitions import *
+from magpie.definitions.pyramid_definitions import HTTPOk, asbool
+from magpie.adapter.utils import get_admin_cookies, get_magpie_url
 import requests
-import json
+import logging
 LOGGER = logging.getLogger("TWITCHER")
 
-from magpie.definitions.twitcher_definitions import *
-from magpie.definitions.pyramid_definitions import ConfigurationError, HTTPOk
-
 
 class MagpieServiceStore(ServiceStore):
     """
     Registry for OWS services. Uses magpie to fetch service url and attributes.
     """
     def __init__(self, registry):
-        try:
-            # add 'http' scheme to url if omitted from config since further 'requests' calls fail without it
-            # mostly for testing when only 'localhost' is specified
-            # otherwise twitcher config should explicitly define it in MAGPIE_URL
-            url_parsed = urlparse(registry.settings.get('magpie.url').strip('/'))
-            if url_parsed.scheme in ['http', 'https']:
-                self.magpie_url = url_parsed.geturl()
-            else:
-                self.magpie_url = 'http://{}'.format(url_parsed.geturl())
-                LOGGER.warn("Missing scheme from MagpieServiceStore url, new value: '{}'".format(self.magpie_url))
-        except AttributeError:
-            #If magpie.url does not exist, calling strip fct over None will raise this issue
-            raise ConfigurationError('magpie.url config cannot be found')
+        self.magpie_url = get_magpie_url(registry)
+        self.twitcher_ssl_verify = asbool(registry.settings.get('twitcher.ows_proxy_ssl_verify', True))
+        self.magpie_admin_token = get_admin_cookies(self.magpie_url, self.twitcher_ssl_verify)
 
     def save_service(self, service, overwrite=True, request=None):
         """
@@ -47,19 +35,19 @@ def list_services(self, request=None):
         """
         Lists all services registered in magpie.
         """
-        my_services = []
-        path = '/users/current/services?inherit=True&cascade=True'
-        response = requests.get('{url}{path}'.format(url=self.magpie_url, path=path),
-                                cookies=request.cookies)
-        if response.status_code != HTTPOk.code:
-            raise response.raise_for_status()
-        services = json.loads(response.text)
-        for service_type in services['services']:
-            for key, service in services['services'][service_type].items():
-                my_services.append(Service(url=service['service_url'],
-                                           name=service['service_name'],
-                                           type=service['service_type']))
-        return my_services
+        # obtain admin access since 'service_url' is only provided on admin routes
+        services = []
+        path = '{}/services'.format(self.magpie_url)
+        resp = requests.get(path, cookies=self.magpie_admin_token, headers={'Accept': 'application/json'})
+        if resp.status_code != HTTPOk.code:
+            raise resp.raise_for_status()
+        json_body = resp.json()
+        for service_type in json_body['services']:
+            for key, service in json_body['services'][service_type].items():
+                services.append(Service(url=service['service_url'],
+                                        name=service['service_name'],
+                                        type=service['service_type']))
+        return services
 
     def fetch_by_name(self, name, request=None):
         """

magpie/adapter/utils.py

@@ -0,0 +1,35 @@
+from magpie.constants import get_constant
+from magpie.definitions.pyramid_definitions import HTTPOk, ConfigurationError, Registry
+from six.moves.urllib.parse import urlparse
+from typing import Dict
+import requests
+import logging
+LOGGER = logging.getLogger("TWITCHER")
+
+
+def get_admin_cookies(magpie_url, verify=True):
+    # type: (str, bool) -> Dict[str,str]
+    magpie_login_url = '{}/signin'.format(magpie_url)
+    cred = {'user_name': get_constant('MAGPIE_ADMIN_USER'), 'password': get_constant('MAGPIE_ADMIN_PASSWORD')}
+    resp = requests.post(magpie_login_url, data=cred, headers={'Accept': 'application/json'}, verify=verify)
+    if resp.status_code != HTTPOk.code:
+        raise resp.raise_for_status()
+    return dict(auth_tkt=resp.cookies.get('auth_tkt'))
+
+
+def get_magpie_url(registry):
+    # type: (Registry) -> str
+    try:
+        # add 'http' scheme to url if omitted from config since further 'requests' calls fail without it
+        # mostly for testing when only 'localhost' is specified
+        # otherwise twitcher config should explicitly define it in MAGPIE_URL
+        url_parsed = urlparse(registry.settings.get('magpie.url').strip('/'))
+        if url_parsed.scheme in ['http', 'https']:
+            return url_parsed.geturl()
+        else:
+            magpie_url = 'http://{}'.format(url_parsed.geturl())
+            LOGGER.warn("Missing scheme from registry url, new value: '{}'".format(magpie_url))
+            return magpie_url
+    except AttributeError:
+        # If magpie.url does not exist, calling strip fct over None will raise this issue
+        raise ConfigurationError('magpie.url config cannot be found')