Merge pull request #82 from Ouranosinc/magpie_adapter

Magpie adapter

magpie/__meta__.py

@@ -3,6 +3,6 @@
 """
 
 
-__version__ = '0.6.1'
+__version__ = '0.6.2'
 __author__ = "Francois-Xavier Derue, Francis Charette-Migneault"
 __email__ = '[email protected]'

magpie/adapter/__init__.py

@@ -0,0 +1,56 @@
+from magpie.definitions.pyramid_definitions import *
+from magpie.definitions.ziggurat_definitions import *
+from magpie.definitions.twitcher_definitions import *
+from magpie.adapter.magpieowssecurity import *
+from magpie.adapter.magpieservice import *
+from magpie.models import get_user
+from magpie.security import auth_config_from_settings
+from magpie.db import *
+import logging
+logger = logging.getLogger(__name__)
+
+
+class MagpieAdapter(AdapterInterface):
+
+    def servicestore_factory(self, registry, database=None):
+        return MagpieServiceStore(registry=registry)
+
+    def owssecurity_factory(self, registry):
+        return MagpieOWSSecurity()
+
+    def configurator_factory(self, settings):
+        # Disable rpcinterface which is conflicting with postgres db
+        settings['twitcher.rpcinterface'] = False
+
+        logger.info('Loading MagpieAdapter config')
+        config = auth_config_from_settings(settings)
+        config.set_request_property(get_user, 'user', reify=True)
+        self.owsproxy_config(settings, config)
+        return config
+
+    def owsproxy_config(self, settings, config):
+        logger.info('Loading MagpieAdapter owsproxy config')
+
+        # use pyramid_tm to hook the transaction lifecycle to the request
+        config.include('pyramid_tm')
+
+        session_factory = get_session_factory(get_engine(settings))
+        config.registry['dbsession_factory'] = session_factory
+
+        # make request.db available for use in Pyramid
+        config.add_request_method(
+            # r.tm is the transaction manager used by pyramid_tm
+            lambda r: get_tm_session(session_factory, r.tm),
+            'db',
+            reify=True
+        )
+
+        logger.info('Adding MagpieAdapter owsproxy routes and views')
+        protected_path = settings.get('twitcher.ows_proxy_protected_path', '/ows')
+        config.add_route('owsproxy', protected_path + '/{service_name}')
+        config.add_route('owsproxy_extra', protected_path + '/{service_name}/{extra_path:.*}')
+        config.add_route('owsproxy_secured', protected_path + '/{service_name}/{access_token}')
+
+        config.add_view(owsproxy, route_name='owsproxy')
+        config.add_view(owsproxy, route_name='owsproxy_extra')
+        config.add_view(owsproxy, route_name='owsproxy_secured')

magpie/adapter/magpieowssecurity.py

@@ -0,0 +1,44 @@
+import tempfile
+from magpie.definitions.twitcher_definitions import *
+from magpie.definitions.pyramid_definitions import *
+from magpie.services import service_factory
+from magpie.models import Service
+from magpie.api.api_except import evaluate_call, verify_param
+
+import logging
+LOGGER = logging.getLogger("TWITCHER")
+
+
+class MagpieOWSSecurity(object):
+
+    def prepare_headers(self, request, access_token):
+        if "esgf_access_token" in access_token.data or "esgf_credentials" in access_token.data:
+            workdir = tempfile.mkdtemp(prefix=request.prefix, dir=request.workdir)
+            if fetch_certificate(workdir=workdir, data=access_token.data):
+                request.headers['X-Requested-Workdir'] = workdir
+                request.headers['X-X509-User-Proxy'] = workdir + '/' + ESGF_CREDENTIALS
+                LOGGER.debug("Prepared request headers.")
+        return request
+
+    def check_request(self, request):
+        twitcher_protected_path = request.registry.settings.get('twitcher.ows_proxy_protected_path', '/ows')
+        if request.path.startswith(twitcher_protected_path):
+            service_name = parse_service_name(request.path, twitcher_protected_path)
+            service = evaluate_call(lambda: Service.by_service_name(service_name, db_session=request.db),
+                                    fallback=lambda: request.db.rollback(),
+                                    httpError=HTTPForbidden, msgOnFail="Service query by name refused by db")
+            verify_param(service, notNone=True, httpError=HTTPNotFound, msgOnFail="Service name not found in db")
+
+            # return a specific type of service, ex: ServiceWPS with all the acl (loaded according to the service_type)
+            service_specific = service_factory(service, request)
+            # should contain all the acl, this the only thing important
+            # parse request (GET/POST) to get the permission requested for that service
+            permission_requested = service_specific.permission_requested()
+
+            if permission_requested:
+                authn_policy = request.registry.queryUtility(IAuthenticationPolicy)
+                authz_policy = request.registry.queryUtility(IAuthorizationPolicy)
+                principals = authn_policy.effective_principals(request)
+                has_permission = authz_policy.permits(service_specific, principals, permission_requested)
+                if not has_permission:
+                    raise OWSAccessForbidden("Not authorized to access this resource.")

magpie/adapter/magpieservice.py

@@ -0,0 +1,87 @@
+"""
+Store adapters to read data from magpie.
+"""
+
+import logging
+import requests
+import json
+LOGGER = logging.getLogger(__name__)
+
+from magpie.definitions.twitcher_definitions import *
+from magpie.definitions.pyramid_definitions import ConfigurationError
+
+
+class MagpieServiceStore(ServiceStore):
+    """
+    Registry for OWS services. Uses magpie to fetch service url and attributes.
+    """
+    def __init__(self, registry):
+        try:
+            self.magpie_url = registry.settings.get('magpie.url').strip('/')
+        except AttributeError:
+            #If magpie.url does not exist, calling strip fct over None will raise this issue
+            raise ConfigurationError('magpie.url config cannot be found')
+
+    def save_service(self, service, overwrite=True, request=None):
+        """
+        Magpie store is read-only, use magpie api to add services
+        """
+        raise NotImplementedError
+
+    def delete_service(self, name, request=None):
+        """
+        Magpie store is read-only, use magpie api to delete services
+        """
+        raise NotImplementedError
+
+    #TODO Now only wps are returned as well as fetch_by_ulr... fetch_by_name has been patched to support other service_type
+    def list_services(self, request=None):
+        """
+        Lists all services registered in magpie.
+        """
+        my_services = []
+        response = requests.get('{url}/services/types/wps'.format(url=self.magpie_url),
+                                cookies=request.cookies)
+        if response.status_code != 200:
+            raise response.raise_for_status()
+        services = json.loads(response.text)
+        if 'wps' in services['services']:
+            for key, service in services['services']['wps'].items():
+                my_services.append(Service(url=service['service_url'],
+                                           name=service['service_name'],
+                                           type=service['service_type']))
+        return my_services
+
+    def fetch_by_name(self, name, request=None):
+        """
+        Gets service for given ``name`` from magpie.
+        """
+        response = requests.get('{url}/services/{name}'.format(url=self.magpie_url, name=name),
+                                cookies=request.cookies)
+        if response.status_code == 404:
+            raise ServiceNotFound
+        if response.status_code != 200:
+            raise response.raise_for_status()
+        services = json.loads(response.text)
+        if name in services:
+            return Service(url=services[name]['service_url'],
+                           name=services[name]['service_name'],
+                           type=services[name]['service_type'])
+        raise ServiceNotFound
+
+    def fetch_by_url(self, url, request=None):
+        """
+        Gets service for given ``url`` from mongodb storage.
+        """
+        services = self.list_services(request=request)
+        for service in services:
+            if service.url == url:
+                return service
+        raise ServiceNotFound
+
+
+    def clear_services(self, request=None):
+        """
+        Magpie store is read-only, use magpie api to delete services
+        """
+        raise NotImplementedError

magpie/api/login/login.py

@@ -7,6 +7,7 @@
 from definitions.ziggurat_definitions import *
 from magpie import *
 from api.api_requests import *
+from api.api_except import *
 from api.management.user.user_utils import create_user
 import requests
 

magpie/definitions/pyramid_definitions.py

@@ -1,6 +1,7 @@
 from pyramid.config import Configurator
 from pyramid.authentication import AuthTktAuthenticationPolicy
 from pyramid.authorization import ACLAuthorizationPolicy
+from pyramid.exceptions import ConfigurationError
 from pyramid.httpexceptions import (
     HTTPOk,
     HTTPCreated,
@@ -15,7 +16,7 @@
     HTTPUnprocessableEntity,
     HTTPInternalServerError,
 )
-from pyramid.interfaces import IAuthenticationPolicy
+from pyramid.interfaces import IAuthenticationPolicy, IAuthorizationPolicy
 from pyramid.response import Response
 from pyramid.view import (
     view_config,

magpie/definitions/twitcher_definitions.py

@@ -0,0 +1,8 @@
+from twitcher.adapter.base import AdapterInterface
+from twitcher.owsproxy import owsproxy
+from twitcher.owsexceptions import OWSAccessForbidden
+from twitcher.utils import parse_service_name
+from twitcher.esgf import fetch_certificate, ESGF_CREDENTIALS
+from twitcher.datatype import Service
+from twitcher.store.base import ServiceStore
+from twitcher.exceptions import ServiceNotFound

magpie/owsrequest.py

@@ -31,7 +31,7 @@ def parse(self, param_list):
             self.params[param_name] = self._get_param_value(param_name)
         return self.params
 
-    def _get_param_value(self):
+    def _get_param_value(self, param):
         raise NotImplementedError
 
 

magpie/security.py

@@ -1,11 +1,36 @@
+from definitions.pyramid_definitions import *
+from definitions.ziggurat_definitions import *
+from api.esgf import esgfopenid
+from common import print_log
 from authomatic import Authomatic, provider_id
 from authomatic.providers import oauth2, openid
-from api.esgf import esgfopenid
+import models
 import os
 import logging
 logger = logging.getLogger(__name__)
 
 
+def auth_config_from_settings(settings):
+    magpie_secret = os.getenv('MAGPIE_SECRET')
+    if magpie_secret is None:
+        print_log('Use default secret from magpie.ini', level=logging.DEBUG)
+        magpie_secret = settings['magpie.secret']
+
+    authn_policy = AuthTktAuthenticationPolicy(
+        magpie_secret,
+        callback=groupfinder,
+    )
+    authz_policy = ACLAuthorizationPolicy()
+
+    config = Configurator(
+        settings=settings,
+        root_factory=models.RootFactory,
+        authentication_policy=authn_policy,
+        authorization_policy=authz_policy
+    )
+    return config
+
+
 def authomatic(request):
     return Authomatic(
         config=authomatic_config(request),

requirements.txt

@@ -2,7 +2,7 @@ bumpversion==0.5.3
 wheel==0.23.0
 watchdog==0.8.3
 flake8==3.5.0
-tox==2.1.1
+tox==3.0
 coverage==4.0
 Sphinx==1.3.1
 #cryptography==1.9

setup.py

@@ -15,9 +15,19 @@
 with open('HISTORY.rst') as history_file:
     HISTORY = history_file.read().replace('.. :changelog:', '')
 
+# See https://github.com/pypa/pip/issues/3610
 REQUIREMENTS = set([])  # use set to have unique packages by name
+LINKS = set([])
 with open('requirements.txt', 'r') as requirements_file:
-    [REQUIREMENTS.add(line.strip()) for line in requirements_file]
+    for line in requirements_file:
+        if 'git+https' in line:
+            pkg = line.split('#')[-1]
+            LINKS.add(line.strip() + '-0')
+            REQUIREMENTS.add(pkg.replace('egg=', '').rstrip())
+        else:
+            REQUIREMENTS.add(line.strip())
+
+LINKS = list(LINKS)
 REQUIREMENTS = list(REQUIREMENTS)
 
 TEST_REQUIREMENTS = [
@@ -61,6 +71,7 @@
                  'magpie'},
     include_package_data=True,
     install_requires=REQUIREMENTS,
+    dependency_links=LINKS,
     zip_safe=False,
 
     # -- self - tests --------------------------------------------------------