Merge pull request #85 from Ouranosinc/api+unittests

API autogen schemas +unittests + resolve include MagpieAdapter
references #68 and #74

.gitignore

@@ -1,9 +1,11 @@
 __pycache__/
 *.pyc
+.coverage
 .idea/
 .eggs/
 .pytest_cache/
 build/
+coverage/
 dist/
 docs/_build/
 docs/magpie.esgf.rst
@@ -17,6 +19,7 @@ env/
 magpieenv/
 magpie/api_docs/
 magpie/api_test/
+magpie/ui/swagger-ui/magpie-rest-api.json
 magpie/bin/
 magpie/include/
 magpie/lib/

Dockerfile

@@ -1,5 +1,5 @@
 FROM ubuntu:16.04
-MAINTAINER Francois-Xavier Derue
+MAINTAINER Francis Charette-Migneault
 
 RUN apt-get update && apt-get install -y \
 	build-essential \

Makefile

@@ -45,7 +45,7 @@ clean-pyc:
 clean-test:
 	rm -fr .tox/
 	rm -f .coverage
-	rm -fr htmlcov/
+	rm -fr coverage/
 
 lint:
 	flake8 magpie tests
@@ -59,8 +59,8 @@ test-all:
 coverage:
 	coverage run --source magpie setup.py test
 	coverage report -m
-	coverage html
-	$(BROWSER) htmlcov/index.html
+	coverage html -d coverage
+	$(BROWSER) coverage/index.html
 
 migrate:
 	alembic upgrade head

README.rst

@@ -6,6 +6,12 @@ Magpie (the smart-bird)
 
 Magpie is service for AuthN/AuthZ accessible via a `RestAPI`_ implemented with the Pyramid web framework. It allows you to manage User/Group/Resource/permission with a postgres database. Behind the scene, it uses `Ziggurat-Foundations`_ and `Authomatic`_.
 
+REST API Documentation
+======================
+
+The documentation is auto-generated and served under `{HOSTNAME}/magpie/api/` using Swagger-UI with tag `latest`.
+For convenience, older API versions are also provided.
+
 
 Build package
 =============

bin/gunicorn

@@ -1,6 +1,5 @@
 import sys
 import gunicorn.app.wsgiapp
-import sys
 
 
 if __name__ == '__main__':

docs/sync.py

@@ -5,7 +5,7 @@
 """
 
 from subprocess import check_call
-from urllib.parse import urljoin
+from six.moves.urllib.parse import urljoin
 from shlex import split
 import argparse
 

magpie/__init__.py

@@ -2,8 +2,12 @@
 
 import os
 import sys
-this_dir = os.path.abspath(os.path.dirname(__file__))
-sys.path.insert(0, this_dir)
+MAGPIE_MODULE_DIR = os.path.abspath(os.path.dirname(__file__))
+MAGPIE_ROOT = os.path.dirname(MAGPIE_MODULE_DIR)
+sys.path.insert(0, MAGPIE_MODULE_DIR)
+
+MAGPIE_PROVIDERS_CONFIG_PATH = '{}/providers.cfg'.format(MAGPIE_ROOT)
+MAGPIE_INI_FILE_PATH = '{}/magpie.ini'.format(MAGPIE_MODULE_DIR)
 
 ADMIN_USER = os.getenv('ADMIN_USER', 'admin')
 ADMIN_GROUP = os.getenv('ADMIN_GROUP', 'administrators')
@@ -12,6 +16,8 @@
 USER_GROUP = os.getenv('USER_GROUP', 'users')
 
 ANONYMOUS_USER = os.getenv('ANONYMOUS_USER', 'anonymous')
+ANONYMOUS_GROUP = ANONYMOUS_USER
+ANONYMOUS_PASSWORD = ANONYMOUS_USER
 
 ADMIN_PERM = 'admin'
 #ADMIN_PERM = NO_PERMISSION_REQUIRED
@@ -29,7 +35,7 @@ def includeme(config):
     config.include('cornice_swagger')
     config.include('pyramid_chameleon')
     config.include('pyramid_mako')
-    config.include('definitions')
-    config.include('api')
-    config.include('db')
-    config.include('ui')
+    config.include('magpie.definitions')
+    config.include('magpie.api')
+    config.include('magpie.db')
+    config.include('magpie.ui')

magpie/__meta__.py

@@ -2,7 +2,9 @@
 General meta information on the magpie package.
 """
 
-
-__version__ = '0.6.2'
+__version__ = '0.6.3'
 __author__ = "Francois-Xavier Derue, Francis Charette-Migneault"
+__maintainer__ = "Francis Charette-Migneault"
 __email__ = '[email protected]'
+__url__ = 'https://github.com/Ouranosinc/Magpie'
+__description__ = "Magpie is a service for AuthN and AuthZ based on Ziggurat-Foundations"

magpie/adapter/__init__.py

@@ -9,10 +9,9 @@
 import logging
 logger = logging.getLogger(__name__)
 
-
 class MagpieAdapter(AdapterInterface):
 
-    def servicestore_factory(self, registry, database=None):
+    def servicestore_factory(self, registry, database=None, headers=None):
         return MagpieServiceStore(registry=registry)
 
     def owssecurity_factory(self, registry):

magpie/alembic/versions/20671b28c538_change_all_linking_k.py

@@ -13,7 +13,7 @@
 
 from alembic import op
 from alembic.context import get_context
-from definitions.sqlalchemy_definitions import *
+from magpie.definitions.sqlalchemy_definitions import *
 
 
 def upgrade():

magpie/alembic/versions/5e7b5346c330_remove_obsolete_personal_groups.py

@@ -19,7 +19,7 @@
 from sqlalchemy.dialects.postgresql.base import PGDialect
 from sqlalchemy.orm import sessionmaker
 from magpie import models, ANONYMOUS_USER, ADMIN_GROUP, USER_GROUP
-from definitions.ziggurat_definitions import *
+from magpie.definitions.ziggurat_definitions import *
 
 Session = sessionmaker()
 

magpie/alembic/versions/a395ef9d3fe6_reference_root_service.py

@@ -15,7 +15,7 @@
 
 from alembic import op
 from alembic.context import get_context
-from definitions.sqlalchemy_definitions import *
+from magpie.definitions.sqlalchemy_definitions import *
 from magpie import models
 from magpie.api.management.resource.resource_utils import get_resource_root_service
 

magpie/alembic/versions/ae1a3c8c7860_tranfer_group_users_admins_users.py

@@ -15,9 +15,9 @@
 
 from alembic import op
 from alembic.context import get_context
-from definitions.sqlalchemy_definitions import *
+from magpie.definitions.sqlalchemy_definitions import *
 from magpie import models, ANONYMOUS_USER
-from definitions.ziggurat_definitions import *
+from magpie.definitions.ziggurat_definitions import *
 
 Session = sessionmaker()
 

magpie/api/__init__.py

@@ -6,10 +6,10 @@ def includeme(config):
     logger.info('Adding api routes ...')
 
     # Add all the admin ui routes
-    config.include('api.esgf')
-    config.include('api.home')
-    config.include('api.login')
-    config.include('api.management')
+    config.include('magpie.api.esgf')
+    config.include('magpie.api.home')
+    config.include('magpie.api.login')
+    config.include('magpie.api.management')
 
     config.add_route('version', '/version')
     config.scan()

magpie/api/api_except.py

@@ -1,6 +1,7 @@
 from pyramid.httpexceptions import *
 from sys import exc_info
 import types
+import six
 
 # control variables to avoid infinite recursion in case of
 # major programming error to avoid application hanging
@@ -11,14 +12,16 @@
 def verify_param(param, paramCompare=None, httpError=HTTPNotAcceptable, httpKWArgs=None, msgOnFail="",
                  content=None, contentType='application/json',
                  notNone=False, notEmpty=False, notIn=False, notEqual=False,
-                 isNone=False,  isEmpty=False,  isIn=False,  isEqual=False, ofType=None, withParam=True):
+                 isNone=False,  isEmpty=False,  isIn=False,  isEqual=False, ofType=None,
+                 withParam=True, paramName=None):
     """
     Evaluate various parameter combinations given the requested flags.
     Given a failing verification, directly raises the specified `httpError`.
     Invalid exceptions generated by this verification process are treated as `HTTPInternalServerError`.
     Exceptions are generated using the standard output method.
 
     :param param: (bool) parameter value to evaluate
+    :param paramName: (str) name of the tested parameter returned in response if specified for debugging purposes
     :param paramCompare:
         other value(s) to test against, can be an iterable (single value resolved as iterable unless None)
         to test for None type, use `isNone`/`notNone` flags instead or `paramCompare`=[None]
@@ -96,9 +99,11 @@ def verify_param(param, paramCompare=None, httpError=HTTPNotAcceptable, httpKWAr
         status = status or (not type(param) == ofType)
     if status:
         if withParam:
-            content[u'param'] = repr(param)
+            content[u'param'] = {u'value': str(param) if type(param) in six.string_types else repr(param)}
+            if paramName is not None:
+                content[u'param'][u'name'] = str(paramName)
             if paramCompare is not None:
-                content[u'paramCompare'] = repr(paramCompare)
+                content[u'param'][u'compare'] = repr(paramCompare)
         raise_http(httpError, httpKWArgs=httpKWArgs, detail=msgOnFail, content=content, contentType=contentType)
 
 
@@ -329,7 +334,7 @@ def generate_response_http_format(httpClass, httpKWArgs, jsonContent, outputType
     try:
         # directly output json if asked with 'application/json'
         if outputType == 'application/json':
-            httpResponse = httpClass(body=jsonContent, content_type='application/json', **httpKWArgs)
+            httpResponse = httpClass(body=jsonContent, content_type='application/json; charset=UTF-8', **httpKWArgs)
 
         # otherwise json is contained within the html <body> section
         elif outputType == 'text/html':
@@ -363,4 +368,4 @@ def isclass(obj):
     :param obj: object to evaluate for class type
     :return: (bool) indicating if `object` is a class
     """
-    return isinstance(obj, (type, types.ClassType))
+    return isinstance(obj, (type, six.class_types))

magpie/api/api_generic.py

@@ -0,0 +1,53 @@
+from magpie.definitions.pyramid_definitions import *
+from magpie.api.api_except import *
+from magpie.api.api_rest_schemas import *
+from magpie import __meta__, db
+
+
+@VersionAPI.get(tags=[APITag], api_security=SecurityEveryoneAPI, response_schemas=Version_GET_responses)
+@view_config(route_name='version', request_method='GET', permission=NO_PERMISSION_REQUIRED)
+def get_version(request):
+    """
+    Version information of the API.
+    """
+    return valid_http(httpSuccess=HTTPOk,
+                      content={u'version': __meta__.__version__, u'db_version': db.get_database_revision(request.db)},
+                      detail=Version_GET_OkResponseSchema.description, contentType='application/json')
+
+
+@notfound_view_config()
+def not_found(request):
+    content = get_request_info(request, default_msg=NotFoundResponseSchema.description)
+    return raise_http(nothrow=True, httpError=HTTPNotFound, contentType='application/json',
+                      detail=content['detail'], content=content)
+
+
+@exception_view_config()
+def internal_server_error(request):
+    content = get_request_info(request, default_msg=InternalServerErrorResponseSchema.description)
+    return raise_http(nothrow=True, httpError=HTTPInternalServerError, contentType='application/json',
+                      detail=content['detail'], content=content)
+
+
+@forbidden_view_config()
+def unauthorized_access(request):
+    # if not overridden, default is HTTPForbidden [403], which is for a slightly different situation
+    # this better reflects the HTTPUnauthorized [401] user access with specified AuthZ headers
+    # [http://www.restapitutorial.com/httpstatuscodes.html]
+    content = get_request_info(request, default_msg=UnauthorizedResponseSchema.description)
+    return raise_http(nothrow=True, httpError=HTTPUnauthorized, contentType='application/json',
+                      detail=content['detail'], content=content)
+
+
+def get_request_info(request, default_msg="undefined"):
+    content = {u'route_name': str(request.upath_info), u'request_url': str(request.url), u'detail': default_msg}
+    if hasattr(request, 'exception'):
+        if hasattr(request.exception, 'json'):
+            if type(request.exception.json) is dict:
+                content.update(request.exception.json)
+        elif isinstance(request.exception, HTTPServerError) and hasattr(request.exception, 'message'):
+            content.update({u'exception': str(request.exception.message)})
+    elif hasattr(request, 'matchdict'):
+        if request.matchdict is not None and request.matchdict != '':
+            content.update(request.matchdict)
+    return content