cli helpers

.dockerignore

@@ -37,6 +37,11 @@ build
 dist
 pip-selfcheck.json
 
+# CI/Test/Deploy
+ci
+hooks
+tests
+
 # Project
 **/custom.ini
 **/config.yaml

.pylintrc

@@ -365,7 +365,7 @@ function-naming-style=snake_case
 #function-rgx=
 
 # Good variable names which should always be accepted, separated by a comma.
-good-names=i,j,k,v,ex,x,y,z,f,h,db,kw,_
+good-names=i,j,k,v,ex,x,y,z,f,h,db,kw,ns,_
 
 # Include a hint for the correct naming format with invalid-name.
 include-naming-hint=no

Dockerfile

@@ -26,16 +26,19 @@ RUN apk update \
     && apk add --virtual .build-deps \
     gcc \
     libffi-dev \
-    python-dev \
+    python3-dev \
     py-pip \
     musl-dev \
     postgresql-dev \
     && pip install --no-cache-dir --upgrade -r requirements-sys.txt \
     && pip install --no-cache-dir -e $MAGPIE_DIR \
     && apk --purge del .build-deps
 
-# install app package source
-COPY ./ $MAGPIE_DIR
+# install app package source, avoid copying the rest
+COPY ./bin $MAGPIE_DIR/bin/
+COPY ./config/magpie.ini $MAGPIE_CONFIG_DIR/magpie.ini
+COPY ./env/*.env.example $MAGPIE_ENV_DIR/
+COPY ./magpie $MAGPIE_DIR/magpie/
 # equivalent of `make install` without conda env and pre-installed packages
 RUN pip install --no-dependencies -e $MAGPIE_DIR
 

HISTORY.rst

@@ -9,6 +9,15 @@ History
 Features / Changes
 ~~~~~~~~~~~~~~~~~~~~~
 * Update this changelog to provide direct URL references to issues and tags from both `GitHub` and `Readthedocs`.
+* Add generic ``magpie_helper`` CLI and prefix others using ``magpie_`` to help finding them in environment.
+* Add minimal tests for CLI helpers to validate they can be found and called as intended
+  (`#74 <https://github.com/Ouranosinc/Magpie/issues/74>`_).
+* Add ``CLI`` tag for running specific tests related to helpers.
+
+Bug Fixes
+~~~~~~~~~~~~~~~~~~~~~
+* Remove some files from built docker image that shouldn't be there with more explicit ``COPY`` operations.
+* Fix ``Dockerfile`` dependency of ``python3-dev`` causing build to fail.
 
 `1.10.2 <https://github.com/Ouranosinc/Magpie/tree/1.10.2>`_ (2020-04-21)
 ------------------------------------------------------------------------------------

docs/utilities.rst

@@ -13,14 +13,39 @@ configuration file or creating basic user accounts. Please refer to their corres
 
 Available helpers:
 
-- ``create_users``
-- ``register_default_users``
-- ``register_providers``
-- ``run_database_migration``
-- ``sync_resources``
+- ``magpie_create_users``
+- ``magpie_register_default_users``
+- ``magpie_register_providers``
+- ``magpie_run_database_migration``
+- ``magpie_sync_resources``
+
+For convenience, a generic CLI ``magpie_helper`` is also provided which allows calling each of the other helper
+operations as *mode*. You can therefore do as follows.
+
+.. code-block:: console
+
+    # list of available 'helper'
+    magpie_helper --help
+    # arguments of the given helper
+    magpie_helper [helper] --help
+
+
+For example, the two statements below are equivalent.
+
+.. code-block:: console
+
+    magpie_helper create_users [...]
+    # OR
+    magpie_create_users [...]
+
+
+When using an ``conda`` environment, you should be able to directly call the ``magpie_helper`` CLI as above if you
+previously installed the package (see `installation`_).
 
 Source code of these helpers can be found `here <https://github.com/Ouranosinc/Magpie/tree/master/magpie/helpers>`_.
 
+.. _installation: installation.rst
+
 .. utilities_connection:
 
 Magpie Connection

magpie/db.py

@@ -83,7 +83,7 @@ def _get(param, names):
 def get_engine(container=None, prefix="sqlalchemy.", **kwargs):
     # type: (Optional[AnySettingsContainer], Str, Any) -> Engine
     settings = get_settings(container or {})
-    settings[prefix + "url"] = get_db_url()
+    settings[prefix + "url"] = get_db_url(settings=settings)
     settings.setdefault(prefix + "pool_pre_ping", True)
     kwargs = kwargs or {}
     kwargs["convert_unicode"] = True
@@ -158,12 +158,12 @@ def get_database_revision(db_session):
     return result["version_num"]
 
 
-def is_database_ready(db_session=None):
-    # type: (Optional[Session]) -> bool
+def is_database_ready(db_session=None, container=None):
+    # type: (Optional[Session], Optional[AnySettingsContainer]) -> bool
     if isinstance(db_session, Session):
         engine = db_session.bind
     else:
-        engine = get_engine(dict())
+        engine = get_engine(container=container)
     inspector = Inspector.from_engine(engine)
     table_names = inspector.get_table_names()
 

magpie/helpers/__init__.py

@@ -0,0 +1,56 @@
+import argparse
+import importlib
+import os
+import sys
+from typing import TYPE_CHECKING
+
+from magpie.__meta__ import __version__
+
+if TYPE_CHECKING:
+    from typing import Callable
+
+
+def magpie_helper_cli():
+    """
+    Groups all sub-helper CLI listed in :py:mod:`magpie.helpers` as a common ``magpie_helper``.
+
+    Dispatches the provided arguments to the appropriate sub-helper CLI as requested. Each sub-helper CLI must implement
+    functions ``make_parser`` and ``main`` to generate the arguments and dispatch them to the corresponding caller.
+    """
+    parser = argparse.ArgumentParser(description="Execute Magpie helper operations.")
+    parser.add_argument("--version", action="version", version="%(prog)s {}".format(__version__),
+                        help="prints the version of the library and exits")
+    subparsers = parser.add_subparsers(title="Helper", dest="helper", description="Name of the helper to execute.")
+    helpers_dir = os.path.dirname(__file__)
+    helper_mods = os.listdir(helpers_dir)
+    helpers = dict()
+    for module_item in sorted(helper_mods):
+        helper_path = os.path.join(helpers_dir, module_item)
+        if os.path.isfile(helper_path) and "__init__" not in module_item and module_item.endswith(".py"):
+            helper_name = module_item.replace(".py", "")
+            helper_root = "magpie.helpers"
+            helper_module = importlib.import_module("{}.{}".format(helper_root, helper_name), helper_root)
+            parser_maker = getattr(helper_module, "make_parser", None)  # type: Callable[[], argparse.ArgumentParser]
+            helper_caller = getattr(helper_module, "main", None)
+            if parser_maker and helper_caller:
+                # add help disabled otherwise conflicts with this main helper's help
+                helper_parser = parser_maker()
+                subparsers.add_parser(helper_name, parents=[helper_parser],
+                                      add_help=False, help=helper_parser.description,
+                                      description=helper_parser.description, usage=helper_parser.usage)
+                helpers[helper_name] = {"caller": helper_caller, "parser": helper_parser}
+    args = sys.argv[1:]       # save as was parse args does, but we must provide them to subparser
+    ns = parser.parse_args()  # if 'helper' is unknown, auto prints the help message with exit(2)
+    helper_name = vars(ns).pop("helper")
+    if not helper_name:
+        parser.print_help()
+        return 0
+    helper_args = args[1:]
+    helper_caller = helpers[helper_name]["caller"]
+    helper_parser = helpers[helper_name]["parser"]
+    result = helper_caller(args=helper_args, parser=helper_parser, namespace=ns)
+    return 0 if result is None else result
+
+
+if __name__ == "__main__":
+    magpie_helper_cli()

magpie/helpers/create_users.py

@@ -5,9 +5,13 @@
 import logging
 import random
 import string
+from typing import TYPE_CHECKING
 
 import requests
 
+if TYPE_CHECKING:
+    from typing import Any, AnyStr, Optional, Sequence
+
 LOGGER = logging.getLogger(__name__)
 COLUMN_SIZE = 60
 
@@ -75,13 +79,21 @@ def delete_users(user_names, magpie_url, magpie_admin_user_name, magpie_admin_pa
     return users
 
 
-def main():
-    parser = argparse.ArgumentParser(description="Create users on Magpie")
+def make_parser():
+    # type: () -> argparse.ArgumentParser
+    parser = argparse.ArgumentParser(description="Create users on a running Magpie instance")
     parser.add_argument("url", help="url used to access the magpie service")
     parser.add_argument("user_name", help="admin username for magpie login")
     parser.add_argument("password", help="admin password for magpie login")
     parser.add_argument("emails", nargs="*", help="list of emails for users to be created")
-    args = parser.parse_args()
+    return parser
+
+
+def main(args=None, parser=None, namespace=None):
+    # type: (Optional[Sequence[AnyStr]], Optional[argparse.ArgumentParser], Optional[argparse.Namespace]) -> Any
+    if not parser:
+        parser = make_parser()
+    args = parser.parse_args(args=args, namespace=namespace)
 
     LOGGER.setLevel(logging.DEBUG)
     logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s", datefmt="%d-%b-%y %H:%M:%S")

magpie/helpers/register_default_users.py

@@ -1,3 +1,4 @@
+import argparse
 import logging
 import time
 from typing import TYPE_CHECKING
@@ -14,7 +15,9 @@
 
 if TYPE_CHECKING:
     # pylint: disable=W0611,unused-import
-    from magpie.typedefs import AnySettingsContainer, Str, Optional  # noqa: F401
+    from magpie.typedefs import AnySettingsContainer, Str  # noqa: F401
+    from typing import Any, AnyStr, Optional, Sequence  # noqa: F401
+
 LOGGER = get_logger(__name__)
 
 
@@ -119,8 +122,8 @@ def init_users_group(db_session, settings=None):
         print_log("MAGPIE_USERS_GROUP already initialized", level=logging.DEBUG)
 
 
-def register_default_users(db_session=None, settings=None):
-    # type: (Optional[Session], Optional[AnySettingsContainer]) -> None
+def register_default_users(db_session=None, settings=None, ini_file_path=None):
+    # type: (Optional[Session], Optional[AnySettingsContainer], Optional[AnyStr]) -> None
     """
     Registers in db every undefined default users and groups matching following variables :
 
@@ -130,7 +133,8 @@ def register_default_users(db_session=None, settings=None):
     - ``MAGPIE_ADMIN_USER``
     """
     if not isinstance(db_session, Session):
-        ini_file_path = get_constant("MAGPIE_INI_FILE_PATH", settings_container=settings)
+        if not ini_file_path:
+            ini_file_path = get_constant("MAGPIE_INI_FILE_PATH", settings_container=settings)
         db_session = db.get_db_session_from_config_ini(ini_file_path)
     if not db.is_database_ready(db_session):
         time.sleep(2)
@@ -143,5 +147,20 @@ def register_default_users(db_session=None, settings=None):
     db_session.close()
 
 
+def make_parser():
+    # type: () -> argparse.ArgumentParser
+    parser = argparse.ArgumentParser(description="Registers default users in Magpie")
+    parser.add_argument("ini_file_path", help="Path of the configuration INI file to use to retrieve required settings")
+    return parser
+
+
+def main(args=None, parser=None, namespace=None):
+    # type: (Optional[Sequence[AnyStr]], Optional[argparse.ArgumentParser], Optional[argparse.Namespace]) -> Any
+    if not parser:
+        parser = make_parser()
+    args = parser.parse_args(args=args, namespace=namespace)
+    return register_default_users(ini_file_path=args.ini_file_path)
+
+
 if __name__ == "__main__":
-    register_default_users()
+    main()

magpie/helpers/register_providers.py

@@ -1,11 +1,17 @@
 import argparse
+from typing import TYPE_CHECKING
 
 from magpie.constants import MAGPIE_INI_FILE_PATH, MAGPIE_PROVIDERS_CONFIG_PATH
 from magpie.db import get_db_session_from_config_ini
 from magpie.register import magpie_register_services_from_config
 
+if TYPE_CHECKING:
+    # pylint: disable=W0611,unused-import
+    from typing import Any, AnyStr, Optional, Sequence  # noqa: F401
 
-def main():
+
+def make_parser():
+    # type: () -> argparse.ArgumentParser
     parser = argparse.ArgumentParser(description="Register service providers into Magpie and Phoenix")
     parser.add_argument("-c", "--config-file", metavar="config_file", dest="config_file",
                         type=str, default=MAGPIE_PROVIDERS_CONFIG_PATH,
@@ -20,14 +26,20 @@ def main():
                         help="push registered Magpie services to sync in Phoenix (default: %(default)s)")
     parser.add_argument("-d", "--use-db-session", default=False, action="store_true", dest="use_db_session",
                         help="update registered services using db session config instead of API (default: %(default)s)")
-    args = parser.parse_args()
+    return parser
+
 
+def main(args=None, parser=None, namespace=None):
+    # type: (Optional[Sequence[AnyStr]], Optional[argparse.ArgumentParser], Optional[argparse.Namespace]) -> Any
+    if not parser:
+        parser = make_parser()
+    args = parser.parse_args(args=args, namespace=namespace)
     db_session = None
     if args.use_db_session:
         db_session = get_db_session_from_config_ini(MAGPIE_INI_FILE_PATH)
-    magpie_register_services_from_config(args.config_file,
-                                         push_to_phoenix=args.phoenix_push, force_update=args.force_update,
-                                         disable_getcapabilities=args.no_getcapabilities, db_session=db_session)
+    return magpie_register_services_from_config(args.config_file,
+                                                push_to_phoenix=args.phoenix_push, force_update=args.force_update,
+                                                disable_getcapabilities=args.no_getcapabilities, db_session=db_session)
 
 
 if __name__ == "__main__":

magpie/helpers/run_db_migration.py

@@ -1,18 +1,31 @@
 import argparse
+from typing import TYPE_CHECKING
 
 from magpie.constants import MAGPIE_INI_FILE_PATH
 from magpie.db import run_database_migration
 
+if TYPE_CHECKING:
+    # pylint: disable=W0611,unused-import
+    from typing import Any, AnyStr, Optional, Sequence  # noqa: F401
 
-def run_database_migration_cli():
-    parser = argparse.ArgumentParser(description="Run Magpie database migration")
+
+def make_parser():
+    # type: () -> argparse.ArgumentParser
+    parser = argparse.ArgumentParser(description="Run Magpie database migration.")
     parser.add_argument("-c", "--config-file", metavar="config_file", dest="config_file", type=str,
                         default=MAGPIE_INI_FILE_PATH,
-                        help="configuration file to employ for database connection settings "
+                        help="Configuration file to employ for database connection settings "
                              "(default: MAGPIE_INI_FILE_PATH='%(default)s)'")
-    args = parser.parse_args()
-    run_database_migration(settings={"magpie.ini_file_path": args.config_file})
+    return parser
+
+
+def main(args=None, parser=None, namespace=None):
+    # type: (Optional[Sequence[AnyStr]], Optional[argparse.ArgumentParser], Optional[argparse.Namespace]) -> Any
+    if not parser:
+        parser = make_parser()
+    args = parser.parse_args(args=args, namespace=namespace)
+    return run_database_migration(settings={"magpie.ini_file_path": args.config_file})
 
 
 if __name__ == "__main__":
-    run_database_migration_cli()
+    main()