update helpers and docs

HISTORY.rst

@@ -9,6 +9,7 @@ History
 Features / Changes
 ~~~~~~~~~~~~~~~~~~~~~
 * Update this changelog to provide direct URL references to issues and tags from both `GitHub` and `Readthedocs`.
+* Add generic ``magpie_helper`` CLI and prefix others using ``magpie_`` to help finding them in environment.
 
 `1.10.2 <https://github.com/Ouranosinc/Magpie/tree/1.10.2>`_ (2020-04-21)
 ------------------------------------------------------------------------------------

docs/utilities.rst

@@ -13,14 +13,39 @@ configuration file or creating basic user accounts. Please refer to their corres
 
 Available helpers:
 
-- ``create_users``
-- ``register_default_users``
-- ``register_providers``
-- ``run_database_migration``
-- ``sync_resources``
+- ``magpie_create_users``
+- ``magpie_register_default_users``
+- ``magpie_register_providers``
+- ``magpie_run_database_migration``
+- ``magpie_sync_resources``
+
+For convenience, a generic CLI ``magpie_helper`` is also provided which allows calling each of the other helper
+operations as *mode*. You can therefore do as follows.
+
+.. code-block:: console
+
+    # list of available 'helper'
+    magpie_helper --help
+    # arguments of the given helper
+    magpie_helper [helper] --help
+
+
+For example, the two statements below are equivalent.
+
+.. code-block:: console
+
+    magpie_helper create_users [...]
+    # OR
+    magpie_create_users [...]
+
+
+When using an ``conda`` environment, you should be able to directly call the ``magpie_helper`` CLI as above if you
+previously installed the package (see `installation`_).
 
 Source code of these helpers can be found `here <https://github.com/Ouranosinc/Magpie/tree/master/magpie/helpers>`_.
 
+.. _installation: installation.rst
+
 .. utilities_connection:
 
 Magpie Connection

magpie/db.py

@@ -83,7 +83,7 @@ def _get(param, names):
 def get_engine(container=None, prefix="sqlalchemy.", **kwargs):
     # type: (Optional[AnySettingsContainer], Str, Any) -> Engine
     settings = get_settings(container or {})
-    settings[prefix + "url"] = get_db_url()
+    settings[prefix + "url"] = get_db_url(settings=settings)
     settings.setdefault(prefix + "pool_pre_ping", True)
     kwargs = kwargs or {}
     kwargs["convert_unicode"] = True
@@ -158,12 +158,12 @@ def get_database_revision(db_session):
     return result["version_num"]
 
 
-def is_database_ready(db_session=None):
-    # type: (Optional[Session]) -> bool
+def is_database_ready(db_session=None, container=None):
+    # type: (Optional[Session], Optional[AnySettingsContainer]) -> bool
     if isinstance(db_session, Session):
         engine = db_session.bind
     else:
-        engine = get_engine(dict())
+        engine = get_engine(container=container)
     inspector = Inspector.from_engine(engine)
     table_names = inspector.get_table_names()
 

magpie/helpers/__init__.py

@@ -24,7 +24,8 @@ def magpie_helper_cli():
         helper_path = os.path.join(helpers_dir, module_item)
         if os.path.isfile(helper_path) and "__init__" not in module_item and module_item.endswith(".py"):
             helper_name = module_item.replace(".py", "")
-            helper_module = importlib.import_module(helper_name, "magpie.helpers")
+            helper_root = "magpie.helpers"
+            helper_module = importlib.import_module("{}.{}".format(helper_root, helper_name), helper_root)
             parser_maker = getattr(helper_module, "make_parser", None)
             helper_caller = getattr(helper_module, "main", None)
             if parser_maker and helper_caller:
@@ -35,6 +36,9 @@ def magpie_helper_cli():
     args = sys.argv[1:]       # save as was parse args does, but we must provide them to subparser
     ns = parser.parse_args()  # if 'helper' is unknown, auto prints the help message with exit(2)
     helper_name = vars(ns).pop("helper")
+    if not helper_name:
+        parser.print_help()
+        return 0
     helper_args = args[1:]
     helper_caller = helpers[helper_name]["caller"]
     helper_parser = helpers[helper_name]["parser"]

magpie/helpers/register_providers.py

@@ -1,11 +1,17 @@
 import argparse
+from typing import TYPE_CHECKING
 
 from magpie.constants import MAGPIE_INI_FILE_PATH, MAGPIE_PROVIDERS_CONFIG_PATH
 from magpie.db import get_db_session_from_config_ini
 from magpie.register import magpie_register_services_from_config
 
+if TYPE_CHECKING:
+    # pylint: disable=W0611,unused-import
+    from typing import Any, AnyStr, Optional, Sequence  # noqa: F401
 
-def main():
+
+def make_parser():
+    # type: () -> argparse.ArgumentParser
     parser = argparse.ArgumentParser(description="Register service providers into Magpie and Phoenix")
     parser.add_argument("-c", "--config-file", metavar="config_file", dest="config_file",
                         type=str, default=MAGPIE_PROVIDERS_CONFIG_PATH,
@@ -20,14 +26,20 @@ def main():
                         help="push registered Magpie services to sync in Phoenix (default: %(default)s)")
     parser.add_argument("-d", "--use-db-session", default=False, action="store_true", dest="use_db_session",
                         help="update registered services using db session config instead of API (default: %(default)s)")
-    args = parser.parse_args()
+    return parser
+
 
+def main(args=None, parser=None, namespace=None):
+    # type: (Optional[Sequence[AnyStr]], Optional[argparse.ArgumentParser], Optional[argparse.Namespace]) -> Any
+    if not parser:
+        parser = make_parser()
+    args = parser.parse_args(args=args, namespace=namespace)
     db_session = None
     if args.use_db_session:
         db_session = get_db_session_from_config_ini(MAGPIE_INI_FILE_PATH)
-    magpie_register_services_from_config(args.config_file,
-                                         push_to_phoenix=args.phoenix_push, force_update=args.force_update,
-                                         disable_getcapabilities=args.no_getcapabilities, db_session=db_session)
+    return magpie_register_services_from_config(args.config_file,
+                                                push_to_phoenix=args.phoenix_push, force_update=args.force_update,
+                                                disable_getcapabilities=args.no_getcapabilities, db_session=db_session)
 
 
 if __name__ == "__main__":

magpie/helpers/sync_resources.py

@@ -1,8 +1,9 @@
 """
 Synchronize local and remote resources.
 
-To implement a new service, see the _SyncServiceInterface class.
+To implement a new service, see :class:`magpie.helpers.sync_services.SyncServiceInterface`.
 """
+import argparse
 import copy
 import datetime
 import logging
@@ -20,7 +21,7 @@
 if TYPE_CHECKING:
     # pylint: disable=W0611,unused-import
     from sqlalchemy.orm.session import Session
-    from magpie.typedefs import Optional  # noqa: F401
+    from typing import Any, AnyStr, Optional, Sequence, Union  # noqa: F401
 
 LOGGER = get_logger(__name__)
 
@@ -319,13 +320,13 @@ def fetch_single_service(service, session):
     _update_db(remote_resources, service_id, session)
 
 
-def fetch():
+def fetch(settings=None):
     """
     Main function to get all remote resources for each service and write to database.
     """
     with transaction.manager:
         LOGGER.info("Getting database session")
-        session = db.get_db_session_from_settings(echo=False)
+        session = db.get_db_session_from_settings(settings=settings, echo=False)
 
         for service_type in SYNC_SERVICES_TYPES:
             LOGGER.info("Fetching data for service type: %s", service_type)
@@ -334,8 +335,11 @@ def fetch():
         transaction.commit()
 
 
-def setup_cron_logger():
-    log_level = logging.INFO
+def setup_cron_logger(log_level=logging.INFO):
+    # type: (Union[AnyStr, int]) -> None
+
+    if not isinstance(log_level, int):
+        log_level = logging.getLevelName(log_level)
 
     log_path = constants.get_constant("MAGPIE_CRON_LOG")
     log_path = os.path.expandvars(log_path)
@@ -356,24 +360,40 @@ def setup_cron_logger():
     LOGGER.setLevel(log_level)
 
 
-def main():
+def make_parser():
+    # type: () -> argparse.ArgumentParser
+    parser = argparse.ArgumentParser(description="Synchronize local and remote resources based on "
+                                                 "Magpie Service synchronization type.")
+    parser.add_argument("--log-level", "-l", dest="log_level", default="INFO",
+                        help="Log level to employ (default: %(default)s).")
+    parser.add_argument("--db", metavar="CONNECTION_URL", dest="db",
+                        help="Magpie database URL to connect to. Otherwise employ typical environment variables.")
+    return parser
+
+
+def main(args=None, parser=None, namespace=None):
+    # type: (Optional[Sequence[AnyStr]], Optional[argparse.ArgumentParser], Optional[argparse.Namespace]) -> Any
     """
     Main entry point for cron service.
     """
     global CRON_SERVICE  # pylint: disable=W0603,global-statement
     CRON_SERVICE = True
 
-    setup_cron_logger()
+    if not parser:
+        parser = make_parser()
+    args = parser.parse_args(args=args, namespace=namespace)
+    settings = {"magpie.db_url": args.db} if args.db else None
 
+    setup_cron_logger(args.log_level)
     LOGGER.info("Magpie cron started.")
 
     try:
-        db_ready = db.is_database_ready()
+        db_ready = db.is_database_ready(container=settings)
         if not db_ready:
             LOGGER.info("Database isn't ready")
             return
         LOGGER.info("Starting to fetch data for all service types")
-        fetch()
+        fetch(settings=settings)
     except Exception:
         LOGGER.exception("An error occurred")
         raise

setup.py

@@ -200,7 +200,7 @@ def _extra_requirements(base_requirements, other_requirements):
             "main = magpie.app:main"
         ],
         "console_scripts": [
-            "magpie_helper = magpie.helpers:magpie_helper_cli"  # redirect to others below
+            "magpie_helper = magpie.helpers:magpie_helper_cli",  # redirect to others below
             "magpie_create_users = magpie.helpers.create_users:main",
             "magpie_register_default_users = magpie.helpers.register_default_users:main",
             "magpie_register_providers = magpie.helpers.register_providers:main",