tests for cli helpers + remove invalid files from docker image

.dockerignore

@@ -37,6 +37,11 @@ build
 dist
 pip-selfcheck.json
 
+# CI/Test/Deploy
+ci
+hooks
+tests
+
 # Project
 **/custom.ini
 **/config.yaml

Dockerfile

@@ -34,8 +34,11 @@ RUN apk update \
     && pip install --no-cache-dir -e $MAGPIE_DIR \
     && apk --purge del .build-deps
 
-# install app package source
-COPY ./ $MAGPIE_DIR
+# install app package source, avoid copying the rest
+COPY ./bin $MAGPIE_DIR/bin/
+COPY ./config/magpie.ini $MAGPIE_CONFIG_DIR/magpie.ini
+COPY ./env/*.env.example $MAGPIE_ENV_DIR/
+COPY ./magpie $MAGPIE_DIR/magpie/
 # equivalent of `make install` without conda env and pre-installed packages
 RUN pip install --no-dependencies -e $MAGPIE_DIR
 

HISTORY.rst

@@ -10,6 +10,13 @@ Features / Changes
 ~~~~~~~~~~~~~~~~~~~~~
 * Update this changelog to provide direct URL references to issues and tags from both `GitHub` and `Readthedocs`.
 * Add generic ``magpie_helper`` CLI and prefix others using ``magpie_`` to help finding them in environment.
+* Add minimal tests for CLI helpers to validate they can be found and called as intended
+  (`#74 <https://github.com/Ouranosinc/Magpie/issues/74>`_).
+* Add ``CLI`` tag for running specific tests related to helpers.
+
+Bug Fixes
+~~~~~~~~~~~~~~~~~~~~~
+* Remove some files from built docker image that shouldn't be there with more explicit ``COPY`` operations.
 
 `1.10.2 <https://github.com/Ouranosinc/Magpie/tree/1.10.2>`_ (2020-04-21)
 ------------------------------------------------------------------------------------

magpie/helpers/__init__.py

@@ -2,9 +2,13 @@
 import importlib
 import os
 import sys
+from typing import TYPE_CHECKING
 
 from magpie.__meta__ import __version__
 
+if TYPE_CHECKING:
+    from typing import Callable
+
 
 def magpie_helper_cli():
     """
@@ -26,12 +30,14 @@ def magpie_helper_cli():
             helper_name = module_item.replace(".py", "")
             helper_root = "magpie.helpers"
             helper_module = importlib.import_module("{}.{}".format(helper_root, helper_name), helper_root)
-            parser_maker = getattr(helper_module, "make_parser", None)
+            parser_maker = getattr(helper_module, "make_parser", None)  # type: Callable[[], argparse.ArgumentParser]
             helper_caller = getattr(helper_module, "main", None)
             if parser_maker and helper_caller:
                 # add help disabled otherwise conflicts with this main helper's help
                 helper_parser = parser_maker()
-                subparsers.add_parser(helper_name, parents=[helper_parser], add_help=False)
+                subparsers.add_parser(helper_name, parents=[helper_parser],
+                                      add_help=False, help=helper_parser.description,
+                                      description=helper_parser.description, usage=helper_parser.usage)
                 helpers[helper_name] = {"caller": helper_caller, "parser": helper_parser}
     args = sys.argv[1:]       # save as was parse args does, but we must provide them to subparser
     ns = parser.parse_args()  # if 'helper' is unknown, auto prints the help message with exit(2)

magpie/helpers/sync_resources.py

@@ -363,7 +363,7 @@ def setup_cron_logger(log_level=logging.INFO):
 def make_parser():
     # type: () -> argparse.ArgumentParser
     parser = argparse.ArgumentParser(description="Synchronize local and remote resources based on "
-                                                 "Magpie Service synchronization type.")
+                                                 "Magpie Service sync-type.")
     parser.add_argument("--log-level", "-l", dest="log_level", default="INFO",
                         help="Log level to employ (default: %(default)s).")
     parser.add_argument("--db", metavar="CONNECTION_URL", dest="db",

setup.cfg

@@ -82,6 +82,7 @@ markers =
 	remote: magpie tests running on remote instance specified by url
 	local: magpie tests running on local instance created by test suite
 	api: magpie API operations
+	cli: magpie CLI helpers
 	ui: magpie UI operations
 	utils: magpie utilities
 	functional: magpie functional operations

setup.py

@@ -204,7 +204,7 @@ def _extra_requirements(base_requirements, other_requirements):
             "magpie_create_users = magpie.helpers.create_users:main",
             "magpie_register_default_users = magpie.helpers.register_default_users:main",
             "magpie_register_providers = magpie.helpers.register_providers:main",
-            "magpie_run_database_migration = magpie.helpers.run_database_migration:main",
+            "magpie_run_db_migration = magpie.helpers.run_db_migration:main",
             "magpie_sync_resources = magpie.helpers.sync_resources:main",
         ],
     }

tests/runner.py

@@ -35,6 +35,7 @@ def filter_test_files(root, filename):
 MAGPIE_TEST_REMOTE = RunOptionDecorator("MAGPIE_TEST_REMOTE")
 MAGPIE_TEST_LOCAL = RunOptionDecorator("MAGPIE_TEST_LOCAL")
 MAGPIE_TEST_API = RunOptionDecorator("MAGPIE_TEST_API")
+MAGPIE_TEST_CLI = RunOptionDecorator("MAGPIE_TEST_CLI")
 MAGPIE_TEST_UI = RunOptionDecorator("MAGPIE_TEST_UI")
 MAGPIE_TEST_UTILS = RunOptionDecorator("MAGPIE_TEST_UTILS")
 MAGPIE_TEST_FUNCTIONAL = RunOptionDecorator("MAGPIE_TEST_FUNCTIONAL")   # operations sequence

tests/test_magpie_api.py

@@ -5,7 +5,7 @@
 test_magpie_api
 ----------------------------------
 
-Tests for ``magpie.api`` module.
+Tests for :mod:`magpie.api` module.
 """
 
 import unittest

tests/test_magpie_helpers.py

@@ -0,0 +1,117 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+test_magpie_helpers
+----------------------------------
+
+Tests for :mod:`magpie.helpers` module.
+"""
+
+import subprocess
+
+from tests import runner
+
+KNOWN_HELPERS = [
+    "create_users",
+    "register_default_users",
+    "register_providers",
+    "run_db_migration",
+    "sync_resources"
+]
+
+
+def run_and_get_output(command, trim=True):
+    proc = subprocess.Popen(command, shell=True, universal_newlines=True, stdout=subprocess.PIPE)
+    out, err = proc.communicate()
+    assert not err
+    out_lines = [line for line in out.splitlines() if not trim or (line and not line.startswith(" "))]
+    return out_lines
+
+
+@runner.MAGPIE_TEST_CLI
+@runner.MAGPIE_TEST_LOCAL
+def test_magpie_helper_help():
+    out_lines = run_and_get_output("magpie_helper --help", trim=False)
+    assert "usage: magpie_helper" in out_lines[0]
+    assert all([helper in out_lines[1] for helper in KNOWN_HELPERS])
+
+
+@runner.MAGPIE_TEST_CLI
+@runner.MAGPIE_TEST_LOCAL
+def test_magpie_create_users_help_via_magpie_helper():
+    out_lines = run_and_get_output("magpie_helper create_users --help")
+    assert "usage: magpie_helper create_users" in out_lines[0]
+    assert "Create users on a running Magpie instance" in out_lines[1]
+
+
+@runner.MAGPIE_TEST_CLI
+@runner.MAGPIE_TEST_LOCAL
+def test_magpie_create_users_help_directly():
+    out_lines = run_and_get_output("magpie_create_users --help")
+    assert "usage: magpie_create_users" in out_lines[0]
+    assert "Create users on a running Magpie instance" in out_lines[1]
+
+
+@runner.MAGPIE_TEST_CLI
+@runner.MAGPIE_TEST_LOCAL
+def test_magpie_register_default_users_help_via_magpie_helper():
+    out_lines = run_and_get_output("magpie_helper register_default_users --help")
+    assert "usage: magpie_helper register_default_users" in out_lines[0]
+    assert "Registers default users in Magpie" in out_lines[1]
+
+
+@runner.MAGPIE_TEST_CLI
+@runner.MAGPIE_TEST_LOCAL
+def test_magpie_register_default_users_help_directly():
+    out_lines = run_and_get_output("magpie_register_default_users --help")
+    assert "usage: magpie_register_default_users" in out_lines[0]
+    assert "Registers default users in Magpie" in out_lines[1]
+
+
+@runner.MAGPIE_TEST_CLI
+@runner.MAGPIE_TEST_LOCAL
+def test_magpie_register_providers_help_via_magpie_helper():
+    out_lines = run_and_get_output("magpie_helper register_providers --help")
+    assert "usage: magpie_helper register_providers" in out_lines[0]
+    assert "Register service providers into Magpie" in out_lines[1]
+
+
+@runner.MAGPIE_TEST_CLI
+@runner.MAGPIE_TEST_LOCAL
+def test_magpie_register_providers_help_directly():
+    out_lines = run_and_get_output("magpie_register_providers --help")
+    assert "usage: magpie_register_providers" in out_lines[0]
+    assert "Register service providers into Magpie" in out_lines[1]
+
+
+@runner.MAGPIE_TEST_CLI
+@runner.MAGPIE_TEST_LOCAL
+def test_magpie_run_db_migration_help_via_magpie_helper():
+    out_lines = run_and_get_output("magpie_helper run_db_migration --help")
+    assert "usage: magpie_helper run_db_migration" in out_lines[0]
+    assert "Run Magpie database migration." in out_lines[1]
+
+
+@runner.MAGPIE_TEST_CLI
+@runner.MAGPIE_TEST_LOCAL
+def test_magpie_run_db_migration_help_directly():
+    out_lines = run_and_get_output("magpie_run_db_migration --help")
+    assert "usage: magpie_run_db_migration" in out_lines[0]
+    assert "Run Magpie database migration." in out_lines[1]
+
+
+@runner.MAGPIE_TEST_CLI
+@runner.MAGPIE_TEST_LOCAL
+def test_magpie_sync_resources_help_via_magpie_helper():
+    out_lines = run_and_get_output("magpie_helper sync_resources --help")
+    assert "usage: magpie_helper sync_resources" in out_lines[0]
+    assert "Synchronize local and remote resources based on Magpie Service sync-type" in out_lines[1]
+
+
+@runner.MAGPIE_TEST_CLI
+@runner.MAGPIE_TEST_LOCAL
+def test_magpie_sync_resources_help_directly():
+    out_lines = run_and_get_output("magpie_sync_resources --help")
+    assert "usage: magpie_sync_resources" in out_lines[0]
+    assert "Synchronize local and remote resources based on Magpie Service sync-type" in out_lines[1]

tests/test_magpie_ui.py

@@ -5,7 +5,7 @@
 test_magpie_ui
 ----------------------------------
 
-Tests for `magpie.ui` module.
+Tests for :mod:`magpie.ui` module.
 """
 
 import unittest