Merge pull request #620 from karandesai-96/integration-tests-automation

Runner script to automate execution of integration tests.

setup.py

@@ -106,6 +106,10 @@
     hook_func = 'astropy.utils.release:' + '_'.join(hook)
     entry_points[hook_ep] = ['%s = %s' % (hook_name, hook_func)]
 
+entry_points['console_scripts'] = [
+    'tardis_test_runner = tardis.tests.integration_tests.runner:run_tests'
+]
+
 #from Cython.Build import cythonize
 #package_info['ext_modules'] = cythonize(package_info['ext_modules'])
 # Include all .c files, recursively, including those generated by

tardis/conftest.py

@@ -70,6 +70,8 @@ def pytest_addoption(parser):
                      help="path to configuration file for integration tests")
     parser.addoption("--generate-reference", action="store_true", default=False,
                      help="execute integration test run to generate reference data")
+    parser.addoption("--less-packets", action="store_true", default=False,
+                     help="Run integration tests with less packets.")
 
 
 # -------------------------------------------------------------------------

tardis/tests/tests_slow/at/config.yml → tardis/tests/integration_tests/at/config.yml

@@ -24,10 +24,8 @@ montecarlo:
     threshold: 0.05
     type: specific
   iterations: 20
-# originally 5.0e+5
-  last_no_of_packets: 5.0e+3
-# originally 5.0e+4
-  no_of_packets: 5.0e+2
+  last_no_of_packets: 5.0e+5
+  no_of_packets: 5.0e+4
   no_of_virtual_packets: 3
   nthreads: 16
   seed: 23111963

tardis/tests/tests_slow/conftest.py → tardis/tests/integration_tests/conftest.py

@@ -5,8 +5,8 @@
 import pytest
 
 from tardis import __githash__ as tardis_githash
-from tardis.tests.tests_slow.report import DokuReport
-from tardis.tests.tests_slow.plot_helpers import PlotUploader
+from tardis.tests.integration_tests.report import DokuReport
+from tardis.tests.integration_tests.plot_helpers import PlotUploader
 
 
 def pytest_configure(config):
@@ -15,7 +15,7 @@ def pytest_configure(config):
         integration_tests_configpath = os.path.expandvars(
             os.path.expanduser(integration_tests_configpath)
         )
-        config.option.integration_tests_config = yaml.load(
+        config.integration_tests_config = yaml.load(
             open(integration_tests_configpath))
 
         if not config.getoption("--generate-reference"):
@@ -24,7 +24,7 @@ def pytest_configure(config):
             # prevent opening dokupath on slave nodes (xdist)
             if not hasattr(config, 'slaveinput'):
                 config.dokureport = DokuReport(
-                    config.option.integration_tests_config['dokuwiki'])
+                    config.integration_tests_config['dokuwiki'])
                 config.pluginmanager.register(config.dokureport)
 
 
@@ -40,7 +40,7 @@ def pytest_terminal_summary(terminalreporter):
             terminalreporter.config.getvalue("integration-tests")):
         # TODO: Add a check whether generation was successful or not.
         terminalreporter.write_sep("-", "Generated reference data: {0}".format(os.path.join(
-            terminalreporter.config.option.integration_tests_config['generate_reference'],
+            terminalreporter.config.integration_tests_config['generate_reference'],
             tardis_githash[:7]
         )))
 
@@ -67,7 +67,7 @@ def plot_object(request):
         os.path.dirname(os.path.realpath(__file__)), "*")) if os.path.isdir(path)
 ])
 def data_path(request):
-    integration_tests_config = request.config.option.integration_tests_config
+    integration_tests_config = request.config.integration_tests_config
     hdf_filename = "{0}.h5".format(os.path.basename(request.param))
 
     path = {

tardis/tests/integration_tests/runner.py

@@ -0,0 +1,68 @@
+import argparse
+import datetime
+import json
+import logging
+import subprocess
+import time
+import yaml
+
+import dokuwiki
+import requests
+
+
+logger = logging.getLogger(__name__)
+
+parser = argparse.ArgumentParser(description="Run slow integration tests")
+parser.add_argument("--integration-tests", dest="yaml_filepath",
+                    help="Path to YAML config file for integration tests.")
+parser.add_argument("--atomic-dataset", dest="atomic_dataset",
+                    help="Path to atomic dataset.")
+parser.add_argument("--less-packets", action="store_true", default=False,
+                    help="Run integration tests with less packets.")
+
+
+def run_tests():
+    args = parser.parse_args()
+
+    integration_tests_config = yaml.load(open(args.yaml_filepath))
+    doku_conn = dokuwiki.DokuWiki(
+        url=integration_tests_config['dokuwiki']['url'],
+        user=integration_tests_config['dokuwiki']['username'],
+        password=integration_tests_config['dokuwiki']['password']
+    )
+    less_packets = "--less-packets" if args.less_packets else ""
+    test_command = [
+        "python", "setup.py", "test",
+        "--test-path=tardis/tests/integration_tests/test_integration.py", "--args",
+        "--capture=no --integration-tests={0} --atomic-dataset={1} --remote-data "
+        "{2}".format(args.yaml_filepath, args.atomic_dataset, less_packets)
+    ]
+    subprocess.call(test_command)
+
+    while True:
+        # Request Github API and get githash of master on Github.
+        gh_request = requests.get(
+            "https://api.github.com/repos/tardis-sn/tardis/branches/master"
+        )
+        gh_master_head_data = json.loads(gh_request.content)
+        gh_tardis_githash = gh_master_head_data['commit']['sha'][:7]
+
+        # Check whether a report of this githash is uploaded on dokuwiki.
+        # If not, then this is a new commit and tests should be executed.
+        dokuwiki_report = doku_conn.pages.get(
+            "reports:{0}".format(gh_tardis_githash)
+        )
+
+        # If dokuwiki returns empty string, then it means that report has not
+        # been created yet.
+        if len(dokuwiki_report) == 0:
+            subprocess.call([
+                "git", "pull", "https://www.github.com/tardis-sn/tardis", "master"
+            ])
+            subprocess.call(test_command)
+        else:
+            checked = datetime.datetime.now()
+            logger.info("Up-to-date. Checked on {0} {1}".format(
+                checked.strftime("%d-%b-%Y"), checked.strftime("%H:%M:%S")
+            ))
+            time.sleep(600)

tardis/tests/tests_slow/test_integration.py → tardis/tests/integration_tests/test_integration.py

@@ -14,7 +14,7 @@
                     reason="integration tests are not included in this run")
 class TestIntegration(object):
     """Slow integration test for various setups present in subdirectories of
-    ``tardis/tests/tests_slow``.
+    ``tardis/tests/integration_tests``.
     """
 
     @classmethod
@@ -42,6 +42,16 @@ def setup(self, request, reference, data_path, atomic_data_fname):
         tardis_config = Configuration.from_yaml(
             self.config_file, atom_data=self.atom_data)
 
+        # Check whether current run is with less packets.
+        if request.config.getoption("--less-packets"):
+            less_packets = request.config.integration_tests_config['less_packets']
+            tardis_config['montecarlo']['no_of_packets'] = (
+                less_packets['no_of_packets']
+            )
+            tardis_config['montecarlo']['last_no_of_packets'] = (
+                less_packets['last_no_of_packets']
+            )
+
         # We now do a run with prepared config and get radial1d model.
         self.result = Radial1DModel(tardis_config)
 

tardis/tests/tests_slow/w7/config.yml → tardis/tests/integration_tests/w7/config.yml

@@ -16,10 +16,8 @@ montecarlo:
     start: 1 angstrom
     stop: 1000000 angstrom
   iterations: 20
-# originally 2.0e+5
-  last_no_of_packets: 2.0e+3
-# originally 4.0e+4
-  no_of_packets: 4.0e+2
+  last_no_of_packets: 2.0e+5
+  no_of_packets: 4.0e+4
   no_of_virtual_packets: 10
   nthreads: 2
   seed: 23111963

tardis/tests/setup_package.py

@@ -2,5 +2,5 @@ def get_package_data():
     return {
         _ASTROPY_PACKAGE_NAME_ + '.tests': ['coveragerc', 'data/*.h5',
                                             'data/*.dat', 'data/*.npy',
-                                            'tests_slow/*/*.yml',
-                                            'tests_slow/*/*.dat']}
+                                            'integration_tests/*/*.yml',
+                                            'integration_tests/*/*.dat']}