Run tests in randome order (#1799)

setup.cfg

@@ -96,6 +96,7 @@ testing =
     pytest-xdist >= 1.31.0
     pytest-mock >= 2
     pytest-env >= 0.6.2
+    pytest-randomly >= 1
     xonsh >= 0.9.16; python_version > '3.4'
 
 [options.package_data]

tests/conftest.py

@@ -24,6 +24,19 @@ def pytest_addoption(parser):
     parser.addoption("--int", action="store_true", default=False, help="run integration tests")
 
 
+def pytest_configure(config):
+    """Ensure randomly is called before we re-order"""
+    manager = config.pluginmanager
+    # noinspection PyProtectedMember
+    order = manager.hook.pytest_collection_modifyitems._nonwrappers
+    dest = next((i for i, p in enumerate(order) if p.plugin is manager.getplugin("randomly")), None)
+    if dest is not None:
+        from_pos = next(i for i, p in enumerate(order) if p.plugin is manager.getplugin(__file__))
+        temp = order[dest]
+        order[dest] = order[from_pos]
+        order[from_pos] = temp
+
+
 def pytest_collection_modifyitems(config, items):
     int_location = os.path.join("tests", "integration", "").rstrip()
     if len(items) == 1: