Port to unittest

Things wrong with pytest:

1. It's trying to do too much and ends up getting in the way.
1.1. Most notably, rerunning tests is pretty broken - see
     pytest-dev/rerun-failures#51
2. Everything needs to be a fixture, instead of being compatible with
   argument-altering decorators.
3. Yet, parametrizing fixtures is an annoying chore.
4. Weird interaction with @patch and its arguments - they are in reversed
   order, which messes up quite a few things.

As a result, we're moving to the standard unittest, which looks similar
to nose. Here are the relevant differences:

- Writing tests:
  - Each test needs to be a member function, named `test_whatever()`.
  - Each test class needs to
    - Inherit from `unittest.TestCase`
    - Be called `ThingTest`
  - Instead of `yield`ing generated tests a `self.subTest()` is called
    in a loop: https://docs.python.org/3/library/unittest.html#distinguishing-test-iterations-using-subtests
- Running tests:
  - There are far fewer command line options and the biggest problem was `--ignore`.
    - We have used `--ignore` to omit libclang tests from that one CI
      run. Now we add a `load_tests()` to `ycmd/tests/clang/__init__.py`. Reference
      https://docs.python.org/3/library/unittest.html#load-tests-protocol

.github/workflows/ci.yml

@@ -73,7 +73,7 @@ jobs:
         echo -e "import coverage\ncoverage.process_startup()" > $(python -c "print(__import__('sysconfig').get_path('purelib'))")/sitecustomize.py
     - name: Run tests
       if: matrix.benchmark == false
-      run: python3 run_tests.py --no-parallel --quiet
+      run: python3 run_tests.py --quiet
     - name: Run benchmarks
       if: matrix.benchmark == true
       run: python3 benchmark.py --quiet
@@ -118,7 +118,7 @@ jobs:
     - name: Lint
       run: |
         YCM_TESTRUN=1 python3 build.py --clang-completer --clang-tidy --valgrind
-        python3 run_tests.py --valgrind --skip-build --no-flake8 --no-parallel --quiet
+        python3 run_tests.py --valgrind --skip-build --no-flake8 --quiet
 
   windows:
     strategy:
@@ -174,7 +174,7 @@ jobs:
       run: python3 benchmark.py --msvc ${{ matrix.msvc }} --quiet
     - name: Run tests
       if: matrix.benchmark == false
-      run: python3 run_tests.py --msvc ${{ matrix.msvc }} --no-parallel --quiet
+      run: python3 run_tests.py --msvc ${{ matrix.msvc }} --quiet
     - name: Upload coverage data
       if: matrix.benchmark == false
       run: codecov --name ${{ matrix.runs-on }}-${{ matrix.python-arch }} >null

.gitignore

@@ -88,8 +88,10 @@ third_party/clangd
 
 # Rust
 third_party/rust-analyzer
+ycmd/tests/rust/testdata/common/Cargo.lock
+ycmd/tests/rust/testdata/macro/Cargo.lock
 ycmd/tests/rust/testdata/common/target
-ycmd/tests/rust/testdata/formatting/target
+ycmd/tests/rust/testdata/macro/target
 
 # generic LSP
 third_party/generic_server/node_modules

.vimspector.json

@@ -30,11 +30,10 @@
 
         "debugOptions": [],
 
-        "module": "pytest",
+        "module": "unittest",
         "python": "${python}",
         "args": [
           "-v",
-          "--ignore=ycmd/tests/python/testdata",
           "${Test}"
         ],
         "env": {

TESTS.md

@@ -49,16 +49,16 @@ To run the full suite, just run `run_tests.py`. Options are:
   Windows only;
 * `--coverage`: generate code coverage data.
 
-Remaining arguments are passed to "pytest" directly. This means that you
+Remaining arguments are passed to "unittest" directly. This means that you
 can run a specific script or a specific test as follows:
 
 * Specific script: `./run_tests.py ycmd/tests/<module_name>.py`
-* Specific test: `./run_tests.py ycmd/tests/<module_name>.py:<function name>`
+* Specific test: `./run_tests.py -k <test name> ycmd/tests/<module_name>.py`
 
 For example:
 
 * `./run_tests.py ycmd/tests/subcommands_test.py`
-* `./run_tests.py ycmd/tests/subcommands_test.py:Subcommands_Basic_test`
+* `./run_tests.py -k Subcommands_Basic_test ycmd/tests/subcommands_test.py`
 
 NOTE: you must have UTF-8 support in your terminal when you do this, e.g.:
 
@@ -76,9 +76,8 @@ C++ coverage testing is available only on Linux/Mac and uses gcov.
 Stricly speaking, we use the `-coverage` option to your compiler, which in the
 case of GNU and LLVM compilers, generate gcov-compatible data.
 
-For Python, there's a coverage module which works nicely with `pytest`. This
-is very useful for highlighting areas of your code which are not covered by the
-automated integration tests.
+For Python, there's a coverage module. This is very useful for highlighting
+areas of your code which are not covered by the automated integration tests.
 
 Run it like this:
 
@@ -88,8 +87,7 @@ Run it like this:
 
 This will print a summary and generate HTML output in `./cover`.
 
-More information: https://coverage.readthedocs.org and
-https://pytest-cov.readthedocs.io/en/stable/
+More information: https://coverage.readthedocs.org
 
 ## Troubleshooting
 

run_tests.py

@@ -9,8 +9,7 @@
 import sys
 import urllib.request
 
-BASE_PYTEST_ARGS = [ '-v', '--color=yes' ]
-
+BASE_UNITTEST_ARGS = [ '-cb' ]
 DIR_OF_THIS_SCRIPT = p.dirname( p.abspath( __file__ ) )
 DIR_OF_THIRD_PARTY = p.join( DIR_OF_THIS_SCRIPT, 'third_party' )
 DIR_OF_WATCHDOG_DEPS = p.join( DIR_OF_THIRD_PARTY, 'watchdog_deps' )
@@ -59,7 +58,6 @@ def RunFlake8():
 
 # Newer completers follow a standard convention of:
 #  - build: --<completer>-completer
-#  - test directory: ycmd/tests/<completer>
 #  - no aliases.
 SIMPLE_COMPLETERS = [
   'clangd',
@@ -69,38 +67,30 @@ def RunFlake8():
 
 # More complex or legacy cases can specify all of:
 #  - build: flags to add to build.py to include this completer
-#  - test: flags to add to run_tests.py when _not_ testing this completer
 #  - aliases?: list of completer aliases for the --completers option
 COMPLETERS = {
   'cfamily': {
     'build': [ '--clang-completer' ],
-    'test': [ '--ignore=ycmd/tests/clang' ],
     'aliases': [ 'c', 'cpp', 'c++', 'objc', 'clang', ]
   },
   'cs': {
     'build': [ '--cs-completer' ],
-    'test': [ '--ignore=ycmd/tests/cs' ],
     'aliases': [ 'omnisharp', 'csharp', 'c#' ]
   },
   'javascript': {
     'build': [ '--js-completer' ],
-    'test': [ '--ignore=ycmd/tests/tern' ],
     'aliases': [ 'js', 'tern' ]
   },
   'typescript': {
     'build': [ '--ts-completer' ],
-    'test': [ '--ignore=ycmd/tests/javascript',
-              '--ignore=ycmd/tests/typescript' ],
     'aliases': [ 'ts' ]
   },
   'python': {
     'build': [],
-    'test': [ '--ignore=ycmd/tests/python' ],
     'aliases': [ 'jedi', 'jedihttp', ]
   },
   'java': {
     'build': [ '--java-completer' ],
-    'test': [ '--ignore=ycmd/tests/java' ],
     'aliases': [ 'jdt' ],
   },
 }
@@ -109,7 +99,6 @@ def RunFlake8():
 for completer in SIMPLE_COMPLETERS:
   COMPLETERS[ completer ] = {
     'build': [ '--{}-completer'.format( completer ) ],
-    'test': [ '--ignore=ycmd/tests/{}'.format( completer ) ],
   }
 
 
@@ -161,18 +150,15 @@ def ParseArguments():
   parser.add_argument( '--valgrind',
                        action = 'store_true',
                        help = 'Run tests inside valgrind.' )
-  parser.add_argument( '--no-parallel', action='store_true',
-                       help='Run tests in serial, default is to parallelize '
-                            'tests execution' )
 
-  parsed_args, pytests_args = parser.parse_known_args()
+  parsed_args, unittest_args = parser.parse_known_args()
 
   parsed_args.completers = FixupCompleters( parsed_args )
 
   if 'COVERAGE' in os.environ:
     parsed_args.coverage = ( os.environ[ 'COVERAGE' ] == 'true' )
 
-  return parsed_args, pytests_args
+  return parsed_args, unittest_args
 
 
 def FixupCompleters( parsed_args ):
@@ -228,26 +214,27 @@ def BuildYcmdLibs( args ):
     subprocess.check_call( build_cmd )
 
 
-def PytestValgrind( parsed_args, extra_pytests_args ):
-  pytests_args = BASE_PYTEST_ARGS
+def UnittestValgrind( parsed_args, extra_unittest_args ):
+  unittest_args = BASE_UNITTEST_ARGS
   if parsed_args.quiet:
-    pytests_args[ 0 ] = '-q'
+    unittest_args.append( '-q' )
 
-  if extra_pytests_args:
-    pytests_args.extend( extra_pytests_args )
+  if extra_unittest_args:
+    unittest_args.extend( extra_unittest_args )
   else:
-    pytests_args += glob.glob(
+    unittest_args += glob.glob(
       p.join( DIR_OF_THIS_SCRIPT, 'ycmd', 'tests', 'bindings', '*_test.py' ) )
-    pytests_args += glob.glob(
+    unittest_args += glob.glob(
       p.join( DIR_OF_THIS_SCRIPT, 'ycmd', 'tests', 'clang', '*_test.py' ) )
-    pytests_args += glob.glob(
+    unittest_args += glob.glob(
       p.join( DIR_OF_THIS_SCRIPT, 'ycmd', 'tests', '*_test.py' ) )
-    # Avoids needing all completers for a valgrind run
-    pytests_args += [ '-m', 'not valgrind_skip' ]
+    # # Avoids needing all completers for a valgrind run
+    # unittest_args += [ '-m', 'not valgrind_skip' ]
 
   new_env = os.environ.copy()
   new_env[ 'PYTHONMALLOC' ] = 'malloc'
   new_env[ 'LD_LIBRARY_PATH' ] = LIBCLANG_DIR
+  new_env[ 'YCM_VALGRIND_RUN' ] = '1'
   cmd = [ 'valgrind',
           '--gen-suppressions=all',
           '--error-exitcode=1',
@@ -257,36 +244,26 @@ def PytestValgrind( parsed_args, extra_pytests_args ):
           '--suppressions=' + p.join( DIR_OF_THIS_SCRIPT,
                                       'valgrind.suppressions' ) ]
   subprocess.check_call( cmd +
-                         [ sys.executable, '-m', 'pytest' ] +
-                         pytests_args,
+                         [ sys.executable, '-m', 'unittest' ] +
+                         unittest_args,
                          env = new_env )
 
 
-def PytestTests( parsed_args, extra_pytests_args ):
-  pytests_args = BASE_PYTEST_ARGS
-  if parsed_args.quiet:
-    pytests_args[ 0 ] = '-q'
+def UnittestTests( parsed_args, extra_unittest_args ):
+  prefer_regular = any( p.isfile( arg ) for arg in extra_unittest_args )
+  unittest_args = BASE_UNITTEST_ARGS
 
-  for key in COMPLETERS:
-    if key not in parsed_args.completers:
-      pytests_args.extend( COMPLETERS[ key ][ 'test' ] )
+  if not prefer_regular:
+    unittest_args += [ '-p', '*_test.py' ]
 
-  if parsed_args.coverage:
-    # We need to exclude the ycmd/tests/python/testdata directory since it
-    # contains Python files and its base name starts with "test".
-    pytests_args += [ '--ignore=ycmd/tests/python/testdata', '--cov=ycmd' ]
-
-  if not parsed_args.no_parallel:
-    # Execute tests in parallel with n workers where n = NUMCPUS. Tests are
-    # grouped by module for test functions and by class for test methods.Groups
-    # are distributed to available workers as whole units. This guarantees that
-    # all tests in a group run in the same process.
-    pytests_args += [ '-n', 'auto', '--dist', 'loadscope' ]
-
-  if extra_pytests_args:
-    pytests_args.extend( extra_pytests_args )
-  else:
-    pytests_args.append( p.join( DIR_OF_THIS_SCRIPT, 'ycmd' ) )
+  if parsed_args.quiet:
+    unittest_args.append( '-q' )
+
+  if extra_unittest_args:
+    unittest_args.extend( extra_unittest_args )
+  if not ( extra_unittest_args or prefer_regular ):
+    unittest_args.append( '-s' )
+    unittest_args.append( 'ycmd.tests' )
 
   env = os.environ.copy()
 
@@ -301,8 +278,16 @@ def PytestTests( parsed_args, extra_pytests_args ):
   else:
     env[ 'LD_LIBRARY_PATH' ] = LIBCLANG_DIR
 
-  subprocess.check_call( [ sys.executable, '-m', 'pytest' ] + pytests_args,
-                         env=env )
+  if parsed_args.coverage:
+    executable = [ sys.executable, '-m', 'coverage', 'run' ]
+  else:
+    executable = [ sys.executable ]
+
+  unittest = [ '-m', 'unittest' ]
+  if not prefer_regular:
+    unittest.append( 'discover' )
+
+  subprocess.check_call( executable + unittest + unittest_args, env=env )
 
 
 # On Windows, distutils.spawn.find_executable only works for .exe files
@@ -371,7 +356,7 @@ def SetUpJavaCompleter():
 
 
 def Main():
-  parsed_args, pytests_args = ParseArguments()
+  parsed_args, unittest_args = ParseArguments()
   if parsed_args.dump_path:
     print( os.environ[ 'PYTHONPATH' ] )
     sys.exit()
@@ -384,9 +369,9 @@ def Main():
     RunFlake8()
   BuildYcmdLibs( parsed_args )
   if parsed_args.valgrind:
-    PytestValgrind( parsed_args, pytests_args )
+    UnittestValgrind( parsed_args, unittest_args )
   else:
-    PytestTests( parsed_args, pytests_args )
+    UnittestTests( parsed_args, unittest_args )
 
 
 if __name__ == "__main__":

test_requirements.txt

@@ -6,8 +6,4 @@ WebTest               >= 2.0.20
 psutil                >= 5.6.6
 coverage              >= 4.2
 codecov               >= 2.0.5
-pytest
-pytest-cov
-pytest-rerunfailures
 requests
-pytest-xdist

valgrind.suppressions

@@ -0,0 +1,16 @@
+{
+   Valgrind bug (https://stackoverflow.com/questions/1542457/memory-leak-reported-by-valgrind-in-dlopen)
+   Memcheck:Leak
+   fun:malloc
+   ...
+   fun:dl_open_worker
+   fun:_dl_catch_exception
+   fun:_dl_open
+   fun:dlopen_doit
+   fun:_dl_catch_exception
+   fun:_dl_catch_error
+   fun:_dlerror_run
+   fun:dlopen@@GLIBC_2.2.5
+   fun:_PyImport_FindSharedFuncptr
+   fun:_PyImport_LoadDynamicModuleWithSpec
+}

ycmd/tests/__init__.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2016-2020 ycmd contributors
+# Copyright (C) 2016-2021 ycmd contributors
 #
 # This file is part of ycmd.
 #
@@ -15,10 +15,35 @@
 # You should have received a copy of the GNU General Public License
 # along with ycmd.  If not, see <http://www.gnu.org/licenses/>.
 
+import functools
 import os
-from ycmd.tests.conftest import * # noqa
+from ycmd.tests.test_utils import ClearCompletionsCache, IsolatedApp, SetUpApp
+
+shared_app = None
 
 
 def PathToTestFile( *args ):
   dir_of_current_script = os.path.dirname( os.path.abspath( __file__ ) )
   return os.path.join( dir_of_current_script, 'testdata', *args )
+
+
+def SharedYcmd( test ):
+  global shared_app
+  if shared_app is None:
+    shared_app = SetUpApp()
+
+  @functools.wraps( test )
+  def Wrapper( test_case_instance, *args, **kwargs ):
+    ClearCompletionsCache()
+    return test( test_case_instance, shared_app, *args, **kwargs )
+  return Wrapper
+
+
+def IsolatedYcmd( custom_options = {} ):
+  def Decorator( test ):
+    @functools.wraps( test )
+    def Wrapper( test_case_instance, *args, **kwargs ):
+      with IsolatedApp( custom_options ) as app:
+        test( test_case_instance, app, *args, **kwargs )
+    return Wrapper
+  return Decorator