WIP/RFC: Add base for testing via pytest directly

Makefile

@@ -1,5 +1,5 @@
 test:
-	pytest test_*.py
+	pytest
 
 build:
 	mkdir $@

setup.cfg

@@ -1,2 +1,5 @@
+[tool:pytest]
+testpaths = test
+
 [flake8]
 max-line-length = 100

test/test_integration.py

@@ -0,0 +1,60 @@
+"""Runs tests from ./vspec in vim-vspec."""
+import os
+import subprocess
+import urllib.request
+import zipfile
+
+import pytest
+
+VSPEC_URL = 'https://github.com/kana/vim-vspec/archive/1.4.1.zip'
+root = os.path.dirname(os.path.dirname(__file__))
+CACHE_FOLDER = os.path.join(root, 'build')
+VSPEC_FOLDER = os.path.join(CACHE_FOLDER, 'vim-vspec-1.4.1')
+VSPEC_RUNNER = os.path.join(VSPEC_FOLDER, 'bin/vspec')
+TEST_DIR = os.path.join(root, 'test', 'vspec')
+
+
+@pytest.fixture(scope='session')
+def install_vspec(request):
+    if not os.path.isdir(CACHE_FOLDER):
+        os.mkdir(CACHE_FOLDER)
+
+    if not os.path.exists(VSPEC_FOLDER):
+        name, hdrs = urllib.request.urlretrieve(VSPEC_URL)
+        z = zipfile.ZipFile(name)
+        for n in z.namelist():
+            dest = os.path.join(CACHE_FOLDER, n)
+            destdir = os.path.dirname(dest)
+            if not os.path.isdir(destdir):
+                os.makedirs(destdir)
+            data = z.read(n)
+            if not os.path.isdir(dest):
+                with open(dest, 'wb') as f:
+                    f.write(data)
+        z.close()
+        os.chmod(VSPEC_RUNNER, 0o777)
+
+
+def get_vspec_tests():
+    for f in os.listdir(TEST_DIR):
+        yield os.path.relpath(os.path.join(TEST_DIR, f))
+
+
+@pytest.mark.parametrize('path', get_vspec_tests())
+def test_integration(install_vspec, path):
+    output = subprocess.check_output(
+        [VSPEC_RUNNER, '.', VSPEC_FOLDER, os.path.relpath(path, root)],
+        cwd=root,
+    )
+    had_ok = False
+    for line in output.splitlines():
+        if (line.startswith(b'not ok') or
+                line.startswith(b'Error') or
+                line.startswith(b'Bail out!')):
+            pytest.fail("{0} failed:\n{1}".format(
+                path, output.decode('utf-8')), pytrace=False)
+        if not had_ok and line.startswith(b'ok'):
+            had_ok = True
+    if not had_ok:
+        pytest.fail("{0} failed: no 'ok' found:\n{1}".format(
+            path, output.decode('utf-8')), pytrace=False)

test/test_units.py

@@ -0,0 +1,77 @@
+import subprocess
+import time
+import shlex
+
+import pytest
+
+
+class VimError(Exception):
+    pass
+
+
+class Vim:
+    def __init__(self, exe, servername):
+        self.base_args = [exe, '-u', 'NONE', '--not-a-term',
+                          '--servername', servername]
+        cmd = self.base_args + ['--startuptime', '/tmp/s']
+        print('Running: %s' % ' '.join(shlex.quote(x) for x in cmd))
+        self._server = self._popen_vim(cmd)
+        self._wait_for_server()
+
+    def _popen_vim(self, cmd):
+        return subprocess.Popen(cmd,
+                                stdout=subprocess.PIPE,
+                                stderr=subprocess.PIPE,
+                                stdin=subprocess.DEVNULL)
+
+    def _wait_for_server(self):
+        waited = 0
+        while not self._eval('1', ignore_error=True):
+            time.sleep(0.1)
+            waited += 0.1
+        print('waited: %.2f' % waited)
+
+    def _eval(self, expr, ignore_error=False):
+        proc = self._popen_vim(self.base_args + ['--remote-expr', expr])
+        (stdout, stderr) = proc.communicate()
+        if not ignore_error:
+            if stderr:
+                raise VimError('Error with --remote-expr: %s' % (stderr,))
+            assert proc.returncode == 0
+        return stdout.decode('utf-8').rstrip('\n')
+
+    def eval(self, expr):
+        return self._eval(expr)
+
+    def sendkeys(self, keys):
+        proc = self._popen_vim(self.base_args + ['--remote-send', keys])
+        (stdout, stderr) = proc.communicate()
+        if stderr:
+            raise VimError('Error with --remote-expr: %s' % (stderr,))
+        assert proc.returncode == 0
+
+
+@pytest.fixture(scope='session')
+def vim_exe():
+    return 'vim'
+
+
+@pytest.fixture(scope='session')
+def vim_servername():
+    return 'JEDI-VIM-TESTS'
+
+
+@pytest.fixture(scope='session')
+def vim(vim_exe, vim_servername):
+    return Vim(vim_exe, vim_servername)
+
+
+def test_smoke(vim):
+    """Tests that this mechanism works."""
+    assert vim.eval('1')
+    vim.sendkeys('ifoo')
+    assert vim.eval('getline(".")') == 'foo'
+    vim.sendkeys('dd')
+    assert vim.eval('getline(".")') == 'foodd'
+    vim.sendkeys(r'\<Esc>dd')
+    assert vim.eval('getline(".")') == ''