From 9b71c78e0d4865b1373954b37fc2e572ad527685 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Mon, 11 Jun 2018 20:22:12 -0500 Subject: [PATCH 01/13] Import parser tests --- readthedocs/config/__init__.py | 0 readthedocs/config/tests/__init__.py | 0 readthedocs/config/tests/test_parser.py | 56 +++++++++++++++++++++++++ 3 files changed, 56 insertions(+) create mode 100644 readthedocs/config/__init__.py create mode 100644 readthedocs/config/tests/__init__.py create mode 100644 readthedocs/config/tests/test_parser.py diff --git a/readthedocs/config/__init__.py b/readthedocs/config/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/readthedocs/config/tests/__init__.py b/readthedocs/config/tests/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/readthedocs/config/tests/test_parser.py b/readthedocs/config/tests/test_parser.py new file mode 100644 index 00000000000..ff116b19f36 --- /dev/null +++ b/readthedocs/config/tests/test_parser.py @@ -0,0 +1,56 @@ +from __future__ import division, print_function, unicode_literals + +from io import StringIO + +from pytest import raises + +from readthedocs.config.parser import ParseError, parse + + +def test_parse_empty_config_file(): + buf = StringIO(u'') + with raises(ParseError): + parse(buf) + + +def test_parse_invalid_yaml(): + buf = StringIO(u'- - !asdf') + with raises(ParseError): + parse(buf) + + +def test_parse_bad_type(): + buf = StringIO(u'Hello') + with raises(ParseError): + parse(buf) + + +def test_parse_single_config(): + buf = StringIO(u'base: path') + config = parse(buf) + assert isinstance(config, list) + assert len(config) == 1 + assert config[0]['base'] == 'path' + + +def test_parse_empty_list(): + buf = StringIO(u'base: []') + config = parse(buf) + assert config[0]['base'] == [] + + +def test_parse_multiple_configs_in_one_file(): + buf = StringIO( + u''' +base: path +--- +base: other_path +name: second +nested: + works: true + ''') + configs = parse(buf) + assert isinstance(configs, list) + assert len(configs) == 2 + assert configs[0]['base'] == 'path' + assert configs[1]['nested'] == {'works': True} From 6b09483e3af215caa447a66a4516b1d6897ed11e Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Mon, 11 Jun 2018 20:25:31 -0500 Subject: [PATCH 02/13] Move code --- readthedocs/config/parser.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 readthedocs/config/parser.py diff --git a/readthedocs/config/parser.py b/readthedocs/config/parser.py new file mode 100644 index 00000000000..81200e8cce6 --- /dev/null +++ b/readthedocs/config/parser.py @@ -0,0 +1,28 @@ +from __future__ import division, print_function, unicode_literals + +import yaml + +__all__ = ('parse', 'ParseError') + + +class ParseError(Exception): + pass + + +def parse(stream): + """ + Take file-like object and return a list of project configurations. + + The files need be valid YAML and only contain mappings as documents. + Everything else raises a ``ParseError``. + """ + try: + configs = list(yaml.safe_load_all(stream)) + except yaml.YAMLError as error: + raise ParseError('YAML: {message}'.format(message=error)) + if not configs: + raise ParseError('Empty config') + for config in configs: + if not isinstance(config, dict): + raise ParseError('Expected mapping') + return configs From 43a3e193ed8d9258b44640957fcba7f9e87b4327 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Mon, 11 Jun 2018 20:43:43 -0500 Subject: [PATCH 03/13] Linter --- readthedocs/config/parser.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/readthedocs/config/parser.py b/readthedocs/config/parser.py index 81200e8cce6..7428da22055 100644 --- a/readthedocs/config/parser.py +++ b/readthedocs/config/parser.py @@ -1,3 +1,6 @@ +# -*- coding: utf-8 -*- +"""YAML parser for the RTD configuration file.""" + from __future__ import division, print_function, unicode_literals import yaml @@ -6,6 +9,9 @@ class ParseError(Exception): + + """Parser related errors.""" + pass From 6352a617da1f9381c0e079e99aaeb0ebb800e53f Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Wed, 13 Jun 2018 14:47:49 -0500 Subject: [PATCH 04/13] Move utils module --- readthedocs/config/tests/test_utils.py | 26 ++++++++++++++++++++++++++ readthedocs/config/tests/utils.py | 13 +++++++++++++ 2 files changed, 39 insertions(+) create mode 100644 readthedocs/config/tests/test_utils.py create mode 100644 readthedocs/config/tests/utils.py diff --git a/readthedocs/config/tests/test_utils.py b/readthedocs/config/tests/test_utils.py new file mode 100644 index 00000000000..b4c1688847a --- /dev/null +++ b/readthedocs/config/tests/test_utils.py @@ -0,0 +1,26 @@ +from .utils import apply_fs + + +def test_apply_fs_with_empty_contents(tmpdir): + # Doesn't do anything if second paramter is empty. + apply_fs(tmpdir, {}) + assert tmpdir.listdir() == [] + + +def test_apply_fs_create_empty_file(tmpdir): + # Create empty file. + apply_fs(tmpdir, {'file': ''}) + assert len(tmpdir.listdir()) == 1 + assert tmpdir.join('file').read() == '' + + +def test_apply_fs_create_file_with_content(tmpdir): + # Create file with content. + apply_fs(tmpdir, {'file': 'content'}) + assert tmpdir.join('file').read() == 'content' + + +def test_apply_fs_create_subdirectory(tmpdir): + # Create file with content. + apply_fs(tmpdir, {'subdir': {'file': 'content'}}) + assert tmpdir.join('subdir', 'file').read() == 'content' diff --git a/readthedocs/config/tests/utils.py b/readthedocs/config/tests/utils.py new file mode 100644 index 00000000000..f8a15038b49 --- /dev/null +++ b/readthedocs/config/tests/utils.py @@ -0,0 +1,13 @@ +def apply_fs(tmpdir, contents): + """ + Create the directory structure specified in ``contents``. It's a dict of + filenames as keys and the file contents as values. If the value is another + dict, it's a subdirectory. + """ + for filename, content in contents.items(): + if hasattr(content, 'items'): + apply_fs(tmpdir.mkdir(filename), content) + else: + file = tmpdir.join(filename) + file.write(content) + return tmpdir From b736357be84e6647bf6d55bbd6cf635873c7f5f8 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Wed, 13 Jun 2018 15:00:38 -0500 Subject: [PATCH 05/13] Isort --- readthedocs/config/tests/test_utils.py | 2 ++ readthedocs/config/tests/utils.py | 3 +++ 2 files changed, 5 insertions(+) diff --git a/readthedocs/config/tests/test_utils.py b/readthedocs/config/tests/test_utils.py index b4c1688847a..6309a15f3f2 100644 --- a/readthedocs/config/tests/test_utils.py +++ b/readthedocs/config/tests/test_utils.py @@ -1,3 +1,5 @@ +from __future__ import division, print_function, unicode_literals + from .utils import apply_fs diff --git a/readthedocs/config/tests/utils.py b/readthedocs/config/tests/utils.py index f8a15038b49..b1b312420bb 100644 --- a/readthedocs/config/tests/utils.py +++ b/readthedocs/config/tests/utils.py @@ -1,3 +1,6 @@ +from __future__ import division, print_function, unicode_literals + + def apply_fs(tmpdir, contents): """ Create the directory structure specified in ``contents``. It's a dict of From 610d092f6b5c76e0c914fca22cda44ae101d61a8 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Wed, 13 Jun 2018 15:04:01 -0500 Subject: [PATCH 06/13] Move tests for find.py --- readthedocs/config/find.py | 6 ++ readthedocs/config/tests/test_find.py | 92 +++++++++++++++++++++++++++ 2 files changed, 98 insertions(+) create mode 100644 readthedocs/config/find.py create mode 100644 readthedocs/config/tests/test_find.py diff --git a/readthedocs/config/find.py b/readthedocs/config/find.py new file mode 100644 index 00000000000..98a21b49f71 --- /dev/null +++ b/readthedocs/config/find.py @@ -0,0 +1,6 @@ +def find_all(path, filenames): + pass + + +def find_one(path, filenames): + pass diff --git a/readthedocs/config/tests/test_find.py b/readthedocs/config/tests/test_find.py new file mode 100644 index 00000000000..a8b26d77af8 --- /dev/null +++ b/readthedocs/config/tests/test_find.py @@ -0,0 +1,92 @@ +import os + +import pytest +import six + +from readthedocs.config.find import find_all, find_one +from .utils import apply_fs + + +def test_find_no_files(tmpdir): + with tmpdir.as_cwd(): + paths = list(find_all(os.getcwd(), ('readthedocs.yml',))) + assert len(paths) == 0 + + +def test_find_at_root(tmpdir): + apply_fs(tmpdir, {'readthedocs.yml': '', 'otherfile.txt': ''}) + + base = str(tmpdir) + paths = list(find_all(base, ('readthedocs.yml',))) + assert paths == [ + os.path.abspath(os.path.join(base, 'readthedocs.yml')) + ] + + +def test_find_nested(tmpdir): + apply_fs(tmpdir, { + 'first': { + 'readthedocs.yml': '', + }, + 'second': { + 'confuser.txt': 'content', + }, + 'third': { + 'readthedocs.yml': 'content', + 'Makefile': '', + }, + }) + apply_fs(tmpdir, {'first/readthedocs.yml': ''}) + + base = str(tmpdir) + paths = list(find_all(base, ('readthedocs.yml',))) + assert set(paths) == set([ + str(tmpdir.join('first', 'readthedocs.yml')), + str(tmpdir.join('third', 'readthedocs.yml')), + ]) + + +def test_find_multiple_files(tmpdir): + apply_fs(tmpdir, { + 'first': { + 'readthedocs.yml': '', + '.readthedocs.yml': 'content', + }, + 'second': { + 'confuser.txt': 'content', + }, + 'third': { + 'readthedocs.yml': 'content', + 'Makefile': '', + }, + }) + apply_fs(tmpdir, {'first/readthedocs.yml': ''}) + + base = str(tmpdir) + paths = list(find_all(base, ('readthedocs.yml', + '.readthedocs.yml'))) + assert paths == [ + str(tmpdir.join('first', 'readthedocs.yml')), + str(tmpdir.join('first', '.readthedocs.yml')), + str(tmpdir.join('third', 'readthedocs.yml')), + ] + + paths = list(find_all(base, ('.readthedocs.yml', + 'readthedocs.yml'))) + assert paths == [ + str(tmpdir.join('first', '.readthedocs.yml')), + str(tmpdir.join('first', 'readthedocs.yml')), + str(tmpdir.join('third', 'readthedocs.yml')), + ] + + +@pytest.mark.skipif(not six.PY2, reason='Only for python2') +@pytest.mark.xfail(raises=UnicodeDecodeError) +def test_find_unicode_path(tmpdir): + base_path = os.path.abspath('integration_tests/bad_encode_project') + assert isinstance(base_path, str) + unicode_base_path = base_path.decode('utf-8') + assert isinstance(unicode_base_path, unicode) + path = find_one(unicode_base_path, ('readthedocs.yml',)) + assert path == '' + assert False, 'The UnicodeDecodeError was not raised' From a4bfff0807d2c864ec1fd84b0c19439652712957 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Wed, 13 Jun 2018 15:14:53 -0500 Subject: [PATCH 07/13] Move implementation from find.py --- readthedocs/config/find.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/readthedocs/config/find.py b/readthedocs/config/find.py index 98a21b49f71..daf1cec9b7d 100644 --- a/readthedocs/config/find.py +++ b/readthedocs/config/find.py @@ -1,6 +1,16 @@ +import os + + def find_all(path, filenames): - pass + path = os.path.abspath(path) + for root, dirs, files in os.walk(path, topdown=True): + dirs.sort() + for filename in filenames: + if filename in files: + yield os.path.abspath(os.path.join(root, filename)) def find_one(path, filenames): - pass + for _path in find_all(path, filenames): + return _path + return '' From 53172166d25c7bb8f361035b16c337b75c770ef5 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Wed, 13 Jun 2018 15:32:58 -0500 Subject: [PATCH 08/13] Add fixture --- .../fixtures/bad_encode_project/schaue p\303\274laylist an.py" | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 "readthedocs/config/tests/fixtures/bad_encode_project/schaue p\303\274laylist an.py" diff --git "a/readthedocs/config/tests/fixtures/bad_encode_project/schaue p\303\274laylist an.py" "b/readthedocs/config/tests/fixtures/bad_encode_project/schaue p\303\274laylist an.py" new file mode 100644 index 00000000000..e69de29bb2d From 81787be364856c2f5e8db1ac75bfe51098ee2024 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Wed, 13 Jun 2018 15:34:35 -0500 Subject: [PATCH 09/13] Fix path for test --- readthedocs/config/tests/test_find.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/readthedocs/config/tests/test_find.py b/readthedocs/config/tests/test_find.py index a8b26d77af8..834af3f8b0f 100644 --- a/readthedocs/config/tests/test_find.py +++ b/readthedocs/config/tests/test_find.py @@ -81,9 +81,11 @@ def test_find_multiple_files(tmpdir): @pytest.mark.skipif(not six.PY2, reason='Only for python2') -@pytest.mark.xfail(raises=UnicodeDecodeError) +@pytest.mark.xfail(raises=UnicodeDecodeError, strict=True) def test_find_unicode_path(tmpdir): - base_path = os.path.abspath('integration_tests/bad_encode_project') + base_path = os.path.abspath( + os.path.join(os.path.dirname(__file__), 'fixtures/bad_encode_project') + ) assert isinstance(base_path, str) unicode_base_path = base_path.decode('utf-8') assert isinstance(unicode_base_path, unicode) From 2e7a38f772ceab7c8d561815ed6bd195d3802859 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Wed, 13 Jun 2018 15:46:16 -0500 Subject: [PATCH 10/13] Fix test about encoding Probably the file was corrupted, the test pass with a proper unicode filename. See https://github.com/rtfd/readthedocs.org/issues/3732#issuecomment-371110187 --- readthedocs/config/tests/test_find.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/readthedocs/config/tests/test_find.py b/readthedocs/config/tests/test_find.py index 834af3f8b0f..0d2190cfd97 100644 --- a/readthedocs/config/tests/test_find.py +++ b/readthedocs/config/tests/test_find.py @@ -81,14 +81,14 @@ def test_find_multiple_files(tmpdir): @pytest.mark.skipif(not six.PY2, reason='Only for python2') -@pytest.mark.xfail(raises=UnicodeDecodeError, strict=True) def test_find_unicode_path(tmpdir): base_path = os.path.abspath( os.path.join(os.path.dirname(__file__), 'fixtures/bad_encode_project') ) assert isinstance(base_path, str) + path = find_one(base_path, ('readthedocs.yml',)) + assert path == '' unicode_base_path = base_path.decode('utf-8') assert isinstance(unicode_base_path, unicode) path = find_one(unicode_base_path, ('readthedocs.yml',)) assert path == '' - assert False, 'The UnicodeDecodeError was not raised' From 5455571a66500f660a0cbca34fcb6dad05a9cdf4 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Wed, 13 Jun 2018 15:55:34 -0500 Subject: [PATCH 11/13] Linter --- readthedocs/config/find.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/readthedocs/config/find.py b/readthedocs/config/find.py index daf1cec9b7d..df83d098785 100644 --- a/readthedocs/config/find.py +++ b/readthedocs/config/find.py @@ -1,7 +1,12 @@ +"""Helper functions to search files.""" + +from __future__ import division, print_function, unicode_literals + import os def find_all(path, filenames): + """Find all files in ``path`` that match in ``filenames``.""" path = os.path.abspath(path) for root, dirs, files in os.walk(path, topdown=True): dirs.sort() @@ -11,6 +16,7 @@ def find_all(path, filenames): def find_one(path, filenames): + """Find the first file in ``path`` that match in ``filenames``.""" for _path in find_all(path, filenames): return _path return '' From 2e45d9d13786f723cf0a77aef82f249119a5260c Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Wed, 13 Jun 2018 16:00:05 -0500 Subject: [PATCH 12/13] Isort --- readthedocs/config/tests/test_find.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/readthedocs/config/tests/test_find.py b/readthedocs/config/tests/test_find.py index 0d2190cfd97..8f420b5e8b1 100644 --- a/readthedocs/config/tests/test_find.py +++ b/readthedocs/config/tests/test_find.py @@ -1,9 +1,12 @@ +from __future__ import division, print_function, unicode_literals + import os import pytest import six from readthedocs.config.find import find_all, find_one + from .utils import apply_fs From 3523c451cf331aaafa7a779994a10be01a1eda87 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Wed, 13 Jun 2018 16:17:50 -0500 Subject: [PATCH 13/13] Remove wrong assert We are using unicode literals --- readthedocs/config/tests/test_find.py | 1 - 1 file changed, 1 deletion(-) diff --git a/readthedocs/config/tests/test_find.py b/readthedocs/config/tests/test_find.py index 8f420b5e8b1..3a74d027fbb 100644 --- a/readthedocs/config/tests/test_find.py +++ b/readthedocs/config/tests/test_find.py @@ -88,7 +88,6 @@ def test_find_unicode_path(tmpdir): base_path = os.path.abspath( os.path.join(os.path.dirname(__file__), 'fixtures/bad_encode_project') ) - assert isinstance(base_path, str) path = find_one(base_path, ('readthedocs.yml',)) assert path == '' unicode_base_path = base_path.decode('utf-8')