From f7dc3bd04f7daf13761f5d36e6e01d8a28d8a607 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Fri, 3 Aug 2018 18:30:14 -0500 Subject: [PATCH 01/20] Test conf.py default value --- .../tests/test_config_integration.py | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/readthedocs/rtd_tests/tests/test_config_integration.py b/readthedocs/rtd_tests/tests/test_config_integration.py index 1673cacf2b6..8444363d637 100644 --- a/readthedocs/rtd_tests/tests/test_config_integration.py +++ b/readthedocs/rtd_tests/tests/test_config_integration.py @@ -642,6 +642,35 @@ def test_sphinx_configuration_default( append_conf.assert_called_once() move.assert_called_once() + @patch('readthedocs.doc_builder.backends.sphinx.BaseSphinx.move') + @patch('readthedocs.doc_builder.backends.sphinx.BaseSphinx.append_conf') + @patch('readthedocs.doc_builder.backends.sphinx.BaseSphinx.run') + def test_sphinx_configuration_default( + self, run, append_conf, move, checkout_path, tmpdir): + """Should be default to find a conf.py file.""" + checkout_path.return_value = str(tmpdir) + + apply_fs(tmpdir, {'conf.py': ''}) + self.create_config_file(tmpdir, {}) + self.project.conf_py_file = '' + self.project.save() + + update_docs = self.get_update_docs_task() + config = update_docs.config + python_env = Virtualenv( + version=self.version, + build_env=update_docs.build_env, + config=config + ) + update_docs.python_env = python_env + + update_docs.build_docs_html() + + args, kwargs = run.call_args + assert kwargs['cwd'] == str(tmpdir) + append_conf.assert_called_once() + move.assert_called_once() + @patch('readthedocs.doc_builder.backends.sphinx.BaseSphinx.move') @patch('readthedocs.doc_builder.backends.sphinx.BaseSphinx.append_conf') @patch('readthedocs.doc_builder.backends.sphinx.BaseSphinx.run') From 09ea0dcb9474215c08b258b78fec609a96f557de Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Mon, 6 Aug 2018 16:07:46 -0500 Subject: [PATCH 02/20] Fix conf.py path --- readthedocs/doc_builder/config.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/readthedocs/doc_builder/config.py b/readthedocs/doc_builder/config.py index 289d0c85eb5..9d714af604f 100644 --- a/readthedocs/doc_builder/config.py +++ b/readthedocs/doc_builder/config.py @@ -31,7 +31,10 @@ def load_yaml_config(version): python_version = 3 if project.python_interpreter == 'python3' else 2 allow_v2 = project.has_feature(Feature.ALLOW_V2_CONFIG_FILE) try: - sphinx_configuration = version.get_conf_py_path() + sphinx_configuration = path.join( + version.get_conf_py_path(), + 'conf.py' + ) except ProjectConfigurationError: sphinx_configuration = None From 0648f65f12bf79d0e046498e52a2fe4a463b723b Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Mon, 6 Aug 2018 16:40:55 -0500 Subject: [PATCH 03/20] Don't skip tests --- readthedocs/rtd_tests/tests/test_config_integration.py | 1 - 1 file changed, 1 deletion(-) diff --git a/readthedocs/rtd_tests/tests/test_config_integration.py b/readthedocs/rtd_tests/tests/test_config_integration.py index 8444363d637..b9fbb4ed23a 100644 --- a/readthedocs/rtd_tests/tests/test_config_integration.py +++ b/readthedocs/rtd_tests/tests/test_config_integration.py @@ -232,7 +232,6 @@ def test_requirements_file(self, load_config): self.assertEqual(config.python.requirements, '__init__.py') -@pytest.mark.skip @pytest.mark.django_db @mock.patch('readthedocs.projects.models.Project.checkout_path') class TestLoadConfigV2(object): From e9db42ceabc4a5a3478456687491fca26426c0bd Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Mon, 6 Aug 2018 16:43:32 -0500 Subject: [PATCH 04/20] Expose config object in builders --- readthedocs/doc_builder/backends/sphinx.py | 7 +++++-- readthedocs/doc_builder/base.py | 1 + 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/readthedocs/doc_builder/backends/sphinx.py b/readthedocs/doc_builder/backends/sphinx.py index 00656f26543..901e7ecaf43 100644 --- a/readthedocs/doc_builder/backends/sphinx.py +++ b/readthedocs/doc_builder/backends/sphinx.py @@ -209,9 +209,12 @@ def build(self): '.', self.sphinx_build_dir, ]) + config_file = self.config.sphinx.configuration cmd_ret = self.run( - *build_command, cwd=project.conf_dir(self.version.slug), - bin_path=self.python_env.venv_bin()) + *build_command, + cwd=os.path.dirname(config_file), + bin_path=self.python_env.venv_bin() + ) return cmd_ret.successful diff --git a/readthedocs/doc_builder/base.py b/readthedocs/doc_builder/base.py index fcb70964709..1ee7df4ccb6 100644 --- a/readthedocs/doc_builder/base.py +++ b/readthedocs/doc_builder/base.py @@ -46,6 +46,7 @@ def __init__(self, build_env, python_env, force=False): self.python_env = python_env self.version = build_env.version self.project = build_env.project + self.config = python_env.config self._force = force self.target = self.project.artifact_path( version=self.version.slug, type_=self.type) From 7fe49ee674ba35e5e208985cad98c90034ae9dba Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Mon, 6 Aug 2018 16:43:56 -0500 Subject: [PATCH 05/20] Use doctype from config --- readthedocs/projects/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readthedocs/projects/tasks.py b/readthedocs/projects/tasks.py index 0ca28d358ef..d5db9297c32 100644 --- a/readthedocs/projects/tasks.py +++ b/readthedocs/projects/tasks.py @@ -701,7 +701,7 @@ def build_docs(self): def build_docs_html(self): """Build HTML docs.""" - html_builder = get_builder_class(self.project.documentation_type)( + html_builder = get_builder_class(self.config.doctype)( build_env=self.build_env, python_env=self.python_env, ) From 954e277cec5fad6d14ea661c6c80f4c86b1a8207 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Mon, 6 Aug 2018 16:44:25 -0500 Subject: [PATCH 06/20] Pass python_env in tests --- .../rtd_tests/tests/test_config_integration.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/readthedocs/rtd_tests/tests/test_config_integration.py b/readthedocs/rtd_tests/tests/test_config_integration.py index b9fbb4ed23a..55471724351 100644 --- a/readthedocs/rtd_tests/tests/test_config_integration.py +++ b/readthedocs/rtd_tests/tests/test_config_integration.py @@ -303,6 +303,12 @@ def test_build_formats_default_empty( self.create_config_file(tmpdir, config) update_docs = self.get_update_docs_task() + python_env = Virtualenv( + version=self.version, + build_env=update_docs.build_env, + config=config + ) + update_docs.python_env = python_env outcomes = update_docs.build_docs() # No extra formats were triggered @@ -327,6 +333,13 @@ def test_build_formats_only_pdf( self.create_config_file(tmpdir, {'formats': ['pdf']}) update_docs = self.get_update_docs_task() + python_env = Virtualenv( + version=self.version, + build_env=update_docs.build_env, + config=update_docs.config + ) + update_docs.python_env = python_env + outcomes = update_docs.build_docs() # Only pdf extra format was triggered From 292f912d9200505a2cad6304f537f2f69ac1a769 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Mon, 6 Aug 2018 18:00:46 -0500 Subject: [PATCH 07/20] Fix old test --- .../rtd_tests/tests/test_doc_builder.py | 49 ++++++++++++++++--- 1 file changed, 42 insertions(+), 7 deletions(-) diff --git a/readthedocs/rtd_tests/tests/test_doc_builder.py b/readthedocs/rtd_tests/tests/test_doc_builder.py index 34c7acfc834..77caf4ec6ea 100644 --- a/readthedocs/rtd_tests/tests/test_doc_builder.py +++ b/readthedocs/rtd_tests/tests/test_doc_builder.py @@ -6,9 +6,9 @@ import tempfile from collections import namedtuple +import mock import pytest import yaml -import mock from django.test import TestCase from django.test.utils import override_settings from django_dynamic_fixture import get @@ -17,6 +17,7 @@ from readthedocs.builds.models import Version from readthedocs.doc_builder.backends.mkdocs import BaseMkdocs, MkdocsHTML from readthedocs.doc_builder.backends.sphinx import BaseSphinx +from readthedocs.doc_builder.python_environments import Virtualenv from readthedocs.projects.exceptions import ProjectConfigurationError from readthedocs.projects.models import Project @@ -35,7 +36,16 @@ def setUp(self): BaseSphinx.type = 'base' BaseSphinx.sphinx_build_dir = tempfile.mkdtemp() - self.base_sphinx = BaseSphinx(build_env=build_env, python_env=None) + + python_env = Virtualenv( + version=self.version, + build_env=build_env, + config=None, + ) + self.base_sphinx = BaseSphinx( + build_env=build_env, + python_env=python_env, + ) @patch( 'readthedocs.doc_builder.backends.sphinx.SPHINX_TEMPLATE_DIR', @@ -95,9 +105,14 @@ def test_append_conf_create_yaml(self, checkout_path, run): os.mkdir(os.path.join(tmpdir, 'docs')) checkout_path.return_value = tmpdir + python_env = Virtualenv( + version=self.version, + build_env=self.build_env, + config=None, + ) self.searchbuilder = MkdocsHTML( build_env=self.build_env, - python_env=None + python_env=python_env, ) self.searchbuilder.append_conf() @@ -150,9 +165,14 @@ def test_append_conf_existing_yaml_on_root(self, checkout_path, run): ) checkout_path.return_value = tmpdir + python_env = Virtualenv( + version=self.version, + build_env=self.build_env, + config=None, + ) self.searchbuilder = MkdocsHTML( build_env=self.build_env, - python_env=None + python_env=python_env, ) self.searchbuilder.append_conf() @@ -204,9 +224,14 @@ def test_override_theme_new_style(self, checkout_path, run): ) checkout_path.return_value = tmpdir + python_env = Virtualenv( + version=self.version, + build_env=self.build_env, + config=None, + ) self.searchbuilder = MkdocsHTML( build_env=self.build_env, - python_env=None + python_env=python_env, ) self.searchbuilder.append_conf() @@ -237,9 +262,14 @@ def test_override_theme_old_style(self, checkout_path, run): ) checkout_path.return_value = tmpdir + python_env = Virtualenv( + version=self.version, + build_env=self.build_env, + config=None, + ) self.searchbuilder = MkdocsHTML( build_env=self.build_env, - python_env=None + python_env=python_env, ) self.searchbuilder.append_conf() @@ -268,9 +298,14 @@ def test_dont_override_theme(self, checkout_path, run): ) checkout_path.return_value = tmpdir + python_env = Virtualenv( + version=self.version, + build_env=self.build_env, + config=None, + ) self.searchbuilder = MkdocsHTML( build_env=self.build_env, - python_env=None + python_env=python_env, ) self.searchbuilder.append_conf() From 19810f4e66eca9d57da2a9165ec83f374eeee039 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Mon, 6 Aug 2018 18:01:12 -0500 Subject: [PATCH 08/20] Implement sphinx key --- readthedocs/config/config.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/readthedocs/config/config.py b/readthedocs/config/config.py index 0607fd3eea1..56e3e1058f0 100644 --- a/readthedocs/config/config.py +++ b/readthedocs/config/config.py @@ -526,6 +526,14 @@ def build(self): def doctype(self): return self.defaults['doctype'] + @property + def sphinx(self): + return Sphinx( + builder=self.doctype, + configuration=self.defaults['sphinx_configuration'], + fail_on_warning=False, + ) + class BuildConfigV2(BuildConfigBase): From fb48f7c6b3e3c8f6ee4fbcc00fe1e13cb55d3384 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Mon, 6 Aug 2018 18:06:02 -0500 Subject: [PATCH 09/20] Implement fail_on_warning --- readthedocs/doc_builder/backends/sphinx.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/readthedocs/doc_builder/backends/sphinx.py b/readthedocs/doc_builder/backends/sphinx.py index 901e7ecaf43..07f564094a0 100644 --- a/readthedocs/doc_builder/backends/sphinx.py +++ b/readthedocs/doc_builder/backends/sphinx.py @@ -199,6 +199,8 @@ def build(self): ] if self._force: build_command.append('-E') + if self.config.sphinx.fail_on_warning: + build_command.append('-W') build_command.extend([ '-b', self.sphinx_builder, From 10839997dc9901e542d9fbae3df097e8045dae8e Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Mon, 6 Aug 2018 18:11:03 -0500 Subject: [PATCH 10/20] Skip tests for mkdocs --- readthedocs/rtd_tests/tests/test_config_integration.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/readthedocs/rtd_tests/tests/test_config_integration.py b/readthedocs/rtd_tests/tests/test_config_integration.py index 55471724351..d8d6c3a4f71 100644 --- a/readthedocs/rtd_tests/tests/test_config_integration.py +++ b/readthedocs/rtd_tests/tests/test_config_integration.py @@ -757,6 +757,7 @@ def test_sphinx_fail_on_warning( append_conf.assert_called_once() move.assert_called_once() + @pytest.mark.skip @patch('readthedocs.doc_builder.backends.mkdocs.BaseMkdocs.move') @patch('readthedocs.doc_builder.backends.mkdocs.BaseMkdocs.append_conf') @patch('readthedocs.doc_builder.backends.mkdocs.BaseMkdocs.run') @@ -795,6 +796,7 @@ def test_mkdocs_configuration( append_conf.assert_called_once() move.assert_called_once() + @pytest.mark.skip @patch('readthedocs.doc_builder.backends.mkdocs.BaseMkdocs.move') @patch('readthedocs.doc_builder.backends.mkdocs.BaseMkdocs.append_conf') @patch('readthedocs.doc_builder.backends.mkdocs.BaseMkdocs.run') From d35c23c5119e80ba6c261c79ea144c6836852a23 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Mon, 6 Aug 2018 18:12:18 -0500 Subject: [PATCH 11/20] Fix raising wrong exception --- readthedocs/config/config.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/readthedocs/config/config.py b/readthedocs/config/config.py index 56e3e1058f0..41be20f2f07 100644 --- a/readthedocs/config/config.py +++ b/readthedocs/config/config.py @@ -967,7 +967,8 @@ def get_configuration_class(version): version = int(version) return configurations_class[version] except (KeyError, ValueError): - raise ConfigError( - 'Invalid version of the configuration file', + raise InvalidConfig( + 'version', code=VERSION_INVALID, + error_message='Invalid version of the configuration file', ) From 58adc35f8ecdf39e0d7dd291b37378461e763df3 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Tue, 7 Aug 2018 10:19:49 -0500 Subject: [PATCH 12/20] Fix tests --- .../rtd_tests/tests/test_doc_builder.py | 40 +++++++++++-------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/readthedocs/rtd_tests/tests/test_doc_builder.py b/readthedocs/rtd_tests/tests/test_doc_builder.py index 77caf4ec6ea..07fffeafd5e 100644 --- a/readthedocs/rtd_tests/tests/test_doc_builder.py +++ b/readthedocs/rtd_tests/tests/test_doc_builder.py @@ -30,23 +30,13 @@ def setUp(self): self.project = Project.objects.get(slug='pip') self.version = self.project.versions.first() - build_env = namedtuple('project', 'version') - build_env.project = self.project - build_env.version = self.version + self.build_env = namedtuple('project', 'version') + self.build_env.project = self.project + self.build_env.version = self.version BaseSphinx.type = 'base' BaseSphinx.sphinx_build_dir = tempfile.mkdtemp() - python_env = Virtualenv( - version=self.version, - build_env=build_env, - config=None, - ) - self.base_sphinx = BaseSphinx( - build_env=build_env, - python_env=python_env, - ) - @patch( 'readthedocs.doc_builder.backends.sphinx.SPHINX_TEMPLATE_DIR', '/tmp/sphinx-template-dir', @@ -57,7 +47,8 @@ def setUp(self): @patch('readthedocs.doc_builder.backends.sphinx.BaseSphinx.run') @patch('readthedocs.builds.models.Version.get_conf_py_path') @patch('readthedocs.builds.models.Project.conf_file') - def test_create_conf_py(self, conf_file, get_conf_py_path, _, get_config_params, create_index, docs_dir): + @patch('readthedocs.projects.models.Project.checkout_path') + def test_create_conf_py(self, checkout_path, conf_file, get_conf_py_path, _, get_config_params, create_index, docs_dir): """ Test for a project without ``conf.py`` file. @@ -69,20 +60,35 @@ def test_create_conf_py(self, conf_file, get_conf_py_path, _, get_config_params, any kind of exception raised by ``append_conf`` (we were originally having a ``TypeError`` because of an encoding problem in Python3) """ + checkout_path.return_value = tempfile.mkdtemp() docs_dir.return_value = tempfile.mkdtemp() create_index.return_value = 'README.rst' get_config_params.return_value = {} get_conf_py_path.side_effect = ProjectConfigurationError conf_file.return_value = tempfile.mktemp() + python_env = Virtualenv( + version=self.version, + build_env=self.build_env, + config=None, + ) + base_sphinx = BaseSphinx( + build_env=self.build_env, + python_env=python_env, + ) try: - self.base_sphinx.append_conf() + base_sphinx.append_conf() except Exception: pytest.fail('Exception was generated when append_conf called.') # Check the content generated by our method is the same than what we # expects from a pre-generated file - generated_conf_py = os.path.join(self.base_sphinx.docs_dir(), 'conf.py') - expected_conf_py = os.path.join(os.path.dirname(__file__), '..', 'files', 'conf.py') + generated_conf_py = os.path.join(base_sphinx.docs_dir(), 'conf.py') + expected_conf_py = os.path.join( + os.path.dirname(__file__), + '..', + 'files', + 'conf.py' + ) with open(generated_conf_py) as gf, open(expected_conf_py) as ef: self.assertEqual(gf.read(), ef.read()) From 819a9fadab6354e4e03113a36f7ca1c96f889ce7 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Fri, 24 Aug 2018 19:22:01 -0500 Subject: [PATCH 13/20] Remove search from tests --- readthedocs/rtd_tests/tests/test_config_integration.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/readthedocs/rtd_tests/tests/test_config_integration.py b/readthedocs/rtd_tests/tests/test_config_integration.py index d8d6c3a4f71..84031a4a744 100644 --- a/readthedocs/rtd_tests/tests/test_config_integration.py +++ b/readthedocs/rtd_tests/tests/test_config_integration.py @@ -289,12 +289,10 @@ def test_report_using_invalid_version(self, checkout_path, tmpdir): @pytest.mark.parametrize('config', [{}, {'formats': []}]) @patch('readthedocs.projects.models.Project.repo_nonblockinglock', new=MagicMock()) - @patch('readthedocs.doc_builder.backends.sphinx.SearchBuilder.build') @patch('readthedocs.doc_builder.backends.sphinx.HtmlBuilder.build') @patch('readthedocs.doc_builder.backends.sphinx.HtmlBuilder.append_conf') def test_build_formats_default_empty( - self, append_conf, html_build, search_build, - checkout_path, config, tmpdir): + self, append_conf, html_build, checkout_path, config, tmpdir): """ The default value for formats is [], which means no extra formats are build. @@ -313,18 +311,16 @@ def test_build_formats_default_empty( # No extra formats were triggered assert outcomes['html'] - assert outcomes['search'] assert not outcomes['localmedia'] assert not outcomes['pdf'] assert not outcomes['epub'] @patch('readthedocs.projects.models.Project.repo_nonblockinglock', new=MagicMock()) @patch('readthedocs.projects.tasks.UpdateDocsTaskStep.build_docs_class') - @patch('readthedocs.doc_builder.backends.sphinx.SearchBuilder.build') @patch('readthedocs.doc_builder.backends.sphinx.HtmlBuilder.build') @patch('readthedocs.doc_builder.backends.sphinx.HtmlBuilder.append_conf') def test_build_formats_only_pdf( - self, append_conf, html_build, search_build, build_docs_class, + self, append_conf, html_build, build_docs_class, checkout_path, tmpdir): """ Only the pdf format is build. @@ -344,7 +340,6 @@ def test_build_formats_only_pdf( # Only pdf extra format was triggered assert outcomes['html'] - assert outcomes['search'] build_docs_class.assert_called_with('sphinx_pdf') assert outcomes['pdf'] assert not outcomes['localmedia'] From e6b9dd5958ec8bf8d1c0e62223fec710dd565f7b Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Mon, 27 Aug 2018 12:54:16 -0500 Subject: [PATCH 14/20] Return the absolute path --- readthedocs/config/config.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/readthedocs/config/config.py b/readthedocs/config/config.py index 41be20f2f07..c38bb5d3dd0 100644 --- a/readthedocs/config/config.py +++ b/readthedocs/config/config.py @@ -530,7 +530,10 @@ def doctype(self): def sphinx(self): return Sphinx( builder=self.doctype, - configuration=self.defaults['sphinx_configuration'], + configuration=os.path.join( + self.base_path, + self.defaults['sphinx_configuration'], + ), fail_on_warning=False, ) From db40aa79522ca445b9fee4b14c12684dcdcb49ff Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Mon, 27 Aug 2018 15:33:07 -0500 Subject: [PATCH 15/20] Check for None --- readthedocs/config/config.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/readthedocs/config/config.py b/readthedocs/config/config.py index c38bb5d3dd0..f17fb88f13d 100644 --- a/readthedocs/config/config.py +++ b/readthedocs/config/config.py @@ -528,12 +528,12 @@ def doctype(self): @property def sphinx(self): + config_file = self.defaults['sphinx_configuration'] + if config_file is not None: + config_file = os.path.join(self.base_path, config_file) return Sphinx( builder=self.doctype, - configuration=os.path.join( - self.base_path, - self.defaults['sphinx_configuration'], - ), + configuration=config_file, fail_on_warning=False, ) From b4c22dd2965e75c80ad88786958278335354b625 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Mon, 27 Aug 2018 15:36:23 -0500 Subject: [PATCH 16/20] Use the config file --- readthedocs/doc_builder/backends/sphinx.py | 26 +++++++++++++++------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/readthedocs/doc_builder/backends/sphinx.py b/readthedocs/doc_builder/backends/sphinx.py index 07f564094a0..5f749d4c245 100644 --- a/readthedocs/doc_builder/backends/sphinx.py +++ b/readthedocs/doc_builder/backends/sphinx.py @@ -42,8 +42,12 @@ class BaseSphinx(BaseBuilder): def __init__(self, *args, **kwargs): super(BaseSphinx, self).__init__(*args, **kwargs) try: + config_file = self.config.sphinx.configuration + config_dir = os.path.dirname(config_file) if config_file else None self.old_artifact_path = os.path.join( - self.project.conf_dir(self.version.slug), self.sphinx_build_dir) + config_dir or self.project.conf_dir(self.version.slug), + self.sphinx_build_dir + ) except ProjectConfigurationError: docs_dir = self.docs_dir() self.old_artifact_path = os.path.join( @@ -69,7 +73,7 @@ def get_config_params(self): # TODO this should be handled better in the theme conf_py_path = os.path.join( os.path.sep, - self.version.get_conf_py_path(), + self.config.sphinx.configuration or self.version.get_conf_py_path(), '', ) remote_version = self.version.commit_name @@ -151,14 +155,15 @@ def get_config_params(self): def append_conf(self, **__): """Find or create a ``conf.py`` with a rendered ``doc_builder/conf.py.tmpl`` appended""" - try: - self.version.get_conf_py_path() - except ProjectConfigurationError: + if self.config.sphinx.configuration is None: master_doc = self.create_index(extension='rst') self._write_config(master_doc=master_doc) try: - outfile_path = self.project.conf_file(self.version.slug) + outfile_path = ( + self.config.sphinx.configuration or + self.project.conf_file(self.version.slug) + ) outfile = codecs.open(outfile_path, encoding='utf-8', mode='a') except (ProjectConfigurationError, IOError): trace = sys.exc_info()[2] @@ -211,7 +216,10 @@ def build(self): '.', self.sphinx_build_dir, ]) - config_file = self.config.sphinx.configuration + config_file = ( + self.config.sphinx.configuration or + self.project.conf_file(self.version.slug) + ) cmd_ret = self.run( *build_command, cwd=os.path.dirname(config_file), @@ -352,7 +360,9 @@ class PdfBuilder(BaseSphinx): def build(self): self.clean() - cwd = self.project.conf_dir(self.version.slug) + config_file = self.config.sphinx.configuration + config_dir = os.path.dirname(config_file) if config_file else None + cwd = config_dir or self.project.conf_dir(self.version.slug) # Default to this so we can return it always. self.run( From 6ee2d663e7671c49c19d586cb1ec8ed582514cdd Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Mon, 27 Aug 2018 15:36:37 -0500 Subject: [PATCH 17/20] Fix test --- readthedocs/rtd_tests/tests/test_config_integration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readthedocs/rtd_tests/tests/test_config_integration.py b/readthedocs/rtd_tests/tests/test_config_integration.py index 84031a4a744..3387c24dae9 100644 --- a/readthedocs/rtd_tests/tests/test_config_integration.py +++ b/readthedocs/rtd_tests/tests/test_config_integration.py @@ -304,7 +304,7 @@ def test_build_formats_default_empty( python_env = Virtualenv( version=self.version, build_env=update_docs.build_env, - config=config + config=update_docs.config ) update_docs.python_env = python_env outcomes = update_docs.build_docs() From aa25732643c5fd3c410191136b8a472c9cbaf663 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Mon, 27 Aug 2018 16:39:14 -0500 Subject: [PATCH 18/20] Fix test --- .../rtd_tests/tests/test_doc_builder.py | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/readthedocs/rtd_tests/tests/test_doc_builder.py b/readthedocs/rtd_tests/tests/test_doc_builder.py index 587e05699e6..478cb376cbc 100644 --- a/readthedocs/rtd_tests/tests/test_doc_builder.py +++ b/readthedocs/rtd_tests/tests/test_doc_builder.py @@ -102,8 +102,9 @@ def test_create_conf_py(self, checkout_path, conf_file, get_conf_py_path, _, get @patch('readthedocs.doc_builder.backends.sphinx.BaseSphinx.get_config_params') @patch('readthedocs.doc_builder.backends.sphinx.BaseSphinx.run') @patch('readthedocs.builds.models.Version.get_conf_py_path') - def test_create_conf_py( - self, get_conf_py_path, _, get_config_params, + @patch('readthedocs.projects.models.Project.checkout_path') + def test_multiple_conf_py( + self, checkout_path, get_conf_py_path, _, get_config_params, create_index, docs_dir): """ Test for a project with multiple ``conf.py`` files. @@ -113,14 +114,24 @@ def test_create_conf_py( """ tmp_docs_dir = py.path.local(tempfile.mkdtemp()) - tmp_docs_dir.join('conf.py').new() - tmp_docs_dir.join('test').mkdir().join('conf.py').new() + tmp_docs_dir.join('conf.py').write('') + tmp_docs_dir.join('test').mkdir().join('conf.py').write('') docs_dir.return_value = str(tmp_docs_dir) + checkout_path.return_value = str(tmp_docs_dir) create_index.return_value = 'README.rst' get_config_params.return_value = {} get_conf_py_path.side_effect = ProjectConfigurationError + python_env = Virtualenv( + version=self.version, + build_env=self.build_env, + config=None, + ) + base_sphinx = BaseSphinx( + build_env=self.build_env, + python_env=python_env, + ) with pytest.raises(ProjectConfigurationError): - self.base_sphinx.append_conf() + base_sphinx.append_conf() @override_settings(PRODUCTION_DOMAIN='readthedocs.org') From 189726ca01f2c5297870e966d3f2dfa4781a6908 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Tue, 28 Aug 2018 21:42:03 -0500 Subject: [PATCH 19/20] Remove repetition --- readthedocs/doc_builder/backends/sphinx.py | 29 +++++++++------------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/readthedocs/doc_builder/backends/sphinx.py b/readthedocs/doc_builder/backends/sphinx.py index 60f3bb2357e..571d53be57e 100644 --- a/readthedocs/doc_builder/backends/sphinx.py +++ b/readthedocs/doc_builder/backends/sphinx.py @@ -41,11 +41,12 @@ class BaseSphinx(BaseBuilder): def __init__(self, *args, **kwargs): super(BaseSphinx, self).__init__(*args, **kwargs) + self.config_file = self.config.sphinx.configuration try: - config_file = self.config.sphinx.configuration - config_dir = os.path.dirname(config_file) if config_file else None + if not self.config_file: + self.config_file = self.project.conf_file(self.version.slug) self.old_artifact_path = os.path.join( - config_dir or self.project.conf_dir(self.version.slug), + os.path.dirname(self.config_file), self.sphinx_build_dir ) except ProjectConfigurationError: @@ -73,7 +74,7 @@ def get_config_params(self): # TODO this should be handled better in the theme conf_py_path = os.path.join( os.path.sep, - self.config.sphinx.configuration or self.version.get_conf_py_path(), + self.config_file, '', ) remote_version = self.version.commit_name @@ -155,16 +156,16 @@ def get_config_params(self): def append_conf(self, **__): """Find or create a ``conf.py`` with a rendered ``doc_builder/conf.py.tmpl`` appended""" - if self.config.sphinx.configuration is None: + if self.config_file is None: master_doc = self.create_index(extension='rst') self._write_config(master_doc=master_doc) try: - outfile_path = ( - self.config.sphinx.configuration or + self.config_file = ( + self.config_file or self.project.conf_file(self.version.slug) ) - outfile = codecs.open(outfile_path, encoding='utf-8', mode='a') + outfile = codecs.open(self.config_file, encoding='utf-8', mode='a') except (ProjectConfigurationError, IOError): trace = sys.exc_info()[2] six.reraise( @@ -188,7 +189,7 @@ def append_conf(self, **__): self.run( 'cat', os.path.relpath( - outfile_path, + self.config_file, self.project.checkout_path(self.version.slug), ), cwd=self.project.checkout_path(self.version.slug), @@ -216,13 +217,9 @@ def build(self): '.', self.sphinx_build_dir, ]) - config_file = ( - self.config.sphinx.configuration or - self.project.conf_file(self.version.slug) - ) cmd_ret = self.run( *build_command, - cwd=os.path.dirname(config_file), + cwd=os.path.dirname(self.config_file), bin_path=self.python_env.venv_bin() ) return cmd_ret.successful @@ -360,9 +357,7 @@ class PdfBuilder(BaseSphinx): def build(self): self.clean() - config_file = self.config.sphinx.configuration - config_dir = os.path.dirname(config_file) if config_file else None - cwd = config_dir or self.project.conf_dir(self.version.slug) + cwd = os.path.dirname(self.config_file) # Default to this so we can return it always. self.run( From dc99eb8daa87e51e3fc3516e47e3fe1aaa725373 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Tue, 28 Aug 2018 21:42:33 -0500 Subject: [PATCH 20/20] Break and fix test --- readthedocs/rtd_tests/tests/test_doc_builder.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/readthedocs/rtd_tests/tests/test_doc_builder.py b/readthedocs/rtd_tests/tests/test_doc_builder.py index 478cb376cbc..4c2f7396991 100644 --- a/readthedocs/rtd_tests/tests/test_doc_builder.py +++ b/readthedocs/rtd_tests/tests/test_doc_builder.py @@ -47,9 +47,10 @@ def setUp(self): @patch('readthedocs.doc_builder.backends.sphinx.BaseSphinx.get_config_params') @patch('readthedocs.doc_builder.backends.sphinx.BaseSphinx.run') @patch('readthedocs.builds.models.Version.get_conf_py_path') - @patch('readthedocs.builds.models.Project.conf_file') @patch('readthedocs.projects.models.Project.checkout_path') - def test_create_conf_py(self, checkout_path, conf_file, get_conf_py_path, _, get_config_params, create_index, docs_dir): + def test_create_conf_py( + self, checkout_path, get_conf_py_path, _, + get_config_params, create_index, docs_dir): """ Test for a project without ``conf.py`` file. @@ -61,12 +62,12 @@ def test_create_conf_py(self, checkout_path, conf_file, get_conf_py_path, _, get any kind of exception raised by ``append_conf`` (we were originally having a ``TypeError`` because of an encoding problem in Python3) """ - checkout_path.return_value = tempfile.mkdtemp() - docs_dir.return_value = tempfile.mkdtemp() + tmp_dir = tempfile.mkdtemp() + checkout_path.return_value = tmp_dir + docs_dir.return_value = tmp_dir create_index.return_value = 'README.rst' get_config_params.return_value = {} get_conf_py_path.side_effect = ProjectConfigurationError - conf_file.return_value = tempfile.mktemp() python_env = Virtualenv( version=self.version, build_env=self.build_env, @@ -91,7 +92,11 @@ def test_create_conf_py(self, checkout_path, conf_file, get_conf_py_path, _, get 'conf.py' ) with open(generated_conf_py) as gf, open(expected_conf_py) as ef: - self.assertEqual(gf.read(), ef.read()) + autogenerated_confpy_lines = 28 + self.assertEqual( + gf.readlines()[:autogenerated_confpy_lines], + ef.readlines()[:autogenerated_confpy_lines] + ) @patch( 'readthedocs.doc_builder.backends.sphinx.SPHINX_TEMPLATE_DIR',