From ed1819c94488b8fb1bbad3ae1c1946abe86ff47c Mon Sep 17 00:00:00 2001 From: Celeborn2BeAlive Date: Fri, 29 May 2020 10:35:42 +0200 Subject: [PATCH 1/7] feat: add trusted option to repository on develop branch, but requires changes to poetry-core also... --- docs/docs/repositories.md | 11 ++++++++++ poetry/factory.py | 1 + poetry/installation/pip_installer.py | 3 +++ poetry/json/schemas/poetry-schema.json | 4 ++++ poetry/repositories/legacy_repository.py | 26 ++++++++++++++++++++++-- 5 files changed, 43 insertions(+), 2 deletions(-) diff --git a/docs/docs/repositories.md b/docs/docs/repositories.md index d281f6ca77a..1eee0968b30 100644 --- a/docs/docs/repositories.md +++ b/docs/docs/repositories.md @@ -134,3 +134,14 @@ default = true ``` A default source will also be the fallback source if you add other sources. + +### Trusting a repository + +You can bypass SSL verification for a repository if you know it can be trusted (useful for corporate private repositories): + +```toml +[[tool.poetry.source]] +name = "foo" +url = "https://foo.bar/simple/" +trusted = true +``` diff --git a/poetry/factory.py b/poetry/factory.py index 48cac9809da..5997870ea32 100644 --- a/poetry/factory.py +++ b/poetry/factory.py @@ -151,4 +151,5 @@ def create_legacy_repository( auth=auth, cert=get_cert(auth_config, name), client_cert=get_client_cert(auth_config, name), + trusted=source.get("trusted", False), ) diff --git a/poetry/installation/pip_installer.py b/poetry/installation/pip_installer.py index 737d619f461..1a2abadf41b 100644 --- a/poetry/installation/pip_installer.py +++ b/poetry/installation/pip_installer.py @@ -58,6 +58,9 @@ def install(self, package, update=False): if repository.client_cert: args += ["--client-cert", str(repository.client_cert)] + if repository.trusted and not parsed.scheme == "http": + args += ["--trusted-host", parsed.hostname] + index_url = repository.authenticated_url args += ["--index-url", index_url] diff --git a/poetry/json/schemas/poetry-schema.json b/poetry/json/schemas/poetry-schema.json index e94b90d28cc..4cd651406ab 100644 --- a/poetry/json/schemas/poetry-schema.json +++ b/poetry/json/schemas/poetry-schema.json @@ -523,6 +523,10 @@ "secondary": { "type": "boolean", "description": "Declare this repository as secondary, i.e. it will only be looked up last for packages." + }, + "trusted": { + "type": "boolean", + "description": "Declare this repository as trusted, i.e. option --trusted-host will be used when installing packages with pip." } } } diff --git a/poetry/repositories/legacy_repository.py b/poetry/repositories/legacy_repository.py index 7baf97ccf86..847f8b5aa74 100644 --- a/poetry/repositories/legacy_repository.py +++ b/poetry/repositories/legacy_repository.py @@ -158,7 +158,14 @@ def clean_link(self, url): class LegacyRepository(PyPiRepository): def __init__( - self, name, url, auth=None, disable_cache=False, cert=None, client_cert=None + self, + name, + url, + auth=None, + disable_cache=False, + cert=None, + client_cert=None, + trusted=False, ): # type: (str, str, Optional[Auth], bool, Optional[Path], Optional[Path]) -> None if name == "pypi": raise ValueError("The name [pypi] is reserved for repositories") @@ -169,6 +176,7 @@ def __init__( self._auth = auth self._client_cert = client_cert self._cert = cert + self._trusted = trusted self._cache_dir = REPOSITORY_CACHE_DIR / name self._cache = CacheManager( { @@ -206,6 +214,10 @@ def cert(self): # type: () -> Optional[Path] def client_cert(self): # type: () -> Optional[Path] return self._client_cert + @property + def trusted(self): + return self._trusted + @property def authenticated_url(self): # type: () -> str if not self._auth: @@ -349,7 +361,17 @@ def _get_release_info(self, name, version): # type: (str, str) -> dict def _get(self, endpoint): # type: (str) -> Union[Page, None] url = self._url + endpoint - response = self.session.get(url) + if self._trusted: + import urllib3 + + urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + response = self.session.get(url, verify=not self._trusted) + if self._trusted: + import urllib3 + + urllib3.warnings.simplefilter( + "default", urllib3.exceptions.InsecureRequestWarning + ) if response.status_code == 404: return From 7c056a759eacb70274824fe2738c82b80dea87ca Mon Sep 17 00:00:00 2001 From: Maayan Bar Date: Sat, 12 Sep 2020 22:17:56 +0300 Subject: [PATCH 2/7] removed redundant double check of http scheme --- poetry/installation/pip_installer.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/poetry/installation/pip_installer.py b/poetry/installation/pip_installer.py index 1a2abadf41b..11ae2562a2c 100644 --- a/poetry/installation/pip_installer.py +++ b/poetry/installation/pip_installer.py @@ -51,6 +51,8 @@ def install(self, package, update=False): ) ) args += ["--trusted-host", parsed.hostname] + elif repository.trusted: + args += ["--trusted-host", parsed.hostname] if repository.cert: args += ["--cert", str(repository.cert)] @@ -58,9 +60,6 @@ def install(self, package, update=False): if repository.client_cert: args += ["--client-cert", str(repository.client_cert)] - if repository.trusted and not parsed.scheme == "http": - args += ["--trusted-host", parsed.hostname] - index_url = repository.authenticated_url args += ["--index-url", index_url] From af7b1e7501dbbeded7fed2655024042eec8212c2 Mon Sep 17 00:00:00 2001 From: Maayan Bar Date: Sat, 12 Sep 2020 22:32:53 +0300 Subject: [PATCH 3/7] Added trusted source tests and fixture to test_factory --- tests/fixtures/with_trusted_source/README.rst | 2 + .../with_trusted_source/pyproject.toml | 62 +++++++++++++++++++ tests/test_factory.py | 6 ++ 3 files changed, 70 insertions(+) create mode 100644 tests/fixtures/with_trusted_source/README.rst create mode 100644 tests/fixtures/with_trusted_source/pyproject.toml diff --git a/tests/fixtures/with_trusted_source/README.rst b/tests/fixtures/with_trusted_source/README.rst new file mode 100644 index 00000000000..f7fe15470f9 --- /dev/null +++ b/tests/fixtures/with_trusted_source/README.rst @@ -0,0 +1,2 @@ +My Package +========== diff --git a/tests/fixtures/with_trusted_source/pyproject.toml b/tests/fixtures/with_trusted_source/pyproject.toml new file mode 100644 index 00000000000..b46d22133b4 --- /dev/null +++ b/tests/fixtures/with_trusted_source/pyproject.toml @@ -0,0 +1,62 @@ +[tool.poetry] +name = "my-package" +version = "1.2.3" +description = "Some description." +authors = [ + "Sébastien Eustace " +] +license = "MIT" + +readme = "README.rst" + +homepage = "https://python-poetry.org" +repository = "https://github.com/python-poetry/poetry" +documentation = "https://python-poetry.org/docs" + +keywords = ["packaging", "dependency", "poetry"] + +classifiers = [ + "Topic :: Software Development :: Build Tools", + "Topic :: Software Development :: Libraries :: Python Modules" +] + +# Requirements +[tool.poetry.dependencies] +python = "~2.7 || ^3.6" +cleo = "^0.6" +pendulum = { git = "https://github.com/sdispater/pendulum.git", branch = "2.0" } +requests = { version = "^2.18", optional = true, extras=[ "security" ] } +pathlib2 = { version = "^2.2", python = "~2.7" } + +orator = { version = "^0.9", optional = true } + +# File dependency +demo = { path = "../distributions/demo-0.1.0-py2.py3-none-any.whl" } + +# Dir dependency with setup.py +my-package = { path = "../project_with_setup/" } + +# Dir dependency with pyproject.toml +simple-project = { path = "../simple_project/" } + + +[tool.poetry.extras] +db = [ "orator" ] + +[tool.poetry.dev-dependencies] +pytest = "~3.4" + + +[tool.poetry.scripts] +my-script = "my_package:main" + + +[tool.poetry.plugins."blogtool.parsers"] +".rst" = "some_module::SomeClass" + + +[[tool.poetry.source]] +name = "foo" +url = "https://foo.bar/simple/" +default = true +trusted = true diff --git a/tests/test_factory.py b/tests/test_factory.py index ffb48965096..bb5bf4f521d 100644 --- a/tests/test_factory.py +++ b/tests/test_factory.py @@ -146,6 +146,12 @@ def test_poetry_with_default_source(): assert 1 == len(poetry.pool.repositories) +def test_poetry_with_trusted_source(): + poetry = Factory().create_poetry(fixtures_dir / "with_trusted_source") + + assert 1 == len(poetry.pool.repositories) + + def test_poetry_with_two_default_sources(): with pytest.raises(ValueError) as e: Factory().create_poetry(fixtures_dir / "with_two_default_sources") From c3dccac26d23b5aa94a96b44894477066a4be06d Mon Sep 17 00:00:00 2001 From: Maayan Bar Date: Sat, 12 Sep 2020 22:33:40 +0300 Subject: [PATCH 4/7] Added trusted source tests to pip_installer --- tests/installation/test_pip_installer.py | 27 ++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tests/installation/test_pip_installer.py b/tests/installation/test_pip_installer.py index f3cf376ff39..80b49aff57f 100644 --- a/tests/installation/test_pip_installer.py +++ b/tests/installation/test_pip_installer.py @@ -152,3 +152,30 @@ def test_requirement_git_develop_true(installer, package_git): expected = ["-e", "git+git@github.com:demo/demo.git@master#egg=demo"] assert expected == result + + +def test_install_with_trusted_host(): + pool = Pool() + host = "foo.bar" + + default = LegacyRepository("default", f"https://{host}", trusted=True) + + pool.add_repository(default, default=True) + + null_env = NullEnv() + + installer = PipInstaller(null_env, NullIO(), pool) + + foo = Package("foo", "0.0.0") + foo.source_type = "legacy" + foo.source_reference = default._name + foo.source_url = default._url + + installer.install(foo) + + assert len(null_env.executed) == 1 + cmd = null_env.executed[0] + assert "--trusted-host" in cmd + trusted_host_index = cmd.index("--trusted-host") + + assert cmd[trusted_host_index + 1] == host From 83fe465e1e5d70e8a277625c1ab51fae0942242a Mon Sep 17 00:00:00 2001 From: Celeborn2BeAlive Date: Sun, 13 Sep 2020 02:50:27 +0200 Subject: [PATCH 5/7] fix: reintroduce response.raise_for_status() --- poetry/repositories/legacy_repository.py | 1 + 1 file changed, 1 insertion(+) diff --git a/poetry/repositories/legacy_repository.py b/poetry/repositories/legacy_repository.py index 1ef86649d51..4e2cbb1db4c 100644 --- a/poetry/repositories/legacy_repository.py +++ b/poetry/repositories/legacy_repository.py @@ -394,6 +394,7 @@ def _get(self, endpoint): # type: (str) -> Union[Page, None] response = self.session.get(url, verify=not self._trusted) if response.status_code == 404: return + response.raise_for_status() except requests.HTTPError as e: raise RepositoryError(e) finally: From fde7deaae9a22d90f500bf974a566f9b8d215ce4 Mon Sep 17 00:00:00 2001 From: Celeborn2BeAlive Date: Sun, 13 Sep 2020 03:40:08 +0200 Subject: [PATCH 6/7] fix: remove f-string syntax for older versions of python --- tests/installation/test_pip_installer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/installation/test_pip_installer.py b/tests/installation/test_pip_installer.py index fec0ffb9dc0..abf99aa97c0 100644 --- a/tests/installation/test_pip_installer.py +++ b/tests/installation/test_pip_installer.py @@ -160,7 +160,7 @@ def test_install_with_trusted_host(): pool = Pool() host = "foo.bar" - default = LegacyRepository("default", f"https://{host}", trusted=True) + default = LegacyRepository("default", "https://{}".format(host), trusted=True) pool.add_repository(default, default=True) From a52032576dede92aea8dfd2b90cad2c0e2ce7d51 Mon Sep 17 00:00:00 2001 From: Celeborn2BeAlive Date: Sun, 13 Sep 2020 04:17:56 +0200 Subject: [PATCH 7/7] refactor: use contextmanager + questions --- poetry/repositories/legacy_repository.py | 32 +++++++++++++++--------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/poetry/repositories/legacy_repository.py b/poetry/repositories/legacy_repository.py index 4e2cbb1db4c..df6f88128b7 100644 --- a/poetry/repositories/legacy_repository.py +++ b/poetry/repositories/legacy_repository.py @@ -3,6 +3,7 @@ import warnings from collections import defaultdict +from contextlib import contextmanager from typing import Generator from typing import Optional from typing import Union @@ -384,24 +385,31 @@ def _get_release_info(self, name, version): # type: (str, str) -> dict return data.asdict() - def _get(self, endpoint): # type: (str) -> Union[Page, None] - url = self._url + endpoint - if self._trusted: - import urllib3 + @contextmanager + def _trust_context(self): + import urllib3 # Not a direct dependency of poetry, should I use requests.packages.urllib3 instead ? + if self._trusted: urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + try: - response = self.session.get(url, verify=not self._trusted) - if response.status_code == 404: - return - response.raise_for_status() - except requests.HTTPError as e: - raise RepositoryError(e) + yield finally: if self._trusted: - import urllib3 - urllib3.warnings.simplefilter( "default", urllib3.exceptions.InsecureRequestWarning ) + + def _get(self, endpoint): # type: (str) -> Union[Page, None] + url = self._url + endpoint + + with self._trust_context() as _: # maybe use urllib3.warnings.catch_warnings() instead ? + try: + response = self.session.get(url, verify=not self._trusted) + if response.status_code == 404: + return + response.raise_for_status() + except requests.HTTPError as e: + raise RepositoryError(e) + return Page(url, response.content, response.headers)