Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fixes #5948 Adds keyring support #5952

Merged
merged 4 commits into from
May 10, 2019
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions news/5948.feature
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Credentials will now be loaded using `keyring` when installed.
15 changes: 15 additions & 0 deletions src/pip/_internal/cli/base_command.py
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,20 @@ def run(self, options, args):
# type: (Values, List[Any]) -> Any
raise NotImplementedError

@classmethod
def _get_index_urls(cls, options):
"""Return a list of index urls from user-provided options."""
index_urls = []
if not getattr(options, "no_index", False):
url = getattr(options, "index_url", None)
if url:
index_urls.append(url)
urls = getattr(options, "extra_index_urls", None)
if urls:
index_urls.extend(urls)
# Return None rather than an empty list
return index_urls or None

def _build_session(self, options, retries=None, timeout=None):
# type: (Values, Optional[int], Optional[int]) -> PipSession
session = PipSession(
Expand All @@ -90,6 +104,7 @@ def _build_session(self, options, retries=None, timeout=None):
),
retries=retries if retries is not None else options.retries,
insecure_hosts=options.trusted_hosts,
index_urls=self._get_index_urls(options),
)

# Handle custom ca-bundles from the user
Expand Down
226 changes: 196 additions & 30 deletions src/pip/_internal/download.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@

import cgi
import email.utils
import getpass
import json
import logging
import mimetypes
Expand All @@ -12,7 +11,7 @@
import shutil
import sys

from pip._vendor import requests, six, urllib3
from pip._vendor import requests, urllib3
from pip._vendor.cachecontrol import CacheControlAdapter
from pip._vendor.cachecontrol.caches import FileCache
from pip._vendor.lockfile import LockError
Expand All @@ -36,9 +35,10 @@
from pip._internal.utils.filesystem import check_path_owner
from pip._internal.utils.glibc import libc_ver
from pip._internal.utils.misc import (
ARCHIVE_EXTENSIONS, ask_path_exists, backup_dir, consume, display_path,
format_size, get_installed_version, rmtree, split_auth_from_netloc,
splitext, unpack_file,
ARCHIVE_EXTENSIONS, ask, ask_input, ask_password, ask_path_exists,
backup_dir, consume, display_path, format_size, get_installed_version,
remove_auth_from_url, rmtree, split_auth_netloc_from_url, splitext,
unpack_file,
)
from pip._internal.utils.temp_dir import TempDirectory
from pip._internal.utils.typing import MYPY_CHECK_RUNNING
Expand All @@ -49,6 +49,7 @@
from typing import (
Optional, Tuple, Dict, IO, Text, Union
)
from optparse import Values
from pip._internal.models.link import Link
from pip._internal.utils.hashes import Hashes
from pip._internal.vcs import AuthInfo
Expand All @@ -58,6 +59,7 @@
except ImportError:
ssl = None


HAS_TLS = (ssl is not None) or IS_PYOPENSSL

__all__ = ['get_file_content',
Expand All @@ -70,6 +72,15 @@
logger = logging.getLogger(__name__)


try:
import keyring # noqa
except ImportError:
keyring = None
except Exception as exc:
logger.warning("Keyring is skipped due to an exception: %s",
str(exc))
keyring = None

# These are environment variables present when running under various
# CI systems. For each variable, some CI systems that use the variable
# are indicated. The collection was chosen so that for each of a number
Expand Down Expand Up @@ -177,46 +188,175 @@ def user_agent():
)


def _get_keyring_auth(url, username):
"""Return the tuple auth for a given url from keyring."""
if not url or not keyring:
return None

try:
try:
get_credential = keyring.get_credential
except AttributeError:
pass
else:
logger.debug("Getting credentials from keyring for %s", url)
cred = get_credential(url, username)
if cred is not None:
return cred.username, cred.password
return None

if username:
logger.debug("Getting password from keyring for %s", url)
password = keyring.get_password(url, username)
if password:
return username, password

except Exception as exc:
logger.warning("Keyring is skipped due to an exception: %s",
str(exc))


class MultiDomainBasicAuth(AuthBase):

def __init__(self, prompting=True):
# type: (bool) -> None
def __init__(self, prompting=True, index_urls=None):
# type: (bool, Optional[Values]) -> None
self.prompting = prompting
self.index_urls = index_urls
self.passwords = {} # type: Dict[str, AuthInfo]
# When the user is prompted to enter credentials and keyring is
# available, we will offer to save them. If the user accepts,
# this value is set to the credentials they entered. After the
# request authenticates, the caller should call
# ``save_credentials`` to save these.
self._credentials_to_save = None # type: Tuple[str, str, str]

def _get_index_url(self, url):
"""Return the original index URL matching the requested URL.

Cached or dynamically generated credentials may work against
the original index URL rather than just the netloc.

The provided url should have had its username and password
removed already. If the original index url had credentials then
they will be included in the return value.

Returns None if no matching index was found, or if --no-index
was specified by the user.
"""
if not url or not self.index_urls:
return None

for u in self.index_urls:
prefix = remove_auth_from_url(u).rstrip("/") + "/"
if url.startswith(prefix):
return u

def _get_new_credentials(self, original_url, allow_netrc=True,
allow_keyring=True):
"""Find and return credentials for the specified URL."""
# Split the credentials and netloc from the url.
url, netloc, url_user_password = split_auth_netloc_from_url(
original_url)

# Start with the credentials embedded in the url
username, password = url_user_password
if username is not None and password is not None:
zooba marked this conversation as resolved.
Show resolved Hide resolved
logger.debug("Found credentials in url for %s", netloc)
return url_user_password

zooba marked this conversation as resolved.
Show resolved Hide resolved
# Find a matching index url for this request
index_url = self._get_index_url(url)
if index_url:
# Split the credentials from the url.
index_info = split_auth_netloc_from_url(index_url)
if index_info:
index_url, _, index_url_user_password = index_info
logger.debug("Found index url %s", index_url)

# If an index URL was found, try its embedded credentials
if index_url and index_url_user_password[0] is not None:
username, password = index_url_user_password
if username is not None and password is not None:
logger.debug("Found credentials in index url for %s", netloc)
return index_url_user_password

def __call__(self, req):
parsed = urllib_parse.urlparse(req.url)

# Split the credentials from the netloc.
netloc, url_user_password = split_auth_from_netloc(parsed.netloc)

# Set the url of the request to the url without any credentials
req.url = urllib_parse.urlunparse(parsed[:1] + (netloc,) + parsed[2:])
# Get creds from netrc if we still don't have them
if allow_netrc:
netrc_auth = get_netrc_auth(original_url)
if netrc_auth:
logger.debug("Found credentials in netrc for %s", netloc)
return netrc_auth

# If we don't have a password and keyring is available, use it.
if allow_keyring:
# The index url is more specific than the netloc, so try it first
kr_auth = (_get_keyring_auth(index_url, username) or
_get_keyring_auth(netloc, username))
if kr_auth:
logger.debug("Found credentials in keyring for %s", netloc)
return kr_auth

return None, None

def _get_url_and_credentials(self, original_url):
"""Return the credentials to use for the provided URL.

If allowed, netrc and keyring may be used to obtain the
correct credentials.

Returns (url_without_credentials, username, password). Note
that even if the original URL contains credentials, this
function may return a different username and password.
"""
url, netloc, _ = split_auth_netloc_from_url(original_url)

# Use any stored credentials that we have for this netloc
username, password = self.passwords.get(netloc, (None, None))

# Use the credentials embedded in the url if we have none stored
if username is None:
username, password = url_user_password

# Get creds from netrc if we still don't have them
if username is None and password is None:
netrc_auth = get_netrc_auth(req.url)
username, password = netrc_auth if netrc_auth else (None, None)
# If nothing cached, acquire new credentials without prompting
# the user (e.g. from netrc, keyring, or similar).
if username is None or password is None:
username, password = self._get_new_credentials(original_url)

if username or password:
if username is not None and password is not None:
# Store the username and password
self.passwords[netloc] = (username, password)

return url, username, password

def __call__(self, req):
# Get credentials for this request
url, username, password = self._get_url_and_credentials(req.url)

# Set the url of the request to the url without any credentials
req.url = url

if username is not None and password is not None:
# Send the basic auth with this request
req = HTTPBasicAuth(username or "", password or "")(req)
req = HTTPBasicAuth(username, password)(req)

# Attach a hook to handle 401 responses
req.register_hook("response", self.handle_401)

return req

# Factored out to allow for easy patching in tests
def _prompt_for_password(self, netloc):
username = ask_input("User for %s: " % netloc)
if not username:
return None, None
auth = _get_keyring_auth(netloc, username)
if auth:
return auth[0], auth[1], False
password = ask_password("Password: ")
return username, password, True

# Factored out to allow for easy patching in tests
def _should_save_password_to_keyring(self):
if not keyring:
return False
return ask("Save credentials to keyring [y/N]: ", ["y", "n"]) == "y"

def handle_401(self, resp, **kwargs):
# We only care about 401 responses, anything else we want to just
# pass through the actual response
Expand All @@ -230,13 +370,17 @@ def handle_401(self, resp, **kwargs):
parsed = urllib_parse.urlparse(resp.url)

# Prompt the user for a new username and password
username = six.moves.input("User for %s: " % parsed.netloc)
password = getpass.getpass("Password: ")
username, password, save = self._prompt_for_password(parsed.netloc)

# Store the new username and password to use for future requests
if username or password:
self._credentials_to_save = None
if username is not None and password is not None:
self.passwords[parsed.netloc] = (username, password)

# Prompt to save the password to keyring
if save and self._should_save_password_to_keyring():
self._credentials_to_save = (parsed.netloc, username, password)

# Consume content and release the original connection to allow our new
# request to reuse the same one.
resp.content
Expand All @@ -246,18 +390,39 @@ def handle_401(self, resp, **kwargs):
req = HTTPBasicAuth(username or "", password or "")(resp.request)
req.register_hook("response", self.warn_on_401)

# On successful request, save the credentials that were used to
# keyring. (Note that if the user responded "no" above, this member
# is not set and nothing will be saved.)
if self._credentials_to_save:
req.register_hook("response", self.save_credentials)

# Send our new request
new_resp = resp.connection.send(req, **kwargs)
new_resp.history.append(resp)

return new_resp

def warn_on_401(self, resp, **kwargs):
# warn user that they provided incorrect credentials
"""Response callback to warn about incorrect credentials."""
if resp.status_code == 401:
logger.warning('401 Error, Credentials not correct for %s',
resp.request.url)

def save_credentials(self, resp, **kwargs):
"""Response callback to save credentials on success."""
assert keyring is not None, "should never reach here without keyring"
if not keyring:
return

creds = self._credentials_to_save
self._credentials_to_save = None
if creds and resp.status_code < 400:
try:
logger.info('Saving credentials to keyring')
keyring.set_password(*creds)
except Exception:
logger.exception('Failed to save credentials')


class LocalFSAdapter(BaseAdapter):

Expand Down Expand Up @@ -373,14 +538,15 @@ def __init__(self, *args, **kwargs):
retries = kwargs.pop("retries", 0)
cache = kwargs.pop("cache", None)
insecure_hosts = kwargs.pop("insecure_hosts", [])
index_urls = kwargs.pop("index_urls", None)

super(PipSession, self).__init__(*args, **kwargs)

# Attach our User Agent to the request
self.headers["User-Agent"] = user_agent()

# Attach our Authentication handler to the session
self.auth = MultiDomainBasicAuth()
self.auth = MultiDomainBasicAuth(index_urls=index_urls)

# Create our urllib3.Retry instance which will allow us to customize
# how we handle retries.
Expand Down
Loading