From 624fa026b0dc3aaeff97cbd76603c2761a38779a Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Wed, 12 Oct 2016 13:52:56 -0700 Subject: [PATCH] Add urllib3 AuthorizedHttp --- docs/conf.py | 5 +- google/auth/transport/__init__.py | 4 + google/auth/transport/urllib3.py | 151 +++++++++++++++++++++++++++++- tests/transport/test_urllib3.py | 114 +++++++++++++++++++++- tox.ini | 1 + 5 files changed, 265 insertions(+), 10 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index ccd864ac7..b70918fe4 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -363,7 +363,10 @@ # Example configuration for intersphinx: refer to the Python standard library. -intersphinx_mapping = {'https://docs.python.org/3.5': None} +intersphinx_mapping = { + 'python': ('https://docs.python.org/3.5', None), + 'urllib3': ('https://urllib3.readthedocs.io/en/latest', None), +} # Autodoc config autoclass_content = 'both' diff --git a/google/auth/transport/__init__.py b/google/auth/transport/__init__.py index 50b1c4393..0e4989ea6 100644 --- a/google/auth/transport/__init__.py +++ b/google/auth/transport/__init__.py @@ -27,6 +27,10 @@ import abc import six +from six.moves import http_client + +DEFAULT_REFRESH_STATUS_CODES = (http_client.UNAUTHORIZED,) +DEFAULT_MAX_REFRESH_ATTEMPTS = 2 @six.add_metaclass(abc.ABCMeta) diff --git a/google/auth/transport/urllib3.py b/google/auth/transport/urllib3.py index be6e55bc8..2f7b5251f 100644 --- a/google/auth/transport/urllib3.py +++ b/google/auth/transport/urllib3.py @@ -18,6 +18,11 @@ import logging +try: + import certifi +except ImportError: # pragma: NO COVER + certifi = None + import urllib3 import urllib3.exceptions @@ -27,7 +32,7 @@ _LOGGER = logging.getLogger(__name__) -class Response(transport.Response): +class _Response(transport.Response): """urllib3 transport response adapter. Args: @@ -50,7 +55,22 @@ def data(self): class Request(transport.Request): - """urllib3 request adapter + """urllib3 request adapter. + + This class is used internally for making requests using various transports + in a consistent way. If you use :class:`AuthorizedHttp` you do not need + to construct or use this class directly. + + This class can be useful if you want to manually refresh a + :class:`~google.auth.credentials.Credentials` instance:: + + import google.auth.transport.urllib3 + import urllib3 + + http = urllib3.PoolManager() + request = google.auth.transport.urllib3.Request(http) + + credentials.refresh(request) Args: http (urllib3.request.RequestMethods): An instance of any urllib3 @@ -79,7 +99,7 @@ def __call__(self, url, method='GET', body=None, headers=None, urllib3 :meth:`urlopen` method. Returns: - Response: The HTTP response. + google.auth.transport.Response: The HTTP response. Raises: google.auth.exceptions.TransportError: If any exception occurred. @@ -93,6 +113,129 @@ def __call__(self, url, method='GET', body=None, headers=None, _LOGGER.debug('Making request: %s %s', method, url) response = self.http.request( method, url, body=body, headers=headers, **kwargs) - return Response(response) + return _Response(response) except urllib3.exceptions.HTTPError as exc: raise exceptions.TransportError(exc) + + +def _make_default_http(): + if certifi: + return urllib3.PoolManager( + cert_reqs='CERT_REQUIRED', + ca_certs=certifi.where()) + else: + return urllib3.PoolManager() + + +class AuthorizedHttp(urllib3.request.RequestMethods): + """A urllib3 HTTP class with credentials. + + This class is used to perform requests to API endpoints that require + authorization:: + + from google.auth.transport.urllib3 import AuthorizedHttp + + authed_http = AuthorizedHttp(credentials) + + response = authed_http.request( + 'GET', 'https://www.googleapis.com/storage/v1/b') + + This class implements :class:`urllib3.request.RequestMethods` and can be + used just like any other :class:`urllib3.PoolManager`. + + The underlying :meth:`urlopen` implementation handles adding the + credentials' headers to the request and refreshing credentials as needed. + + Args: + credentials (google.auth.credentials.Credentials): The credentials to + add to the request. + http (urllib3.PoolManager): The underlying HTTP object to + use to make requests. If not specified, a + :class:`urllib3.PoolManager` instance will be constructed with + sane defaults. + refresh_status_codes (Sequence[int]): Which HTTP status code indicate + that credentials should be refreshed and the request should be + retried. + max_refresh_attempts (int): The maximum number of times to attempt to + refresh the credentials and retry the request. + """ + def __init__(self, credentials, http=None, + refresh_status_codes=transport.DEFAULT_REFRESH_STATUS_CODES, + max_refresh_attempts=transport.DEFAULT_MAX_REFRESH_ATTEMPTS): + + if http is None: + http = _make_default_http() + + self.http = http + self.credentials = credentials + self._refresh_status_codes = refresh_status_codes + self._max_refresh_attempts = max_refresh_attempts + # Request instance used by internal methods (for example, + # credentials.refresh). + self.__request = Request(self.http) + + def urlopen(self, method, url, body=None, headers=None, **kwargs): + """Implementation of urllib3's urlopen.""" + + # Use a kwarg for this instead of an attribute to maintain + # thread-safety. + _credential_refresh_attempt = kwargs.pop( + '_credential_refresh_attempt', 0) + + if headers is None: + headers = self.headers + + # Make a copy of the headers. They will be modified by the credentials + # and we want to pass the original headers if we recurse. + request_headers = headers.copy() + + self.credentials.before_request( + self.__request, method, url, request_headers) + + response = self.http.urlopen( + method, url, body=body, headers=request_headers, **kwargs) + + # If the response indicated that the credentials needed to be + # refreshed, then refresh the credentials and re-attempt the + # request. + # A stored token may expire between the time it is retrieved and + # the time the request is made, so we may need to try twice. + # The reason urllib3's retries aren't used is because they + # don't allow you to modify the request headers. :/ + if (response.status in self._refresh_status_codes + and _credential_refresh_attempt < self._max_refresh_attempts): + + _LOGGER.info( + 'Refreshing credentials due to a %s response. Attempt %s/%s.', + response.status, _credential_refresh_attempt + 1, + self._max_refresh_attempts) + + self.credentials.refresh(self.__request) + + # Recurse. Pass in the original headers, not our modified set. + return self.urlopen( + method, url, body=body, headers=headers, + _credential_refresh_attempt=_credential_refresh_attempt + 1, + **kwargs) + + return response + + # Proxy methods for compliance with the urllib3.PoolManager interface + + def __enter__(self): + """Proxy to ``self.http``.""" + return self.http.__enter__() + + def __exit__(self, exc_type, exc_val, exc_tb): + """Proxy to ``self.http``.""" + return self.http.__exit__(exc_type, exc_val, exc_tb) + + @property + def headers(self): + """Proxy to ``self.http``.""" + return self.http.headers + + @headers.setter + def headers(self, value): + """Proxy to ``self.http``.""" + self.http.headers = value diff --git a/tests/transport/test_urllib3.py b/tests/transport/test_urllib3.py index bdd5ac9f4..da913b5fb 100644 --- a/tests/transport/test_urllib3.py +++ b/tests/transport/test_urllib3.py @@ -13,6 +13,7 @@ # limitations under the License. import mock +from six.moves import http_client import urllib3 import google.auth.transport.urllib3 @@ -24,10 +25,113 @@ def make_request(self): http = urllib3.PoolManager() return google.auth.transport.urllib3.Request(http) + def test_timeout(self): + http = mock.Mock() + request = google.auth.transport.urllib3.Request(http) + request(url='http://example.com', method='GET', timeout=5) -def test_timeout(): - http = mock.Mock() - request = google.auth.transport.urllib3.Request(http) - request(url='http://example.com', method='GET', timeout=5) + assert http.request.call_args[1]['timeout'] == 5 - assert http.request.call_args[1]['timeout'] == 5 + +def test__make_default_http_with_certfi(): + http = google.auth.transport.urllib3._make_default_http() + assert 'cert_reqs' in http.connection_pool_kw + + +@mock.patch.object(google.auth.transport.urllib3, 'certifi', False) +def test__make_default_http_without_certfi(): + http = google.auth.transport.urllib3._make_default_http() + assert 'cert_reqs' not in http.connection_pool_kw + + +class MockCredentials(object): + def __init__(self, token='token'): + self.token = token + + def apply(self, headers): + headers['authorization'] = self.token + + def before_request(self, request, method, url, headers): + self.apply(headers) + + def refresh(self, request): + self.token = self.token + '1' + + +class MockHttp(object): + def __init__(self, responses, headers=None): + self.responses = responses + self.requests = [] + self.headers = headers or {} + + def urlopen(self, method, url, body=None, headers=None, **kwargs): + self.requests.append((method, url, body, headers)) + return self.responses.pop(0) + + +class MockResponse(object): + def __init__(self, status=http_client.OK, data=None): + self.status = status + self.data = data + + +class TestAuthorizedHttp(object): + TEST_URL = 'http://example.com' + + def test_authed_http_defaults(self): + authed_http = google.auth.transport.urllib3.AuthorizedHttp( + mock.sentinel.credentials) + + assert authed_http.credentials == mock.sentinel.credentials + assert authed_http.http is not None + + def test_urlopen_no_refresh(self): + mock_credentials = mock.Mock(wraps=MockCredentials()) + mock_response = MockResponse() + mock_http = MockHttp([mock_response]) + + authed_http = google.auth.transport.urllib3.AuthorizedHttp( + mock_credentials, http=mock_http) + + response = authed_http.urlopen('GET', self.TEST_URL) + + assert response == mock_response + assert mock_credentials.before_request.called + assert not mock_credentials.refresh.called + assert mock_http.requests == [ + ('GET', self.TEST_URL, None, {'authorization': 'token'})] + + def test_urlopen_refresh(self): + mock_credentials = mock.Mock(wraps=MockCredentials()) + mock_final_response = MockResponse(status=http_client.OK) + # First request will 403, second request will succeed. + mock_http = MockHttp([ + MockResponse(status=http_client.UNAUTHORIZED), + mock_final_response]) + + authed_http = google.auth.transport.urllib3.AuthorizedHttp( + mock_credentials, http=mock_http) + + response = authed_http.urlopen('GET', 'http://example.com') + + assert response == mock_final_response + assert mock_credentials.before_request.call_count == 2 + assert mock_credentials.refresh.called + assert mock_http.requests == [ + ('GET', self.TEST_URL, None, {'authorization': 'token'}), + ('GET', self.TEST_URL, None, {'authorization': 'token1'})] + + def test_proxies(self): + mock_http = mock.MagicMock() + + authed_http = google.auth.transport.urllib3.AuthorizedHttp( + None, http=mock_http) + + with authed_http: + pass + + assert mock_http.__enter__.called + assert mock_http.__exit__.called + + authed_http.headers = mock.sentinel.headers + assert authed_http.headers == mock_http.headers diff --git a/tox.ini b/tox.ini index 5f2df1c92..ffaef944e 100644 --- a/tox.ini +++ b/tox.ini @@ -9,6 +9,7 @@ deps = pytest-cov pytest-localserver urllib3 + certifi commands = py.test --cov=google.auth --cov=google.oauth2 --cov=tests {posargs:tests}