feat: pluggable URL parsing

examples/docker-info-alt2.py

@@ -0,0 +1,21 @@
+#!/usr/bin/env python
+
+import json
+from requests.compat import urlparse
+
+from requests_unixsocket import Session, Settings
+
+
+def custom_urlparse(url):
+    parsed_url = urlparse(url)
+    return Settings.ParseResult(
+        sockpath=parsed_url.path,
+        reqpath=parsed_url.fragment,
+    )
+
+
+session = Session(settings=Settings(urlparse=custom_urlparse))
+
+r = session.get('http+unix://sock.localhost/var/run/docker.sock#/info')
+registry_config = r.json()['RegistryConfig']
+print(json.dumps(registry_config, indent=4))

requests_unixsocket/__init__.py

@@ -1,19 +1,25 @@
-import requests
 import sys
 
+import requests
+
 from .adapters import UnixAdapter
+from .settings import default_scheme, default_settings, Settings
 
-DEFAULT_SCHEME = 'http+unix://'
+# for backwards compatibility
+# https://github.com/httpie/httpie-unixsocket uses this for example
+DEFAULT_SCHEME = default_scheme
 
 
 class Session(requests.Session):
-    def __init__(self, url_scheme=DEFAULT_SCHEME, *args, **kwargs):
+    def __init__(self, url_scheme=default_scheme, settings=None,
+                 *args, **kwargs):
         super(Session, self).__init__(*args, **kwargs)
-        self.mount(url_scheme, UnixAdapter())
+        self.settings = settings or default_settings
+        self.mount(url_scheme, UnixAdapter(settings=self.settings))
 
 
 class monkeypatch(object):
-    def __init__(self, url_scheme=DEFAULT_SCHEME):
+    def __init__(self, url_scheme=default_scheme):
         self.session = Session()
         requests = self._get_global_requests_module()
 
@@ -75,3 +81,13 @@ def delete(url, **kwargs):
 def options(url, **kwargs):
     kwargs.setdefault('allow_redirects', True)
     return request('options', url, **kwargs)
+
+
+__all__ = [
+    default_scheme, DEFAULT_SCHEME,
+    default_settings,
+    monkeypatch,
+    Session,
+    Settings,
+    request, get, head, post, patch, put, delete, options,
+]

requests_unixsocket/adapters.py

@@ -1,7 +1,11 @@
+# This file contains code that was adapted from some code from docker-py
+# (Apache License 2.0)
+# https://github.com/docker/docker-py/blob/master/docker/transport/unixconn.py
+
 import socket
 
 from requests.adapters import HTTPAdapter
-from requests.compat import urlparse, unquote
+from requests.compat import urlparse
 
 try:
     import http.client as httplib
@@ -13,21 +17,17 @@
 except ImportError:
     import urllib3
 
+from .settings import default_settings
 
-# The following was adapted from some code from docker-py
-# https://github.com/docker/docker-py/blob/master/docker/transport/unixconn.py
-class UnixHTTPConnection(httplib.HTTPConnection, object):
 
-    def __init__(self, unix_socket_url, timeout=60):
-        """Create an HTTP connection to a unix domain socket
+class UnixHTTPConnection(httplib.HTTPConnection, object):
 
-        :param unix_socket_url: A URL with a scheme of 'http+unix' and the
-        netloc is a percent-encoded path to a unix domain socket. E.g.:
-        'http+unix://%2Ftmp%2Fprofilesvc.sock/status/pid'
-        """
+    def __init__(self, url, timeout=60, settings=None):
+        """Create an HTTP connection to a unix domain socket"""
         super(UnixHTTPConnection, self).__init__('localhost', timeout=timeout)
-        self.unix_socket_url = unix_socket_url
+        self.url = url
         self.timeout = timeout
+        self.settings = settings
         self.sock = None
 
     def __del__(self):  # base class does not have d'tor
@@ -37,27 +37,34 @@ def __del__(self):  # base class does not have d'tor
     def connect(self):
         sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
         sock.settimeout(self.timeout)
-        socket_path = unquote(urlparse(self.unix_socket_url).netloc)
-        sock.connect(socket_path)
+        sockpath = self.settings.urlparse(self.url).sockpath
+        sock.connect(sockpath)
         self.sock = sock
 
 
 class UnixHTTPConnectionPool(urllib3.connectionpool.HTTPConnectionPool):
 
-    def __init__(self, socket_path, timeout=60):
+    def __init__(self, socket_path, timeout=60, settings=None):
         super(UnixHTTPConnectionPool, self).__init__(
             'localhost', timeout=timeout)
         self.socket_path = socket_path
         self.timeout = timeout
+        self.settings = settings
 
     def _new_conn(self):
-        return UnixHTTPConnection(self.socket_path, self.timeout)
+        return UnixHTTPConnection(
+            url=self.socket_path,
+            timeout=self.timeout,
+            settings=self.settings,
+        )
 
 
 class UnixAdapter(HTTPAdapter):
-
-    def __init__(self, timeout=60, pool_connections=25, *args, **kwargs):
+    def __init__(self, timeout=60, pool_connections=25,
+                 settings=None,
+                 *args, **kwargs):
         super(UnixAdapter, self).__init__(*args, **kwargs)
+        self.settings = settings or default_settings
         self.timeout = timeout
         self.pools = urllib3._collections.RecentlyUsedContainer(
             pool_connections, dispose_func=lambda p: p.close()
@@ -76,13 +83,17 @@ def get_connection(self, url, proxies=None):
             if pool:
                 return pool
 
-            pool = UnixHTTPConnectionPool(url, self.timeout)
+            pool = UnixHTTPConnectionPool(
+                socket_path=url,
+                settings=self.settings,
+                timeout=self.timeout,
+            )
             self.pools[url] = pool
 
         return pool
 
     def request_url(self, request, proxies):
-        return request.path_url
+        return self.settings.urlparse(request.url).reqpath
 
     def close(self):
         self.pools.clear()

requests_unixsocket/settings.py

@@ -0,0 +1,26 @@
+from collections import namedtuple
+
+from requests.compat import urlparse, unquote
+
+
+class Settings(object):
+    class ParseResult(namedtuple('ParseResult', 'sockpath reqpath')):
+        pass
+
+    def __init__(self, urlparse=None):
+        self.urlparse = urlparse
+
+
+def default_urlparse(url):
+    parsed_url = urlparse(url)
+    reqpath = parsed_url.path
+    if parsed_url.query:
+        reqpath += '?' + parsed_url.query
+    return Settings.ParseResult(
+        sockpath=unquote(parsed_url.netloc),
+        reqpath=reqpath,
+    )
+
+
+default_scheme = 'http+unix://'
+default_settings = Settings(urlparse=default_urlparse)

requests_unixsocket/tests/test_requests_unixsocket.py

@@ -4,22 +4,54 @@
 """Tests for requests_unixsocket"""
 
 import logging
+import os
+import stat
 
 import pytest
 import requests
+from requests.compat import urlparse
 
-import requests_unixsocket
+from requests_unixsocket import monkeypatch, Session, Settings, UnixAdapter
 from requests_unixsocket.testutils import UnixSocketServerThread
 
 
 logger = logging.getLogger(__name__)
 
 
+def is_socket(path):
+    try:
+        mode = os.stat(path).st_mode
+        return stat.S_ISSOCK(mode)
+    except OSError:
+        return False
+
+
+def get_sock_prefix(path):
+    """Keep going up directory tree until we find a socket"""
+
+    sockpath = path
+    reqpath_parts = []
+
+    while not is_socket(sockpath):
+        sockpath, tail = os.path.split(sockpath)
+        reqpath_parts.append(tail)
+
+    return Settings.ParseResult(
+        sockpath=sockpath,
+        reqpath='/' + os.path.join(*reversed(reqpath_parts)),
+    )
+
+
+alt_settings_1 = Settings(
+    urlparse=lambda url: get_sock_prefix(urlparse(url).path),
+)
+
+
 def test_use_UnixAdapter_directly():
     """Test using UnixAdapter directly, because
     https://github.com/httpie/httpie-unixsocket does this
     """
-    adapter = requests_unixsocket.UnixAdapter()
+    adapter = UnixAdapter()
     prepared_request = requests.Request(
         method='GET',
         url='http+unix://%2Fvar%2Frun%2Fdocker.sock/info',
@@ -30,7 +62,7 @@ def test_use_UnixAdapter_directly():
 
 def test_unix_domain_adapter_ok():
     with UnixSocketServerThread() as usock_thread:
-        session = requests_unixsocket.Session('http+unix://')
+        session = Session('http+unix://')
         urlencoded_usock = requests.compat.quote_plus(usock_thread.usock)
         url = 'http+unix://%s/path/to/page' % urlencoded_usock
 
@@ -46,7 +78,35 @@ def test_unix_domain_adapter_ok():
             assert r.headers['X-Transport'] == 'unix domain socket'
             assert r.headers['X-Requested-Path'] == '/path/to/page'
             assert r.headers['X-Socket-Path'] == usock_thread.usock
-            assert isinstance(r.connection, requests_unixsocket.UnixAdapter)
+            assert isinstance(r.connection, UnixAdapter)
+            assert r.url.lower() == url.lower()
+            if method == 'head':
+                assert r.text == ''
+            else:
+                assert r.text == 'Hello world!'
+
+
+def test_unix_domain_adapter_alt_settings_1_ok():
+    with UnixSocketServerThread() as usock_thread:
+        session = Session(
+            url_scheme='http+unix://',
+            settings=alt_settings_1,
+        )
+        url = 'http+unix://localhost%s/path/to/page' % usock_thread.usock
+
+        for method in ['get', 'post', 'head', 'patch', 'put', 'delete',
+                       'options']:
+            logger.debug('Calling session.%s(%r) ...', method, url)
+            r = getattr(session, method)(url)
+            logger.debug(
+                'Received response: %r with text: %r and headers: %r',
+                r, r.text, r.headers)
+            assert r.status_code == 200
+            assert r.headers['server'] == 'waitress'
+            assert r.headers['X-Transport'] == 'unix domain socket'
+            assert r.headers['X-Requested-Path'] == '/path/to/page'
+            assert r.headers['X-Socket-Path'] == usock_thread.usock
+            assert isinstance(r.connection, UnixAdapter)
             assert r.url.lower() == url.lower()
             if method == 'head':
                 assert r.text == ''
@@ -56,7 +116,7 @@ def test_unix_domain_adapter_ok():
 
 def test_unix_domain_adapter_url_with_query_params():
     with UnixSocketServerThread() as usock_thread:
-        session = requests_unixsocket.Session('http+unix://')
+        session = Session('http+unix://')
         urlencoded_usock = requests.compat.quote_plus(usock_thread.usock)
         url = ('http+unix://%s'
                '/containers/nginx/logs?timestamp=true' % urlencoded_usock)
@@ -74,7 +134,34 @@ def test_unix_domain_adapter_url_with_query_params():
             assert r.headers['X-Requested-Path'] == '/containers/nginx/logs'
             assert r.headers['X-Requested-Query-String'] == 'timestamp=true'
             assert r.headers['X-Socket-Path'] == usock_thread.usock
-            assert isinstance(r.connection, requests_unixsocket.UnixAdapter)
+            assert isinstance(r.connection, UnixAdapter)
+            assert r.url.lower() == url.lower()
+            if method == 'head':
+                assert r.text == ''
+            else:
+                assert r.text == 'Hello world!'
+
+
+def test_unix_domain_adapter_url_with_fragment():
+    with UnixSocketServerThread() as usock_thread:
+        session = Session('http+unix://')
+        urlencoded_usock = requests.compat.quote_plus(usock_thread.usock)
+        url = ('http+unix://%s'
+               '/containers/nginx/logs#some-fragment' % urlencoded_usock)
+
+        for method in ['get', 'post', 'head', 'patch', 'put', 'delete',
+                       'options']:
+            logger.debug('Calling session.%s(%r) ...', method, url)
+            r = getattr(session, method)(url)
+            logger.debug(
+                'Received response: %r with text: %r and headers: %r',
+                r, r.text, r.headers)
+            assert r.status_code == 200
+            assert r.headers['server'] == 'waitress'
+            assert r.headers['X-Transport'] == 'unix domain socket'
+            assert r.headers['X-Requested-Path'] == '/containers/nginx/logs'
+            assert r.headers['X-Socket-Path'] == usock_thread.usock
+            assert isinstance(r.connection, UnixAdapter)
             assert r.url.lower() == url.lower()
             if method == 'head':
                 assert r.text == ''
@@ -83,7 +170,7 @@ def test_unix_domain_adapter_url_with_query_params():
 
 
 def test_unix_domain_adapter_connection_error():
-    session = requests_unixsocket.Session('http+unix://')
+    session = Session('http+unix://')
 
     for method in ['get', 'post', 'head', 'patch', 'put', 'delete', 'options']:
         with pytest.raises(requests.ConnectionError):
@@ -92,7 +179,7 @@ def test_unix_domain_adapter_connection_error():
 
 
 def test_unix_domain_adapter_connection_proxies_error():
-    session = requests_unixsocket.Session('http+unix://')
+    session = Session('http+unix://')
 
     for method in ['get', 'post', 'head', 'patch', 'put', 'delete', 'options']:
         with pytest.raises(ValueError) as excinfo:
@@ -105,7 +192,7 @@ def test_unix_domain_adapter_connection_proxies_error():
 
 def test_unix_domain_adapter_monkeypatch():
     with UnixSocketServerThread() as usock_thread:
-        with requests_unixsocket.monkeypatch('http+unix://'):
+        with monkeypatch('http+unix://'):
             urlencoded_usock = requests.compat.quote_plus(usock_thread.usock)
             url = 'http+unix://%s/path/to/page' % urlencoded_usock
 
@@ -122,7 +209,7 @@ def test_unix_domain_adapter_monkeypatch():
                 assert r.headers['X-Requested-Path'] == '/path/to/page'
                 assert r.headers['X-Socket-Path'] == usock_thread.usock
                 assert isinstance(r.connection,
-                                  requests_unixsocket.UnixAdapter)
+                                  UnixAdapter)
                 assert r.url.lower() == url.lower()
                 if method == 'head':
                     assert r.text == ''

tox.ini

@@ -26,6 +26,11 @@ deps =
     coverage
     {[testenv]deps}
 
+[testenv:dev]
+deps =
+    python-semantic-release
+    {[testenv]deps}
+
 [testenv:doctest]
 # note this only works under python 3 because of unicode literals
 commands =