From bb4b6517823ccfd0a035f40638fdf7323b61ef99 Mon Sep 17 00:00:00 2001 From: quancore Date: Thu, 2 Jul 2020 22:45:45 +0200 Subject: [PATCH 1/2] Adding optional SIGINT handler. It raises RuntimeError when run on a thread other then main thread since add_signal_handler only work in the main thread --- proxybroker/api.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/proxybroker/api.py b/proxybroker/api.py index 9c94247..cb99f75 100644 --- a/proxybroker/api.py +++ b/proxybroker/api.py @@ -43,6 +43,8 @@ class Broker: (optional) Flag indicating whether to check the SSL certificates. Set to True to check ssl certifications :param loop: (optional) asyncio compatible event loop + :param stop_broker_on_sigint: (optional) whether set SIGINT signal on broker object. + Useful for a thread other than main thread. .. deprecated:: 0.2.0 Use :attr:`max_conn` and :attr:`max_tries` instead of @@ -59,6 +61,7 @@ def __init__( providers=None, verify_ssl=False, loop=None, + stop_broker_on_sigint=True, **kwargs, ): self._loop = loop or asyncio.get_event_loop() @@ -101,13 +104,13 @@ def __init__( p if isinstance(p, Provider) else Provider(p) for p in (providers or PROVIDERS) ] - - try: - self._loop.add_signal_handler(signal.SIGINT, self.stop) - # add_signal_handler() is not implemented on Win - # https://docs.python.org/3.5/library/asyncio-eventloops.html#windows - except NotImplementedError: - pass + if stop_broker_on_sigint: + try: + self._loop.add_signal_handler(signal.SIGINT, self.stop) + # add_signal_handler() is not implemented on Win + # https://docs.python.org/3.5/library/asyncio-eventloops.html#windows + except NotImplementedError: + pass async def grab(self, *, countries=None, limit=0): """Gather proxies from the providers without checking. From 525adf5a8e8424eb37d826d88c279411cd9cc3c5 Mon Sep 17 00:00:00 2001 From: quancore Date: Sat, 4 Jul 2020 19:41:09 +0200 Subject: [PATCH 2/2] bugfix on get_real_ext_ip method --- proxybroker/resolver.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/proxybroker/resolver.py b/proxybroker/resolver.py index 1a661d4..80c304c 100644 --- a/proxybroker/resolver.py +++ b/proxybroker/resolver.py @@ -36,6 +36,8 @@ class Resolver: 'http://ipinfo.io/ip', 'http://ifconfig.io/ip', ] + # the list of resolvers will point a copy of original one + _temp_host = [] def __init__(self, timeout=5, loop=None): self._timeout = timeout @@ -85,13 +87,16 @@ def get_ip_info(ip): return GeoData(code, name, region_code, region_name, city_name) def _pop_random_ip_host(self): - host = random.choice(self._ip_hosts) - self._ip_hosts.remove(host) + host = random.choice(self._temp_host) + self._temp_host.remove(host) return host async def get_real_ext_ip(self): """Return real external IP address.""" - while self._ip_hosts: + # make a copy of original one to temp one + # so original one will stay no change + self._temp_host = self._ip_hosts.copy() + while self._temp_host: try: timeout = aiohttp.ClientTimeout(total=self._timeout) async with aiohttp.ClientSession(