[core] Rework HTTP networking

test/test_InfoExtractor.py

@@ -4,10 +4,13 @@
 import sys
 import unittest
 
+from yt_dlp.networking import UrllibRH
+from test.test_networking import RequestHandlerTestBase, with_request_handlers
+
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 import threading
-from test.helper import FakeYDL, expect_dict, expect_value, http_server_port
+from test.helper import expect_dict, expect_value, http_server_port
 
 from yt_dlp.compat import compat_etree_fromstring, compat_http_server
 from yt_dlp.extractor import YoutubeIE, get_info_extractor
@@ -41,9 +44,9 @@ class DummyIE(InfoExtractor):
     pass
 
 
-class TestInfoExtractor(unittest.TestCase):
+class TestInfoExtractor(RequestHandlerTestBase, unittest.TestCase):
     def setUp(self):
-        self.ie = DummyIE(FakeYDL())
+        self.ie = DummyIE(self.make_ydl())
 
     def test_ie_key(self):
         self.assertEqual(get_info_extractor(YoutubeIE.ie_key()), YoutubeIE)
@@ -303,14 +306,16 @@ def test_search_json_ld_realworld(self):
                 expected_dict
             )
 
+    @with_request_handlers(handlers=[UrllibRH])
     def test_download_json(self):
+        ie = DummyIE(self.make_ydl())
         uri = encode_data_uri(b'{"foo": "blah"}', 'application/json')
-        self.assertEqual(self.ie._download_json(uri, None), {'foo': 'blah'})
+        self.assertEqual(ie._download_json(uri, None), {'foo': 'blah'})
         uri = encode_data_uri(b'callback({"foo": "blah"})', 'application/javascript')
-        self.assertEqual(self.ie._download_json(uri, None, transform_source=strip_jsonp), {'foo': 'blah'})
+        self.assertEqual(ie._download_json(uri, None, transform_source=strip_jsonp), {'foo': 'blah'})
         uri = encode_data_uri(b'{"foo": invalid}', 'application/json')
-        self.assertRaises(ExtractorError, self.ie._download_json, uri, None)
-        self.assertEqual(self.ie._download_json(uri, None, fatal=False), None)
+        self.assertRaises(ExtractorError, ie._download_json, uri, None)
+        self.assertEqual(ie._download_json(uri, None, fatal=False), None)
 
     def test_parse_html5_media_entries(self):
         # inline video tag
@@ -1648,6 +1653,7 @@ def test_parse_xspf(self):
                 for i in range(len(entries)):
                     expect_dict(self, entries[i], expected_entries[i])
 
+    @with_request_handlers()
     def test_response_with_expected_status_returns_content(self):
         # Checks for mitigations against the effects of
         # <https://bugs.python.org/issue15002> that affect Python 3.4.1+, which
@@ -1662,7 +1668,7 @@ def test_response_with_expected_status_returns_content(self):
         server_thread.daemon = True
         server_thread.start()
 
-        (content, urlh) = self.ie._download_webpage_handle(
+        content, urlh = DummyIE(self.make_ydl())._download_webpage_handle(
             'http://127.0.0.1:%d/teapot' % port, None,
             expected_status=TEAPOT_RESPONSE_STATUS)
         self.assertEqual(content, TEAPOT_RESPONSE_BODY)

test/test_YoutubeDL.py

@@ -26,6 +26,7 @@
     OnDemandPagedList,
     int_or_none,
     match_filter_func,
+    RequestError,
 )
 
 TEST_URL = 'http://localhost/sample.mp4'
@@ -1101,7 +1102,7 @@ def test_selection(params, expected_ids, evaluate_all=False):
     def test_urlopen_no_file_protocol(self):
         # see https://github.com/ytdl-org/youtube-dl/issues/8227
         ydl = YDL()
-        self.assertRaises(compat_urllib_error.URLError, ydl.urlopen, 'file:///etc/passwd')
+        self.assertRaises(RequestError, ydl.urlopen, 'file:///etc/passwd')
 
     def test_do_not_override_ie_key_in_url_transparent(self):
         ydl = YDL()

test/test_YoutubeDLCookieJar.py

@@ -7,7 +7,7 @@
 
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-from yt_dlp.utils import YoutubeDLCookieJar
+from yt_dlp.cookies import YoutubeDLCookieJar
 
 
 class TestYoutubeDLCookieJar(unittest.TestCase):

test/test_download.py

@@ -21,15 +21,12 @@
 )
 
 import yt_dlp.YoutubeDL
-from yt_dlp.compat import (
-    compat_http_client,
-    compat_HTTPError,
-    compat_urllib_error,
-)
 from yt_dlp.extractor import get_info_extractor
 from yt_dlp.utils import (
     DownloadError,
     ExtractorError,
+    HTTPError,
+    TransportError,
     UnavailableVideoError,
     format_bytes,
 )
@@ -167,7 +164,7 @@ def try_rm_tcs_files(tcs=None):
                         force_generic_extractor=params.get('force_generic_extractor', False))
                 except (DownloadError, ExtractorError) as err:
                     # Check if the exception is not a network related one
-                    if not err.exc_info[0] in (compat_urllib_error.URLError, socket.timeout, UnavailableVideoError, compat_http_client.BadStatusLine) or (err.exc_info[0] == compat_HTTPError and err.exc_info[1].code == 503):
+                    if not isinstance(err.exc_info[1], (TransportError, UnavailableVideoError)) or (isinstance(err.exc_info[1], HTTPError) and err.exc_info[1].code == 503):
                         raise
 
                     if try_num == RETRIES:
@@ -250,7 +247,7 @@ def try_rm_tcs_files(tcs=None):
                 # extractor returns full results even with extract_flat
                 res_tcs = [{'info_dict': e} for e in res_dict['entries']]
                 try_rm_tcs_files(res_tcs)
-
+            ydl.close()
     return test_template
 
 

test/test_downloader_http.py

@@ -5,6 +5,8 @@
 import sys
 import unittest
 
+from test.test_networking import RequestHandlerTestBase, with_request_handlers
+
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 import threading
@@ -76,7 +78,7 @@ def error(self, msg):
         pass
 
 
-class TestHttpFD(unittest.TestCase):
+class TestHttpFD(RequestHandlerTestBase, unittest.TestCase):
     def setUp(self):
         self.httpd = compat_http_server.HTTPServer(
             ('127.0.0.1', 0), HTTPTestRequestHandler)
@@ -87,7 +89,7 @@ def setUp(self):
 
     def download(self, params, ep):
         params['logger'] = FakeLogger()
-        ydl = YoutubeDL(params)
+        ydl = self.make_ydl(params, fake=False)
         downloader = HttpFD(ydl, params)
         filename = 'testfile.mp4'
         try_rm(encodeFilename(filename))
@@ -101,9 +103,11 @@ def download_all(self, params):
         for ep in ('regular', 'no-content-length', 'no-range', 'no-range-no-content-length'):
             self.download(params, ep)
 
+    @with_request_handlers()
     def test_regular(self):
         self.download_all({})
 
+    @with_request_handlers()
     def test_chunked(self):
         self.download_all({
             'http_chunk_size': 1000,