Merge branch 'yt-dlp:master' into ie-piramidetv

.github/ISSUE_TEMPLATE/1_broken_site.yml

@@ -77,3 +77,8 @@ body:
       render: shell
     validations:
       required: true
+  - type: markdown
+    attributes:
+      value: |
+        ### NOTE: Due to a recent increase in malicious spam activity, this issue will be automatically locked until it is triaged by a maintainer.
+        ### If you receive any replies asking you download a file, do NOT follow the download links!

.github/ISSUE_TEMPLATE/2_site_support_request.yml

@@ -89,3 +89,8 @@ body:
       render: shell
     validations:
       required: true
+  - type: markdown
+    attributes:
+      value: |
+        ### NOTE: Due to a recent increase in malicious spam activity, this issue will be automatically locked until it is triaged by a maintainer.
+        ### If you receive any replies asking you download a file, do NOT follow the download links!

.github/ISSUE_TEMPLATE/3_site_feature_request.yml

@@ -85,3 +85,8 @@ body:
       render: shell
     validations:
       required: true
+  - type: markdown
+    attributes:
+      value: |
+        ### NOTE: Due to a recent increase in malicious spam activity, this issue will be automatically locked until it is triaged by a maintainer.
+        ### If you receive any replies asking you download a file, do NOT follow the download links!

.github/ISSUE_TEMPLATE/4_bug_report.yml

@@ -70,3 +70,8 @@ body:
       render: shell
     validations:
       required: true
+  - type: markdown
+    attributes:
+      value: |
+        ### NOTE: Due to a recent increase in malicious spam activity, this issue will be automatically locked until it is triaged by a maintainer.
+        ### If you receive any replies asking you download a file, do NOT follow the download links!

.github/ISSUE_TEMPLATE/5_feature_request.yml

@@ -64,3 +64,8 @@ body:
         [youtube] Extracting URL: https://www.youtube.com/watch?v=BaW_jenozKc
         <more lines>
       render: shell
+  - type: markdown
+    attributes:
+      value: |
+        ### NOTE: Due to a recent increase in malicious spam activity, this issue will be automatically locked until it is triaged by a maintainer.
+        ### If you receive any replies asking you download a file, do NOT follow the download links!

.github/ISSUE_TEMPLATE/6_question.yml

@@ -70,3 +70,8 @@ body:
         [youtube] Extracting URL: https://www.youtube.com/watch?v=BaW_jenozKc
         <more lines>
       render: shell
+  - type: markdown
+    attributes:
+      value: |
+        ### NOTE: Due to a recent increase in malicious spam activity, this issue will be automatically locked until it is triaged by a maintainer.
+        ### If you receive any replies asking you download a file, do NOT follow the download links!

.github/workflows/antispam.yaml

@@ -0,0 +1,20 @@
+name: Anti-Spam
+on:
+  issues:
+    types: [opened]
+
+permissions:
+  issues: write
+
+jobs:
+  lockdown:
+    name: Issue Lockdown
+    runs-on: ubuntu-latest
+    steps:
+      - name: "Lock new issue"
+        env:
+          GH_TOKEN: ${{ github.token }}
+          ISSUE_NUMBER: ${{ github.event.issue.number }}
+          REPOSITORY: ${{ github.repository }}
+        run: |
+          gh issue lock "${ISSUE_NUMBER}" -r too_heated -R "${REPOSITORY}"

.github/workflows/core.yml

@@ -55,6 +55,7 @@ jobs:
     - name: Install test requirements
       run: python3 ./devscripts/install_deps.py --include test --include curl-cffi
     - name: Run tests
+      timeout-minutes: 15
       continue-on-error: False
       run: |
         python3 -m yt_dlp -v || true  # Print debug head

.github/workflows/quick-test.yml

@@ -17,6 +17,7 @@ jobs:
     - name: Install test requirements
       run: python3 ./devscripts/install_deps.py --include test
     - name: Run tests
+      timeout-minutes: 15
       run: |
         python3 -m yt_dlp -v || true
         python3 ./devscripts/run_tests.py core

devscripts/make_issue_template.py

@@ -46,6 +46,11 @@
       render: shell
     validations:
       required: true
+  - type: markdown
+    attributes:
+      value: |
+        ### NOTE: Due to a recent increase in malicious spam activity, this issue will be automatically locked until it is triaged by a maintainer.
+        ### If you receive any replies asking you download a file, do NOT follow the download links!
 '''.strip()
 
 NO_SKIP = '''

pyproject.toml

@@ -49,7 +49,7 @@ dependencies = [
     "pycryptodomex",
     "requests>=2.32.2,<3",
     "urllib3>=1.26.17,<3",
-    "websockets>=12.0",
+    "websockets>=13.0",
 ]
 
 [project.optional-dependencies]

test/test_websockets.py

@@ -88,7 +88,7 @@ def create_wss_websocket_server():
     certfn = os.path.join(TEST_DIR, 'testcert.pem')
     sslctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
     sslctx.load_cert_chain(certfn, None)
-    return create_websocket_server(ssl_context=sslctx)
+    return create_websocket_server(ssl=sslctx)
 
 
 MTLS_CERT_DIR = os.path.join(TEST_DIR, 'testdata', 'certificate')
@@ -103,7 +103,7 @@ def create_mtls_wss_websocket_server():
     sslctx.load_verify_locations(cafile=cacertfn)
     sslctx.load_cert_chain(certfn, None)
 
-    return create_websocket_server(ssl_context=sslctx)
+    return create_websocket_server(ssl=sslctx)
 
 
 def create_legacy_wss_websocket_server():
@@ -112,7 +112,7 @@ def create_legacy_wss_websocket_server():
     sslctx.maximum_version = ssl.TLSVersion.TLSv1_2
     sslctx.set_ciphers('SHA1:AESCCM:aDSS:eNULL:aNULL')
     sslctx.load_cert_chain(certfn, None)
-    return create_websocket_server(ssl_context=sslctx)
+    return create_websocket_server(ssl=sslctx)
 
 
 def ws_validate_and_send(rh, req):
@@ -139,7 +139,7 @@ def setup_class(cls):
         cls.wss_thread, cls.wss_port = create_wss_websocket_server()
         cls.wss_base_url = f'wss://127.0.0.1:{cls.wss_port}'
 
-        cls.bad_wss_thread, cls.bad_wss_port = create_websocket_server(ssl_context=ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER))
+        cls.bad_wss_thread, cls.bad_wss_port = create_websocket_server(ssl=ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER))
         cls.bad_wss_host = f'wss://127.0.0.1:{cls.bad_wss_port}'
 
         cls.mtls_wss_thread, cls.mtls_wss_port = create_mtls_wss_websocket_server()

yt_dlp/extractor/_extractors.py

@@ -2316,6 +2316,7 @@
     VideomoreVideoIE,
 )
 from .videopress import VideoPressIE
+from .vidflex import VidflexIE
 from .vidio import (
     VidioIE,
     VidioLiveIE,

yt_dlp/extractor/asobistage.py

@@ -101,9 +101,10 @@ def _real_initialize(self):
         self._HEADERS['Authorization'] = f'Bearer {token}'
 
     def _real_extract(self, url):
-        video_id, event, type_, slug = self._match_valid_url(url).group('id', 'event', 'type', 'slug')
+        webpage, urlh = self._download_webpage_handle(url, self._match_id(url))
+        video_id, event, type_, slug = self._match_valid_url(urlh.url).group('id', 'event', 'type', 'slug')
         video_type = {'archive': 'archives', 'player': 'broadcasts'}[type_]
-        webpage = self._download_webpage(url, video_id)
+
         event_data = traverse_obj(
             self._search_nextjs_data(webpage, video_id, default={}),
             ('props', 'pageProps', 'eventCMSData', {

yt_dlp/extractor/eurosport.py

@@ -3,7 +3,12 @@
 
 
 class EurosportIE(InfoExtractor):
-    _VALID_URL = r'https?://www\.eurosport\.com/\w+/(?:[\w-]+/[\d-]+/)?[\w-]+_(?P<id>vid\d+)'
+    _VALID_URL = r'''(?x)
+        https?://(?:
+            (?:(?:www|espanol)\.)?eurosport\.(?:com(?:\.tr)?|de|dk|es|fr|hu|it|nl|no|ro)|
+            eurosport\.tvn24\.pl
+        )/[\w-]+/(?:[\w-]+/[\d-]+/)?[\w.-]+_(?P<id>vid\d+)
+    '''
     _TESTS = [{
         'url': 'https://www.eurosport.com/tennis/roland-garros/2022/highlights-rafael-nadal-brushes-aside-caper-ruud-to-win-record-extending-14th-french-open-title_vid1694147/video.shtml',
         'info_dict': {
@@ -70,13 +75,50 @@ class EurosportIE(InfoExtractor):
             'duration': 105.0,
             'upload_date': '20230518',
         },
+    }, {
+        'url': 'https://www.eurosport.de/radsport/vuelta-a-espana/2024/vuelta-a-espana-2024-wout-van-aert-und-co.-verzweifeln-an-mcnulty-zeitfahr-krimi-in-lissabon_vid2219478/video.shtml',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.eurosport.dk/speedway/mikkel-michelsen-misser-finalen-i-cardiff-se-danskeren-i-semifinalen-her_vid2219363/video.shtml',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.eurosport.nl/mixed-martial-arts/ufc/2022/ufc-305-respect-tussen-adesanya-en-du-plessis_vid2219650/video.shtml',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.eurosport.es/ciclismo/la-vuelta-2024-carlos-rodriguez-olvida-la-crono-y-ya-espera-que-llegue-la-montana-no-me-encontre-nada-comodo_vid2219682/video.shtml',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.eurosport.fr/football/supercoupe-d-europe/2024-2025/kylian-mbappe-vinicius-junior-eduardo-camavinga-touche.-extraits-de-l-entrainement-du-real-madrid-en-video_vid2216993/video.shtml',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.eurosport.it/calcio/serie-a/2024-2025/samardzic-a-bergamo-per-le-visite-mediche-con-l-atalanta_vid2219680/video.shtml',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.eurosport.hu/kerekpar/vuelta-a-espana/2024/dramai-harc-a-masodpercekert-meglepetesgyoztes-a-vuelta-nyitoszakaszan_vid2219481/video.shtml',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.eurosport.no/golf/fedex-st-jude-championship/2024/ligger-pa-andreplass-sa-skjer-dette-drama_vid30000618/video.shtml',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.eurosport.no/golf/fedex-st-jude-championship/2024/ligger-pa-andreplass-sa-skjer-dette-drama_vid2219531/video.shtml',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.eurosport.ro/tenis/western-southern-open-2/2024/rezumatul-partidei-dintre-zverev-si-shelton-de-la-cincinnati_vid2219657/video.shtml',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.eurosport.com.tr/hentbol/olympic-games-paris-2024/2024/paris-2024-denmark-ile-germany-olimpiyatlarin-onemli-anlari_vid2215836/video.shtml',
+        'only_matching': True,
+    }, {
+        'url': 'https://eurosport.tvn24.pl/kolarstwo/tour-de-france-kobiet/2024/kasia-niewiadoma-przed-ostatnim-8.-etapem-tour-de-france-kobiet_vid2219765/video.shtml',
+        'only_matching': True,
     }]
 
     _TOKEN = None
 
     # actually defined in https://netsport.eurosport.io/?variables={"databaseId":<databaseId>,"playoutType":"VDP"}&extensions={"persistedQuery":{"version":1 ..
     # but this method require to get sha256 hash
     _GEO_COUNTRIES = ['DE', 'NL', 'EU', 'IT', 'FR']  # Not complete list but it should work
+    _GEO_BYPASS = False
 
     def _real_initialize(self):
         if EurosportIE._TOKEN is None:
@@ -98,13 +140,13 @@ def _real_extract(self, url):
         for stream_type in json_data['attributes']['streaming']:
             if stream_type == 'hls':
                 fmts, subs = self._extract_m3u8_formats_and_subtitles(
-                    traverse_obj(json_data, ('attributes', 'streaming', stream_type, 'url')), display_id, ext='mp4')
+                    traverse_obj(json_data, ('attributes', 'streaming', stream_type, 'url')), display_id, ext='mp4', fatal=False)
             elif stream_type == 'dash':
                 fmts, subs = self._extract_mpd_formats_and_subtitles(
-                    traverse_obj(json_data, ('attributes', 'streaming', stream_type, 'url')), display_id)
+                    traverse_obj(json_data, ('attributes', 'streaming', stream_type, 'url')), display_id, fatal=False)
             elif stream_type == 'mss':
                 fmts, subs = self._extract_ism_formats_and_subtitles(
-                    traverse_obj(json_data, ('attributes', 'streaming', stream_type, 'url')), display_id)
+                    traverse_obj(json_data, ('attributes', 'streaming', stream_type, 'url')), display_id, fatal=False)
 
             formats.extend(fmts)
             self._merge_subtitles(subs, target=subtitles)

yt_dlp/extractor/radiko.py

@@ -7,6 +7,7 @@
 from ..utils import (
     ExtractorError,
     clean_html,
+    join_nonempty,
     time_seconds,
     try_call,
     unified_timestamp,
@@ -167,7 +168,7 @@ def _extract_performers(self, prog):
 
 
 class RadikoIE(RadikoBaseIE):
-    _VALID_URL = r'https?://(?:www\.)?radiko\.jp/#!/ts/(?P<station>[A-Z0-9-]+)/(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?radiko\.jp/#!/ts/(?P<station>[A-Z0-9-]+)/(?P<timestring>\d+)'
 
     _TESTS = [{
         # QRR (文化放送) station provides <desc>
@@ -183,8 +184,9 @@ class RadikoIE(RadikoBaseIE):
     }]
 
     def _real_extract(self, url):
-        station, video_id = self._match_valid_url(url).groups()
-        vid_int = unified_timestamp(video_id, False)
+        station, timestring = self._match_valid_url(url).group('station', 'timestring')
+        video_id = join_nonempty(station, timestring)
+        vid_int = unified_timestamp(timestring, False)
         prog, station_program, ft, radio_begin, radio_end = self._find_program(video_id, station, vid_int)
 
         auth_token, area_id = self._auth_client()
@@ -207,7 +209,7 @@ def _real_extract(self, url):
                     'ft': radio_begin,
                     'end_at': radio_end,
                     'to': radio_end,
-                    'seek': video_id,
+                    'seek': timestring,
                 },
             ),
         }

yt_dlp/extractor/rutube.py

@@ -6,6 +6,7 @@
     determine_ext,
     int_or_none,
     parse_qs,
+    traverse_obj,
     try_get,
     unified_timestamp,
     url_or_none,
@@ -80,6 +81,8 @@ def _extract_formats(self, options, video_id):
                     'url': format_url,
                     'format_id': format_id,
                 })
+        for hls_url in traverse_obj(options, ('live_streams', 'hls', ..., 'url', {url_or_none})):
+            formats.extend(self._extract_m3u8_formats(hls_url, video_id, ext='mp4', fatal=False))
         return formats
 
     def _download_and_extract_formats(self, video_id, query=None):
@@ -90,7 +93,7 @@ def _download_and_extract_formats(self, video_id, query=None):
 class RutubeIE(RutubeBaseIE):
     IE_NAME = 'rutube'
     IE_DESC = 'Rutube videos'
-    _VALID_URL = r'https?://rutube\.ru/(?:video(?:/private)?|(?:play/)?embed)/(?P<id>[\da-z]{32})'
+    _VALID_URL = r'https?://rutube\.ru/(?:(?:live/)?video(?:/private)?|(?:play/)?embed)/(?P<id>[\da-z]{32})'
     _EMBED_REGEX = [r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//rutube\.ru/(?:play/)?embed/[\da-z]{32}.*?)\1']
 
     _TESTS = [{
@@ -164,6 +167,29 @@ class RutubeIE(RutubeBaseIE):
             'uploader': 'Стас Быков',
         },
         'expected_warnings': ['Unable to download f4m'],
+    }, {
+        'url': 'https://rutube.ru/live/video/c58f502c7bb34a8fcdd976b221fca292/',
+        'info_dict': {
+            'id': 'c58f502c7bb34a8fcdd976b221fca292',
+            'ext': 'mp4',
+            'categories': ['Телепередачи'],
+            'description': '',
+            'thumbnail': 'http://pic.rutubelist.ru/video/14/19/14190807c0c48b40361aca93ad0867c7.jpg',
+            'live_status': 'is_live',
+            'age_limit': 0,
+            'uploader_id': '23460655',
+            'timestamp': 1652972968,
+            'view_count': int,
+            'upload_date': '20220519',
+            'title': r're:Первый канал. Прямой эфир \d{4}-\d{2}-\d{2} \d{2}:\d{2}$',
+            'uploader': 'Первый канал',
+        },
+    }, {
+        'url': 'https://rutube.ru/video/5ab908fccfac5bb43ef2b1e4182256b0/',
+        'only_matching': True,
+    }, {
+        'url': 'https://rutube.ru/live/video/private/c58f502c7bb34a8fcdd976b221fca292/',
+        'only_matching': True,
     }]
 
     @classmethod

yt_dlp/extractor/tvn24.py

@@ -8,7 +8,7 @@
 
 class TVN24IE(InfoExtractor):
     _WORKING = False
-    _VALID_URL = r'https?://(?:(?:[^/]+)\.)?tvn24(?:bis)?\.pl/(?:[^/]+/)*(?P<id>[^/]+)'
+    _VALID_URL = r'https?://(?:(?!eurosport)[^/]+\.)?tvn24(?:bis)?\.pl/(?:[^/?#]+/)*(?P<id>[^/?#]+)'
     _TESTS = [{
         'url': 'http://www.tvn24.pl/wiadomosci-z-kraju,3/oredzie-artura-andrusa,702428.html',
         'md5': 'fbdec753d7bc29d96036808275f2130c',