From 8861b4b414f3cd5c8badf6f78c1fa01e8e259c21 Mon Sep 17 00:00:00 2001 From: Elliot Algase Date: Fri, 7 Jun 2019 23:57:46 -0400 Subject: [PATCH 1/2] [asiancrush] fix extractor, add support for yuyutv and midnightpulp --- youtube_dl/extractor/asiancrush.py | 108 +++++++++++++++++++++-------- 1 file changed, 81 insertions(+), 27 deletions(-) diff --git a/youtube_dl/extractor/asiancrush.py b/youtube_dl/extractor/asiancrush.py index 6d71c5ad5f8..de9bf2bdec2 100644 --- a/youtube_dl/extractor/asiancrush.py +++ b/youtube_dl/extractor/asiancrush.py @@ -5,14 +5,13 @@ from .common import InfoExtractor from .kaltura import KalturaIE -from ..utils import ( - extract_attributes, - remove_end, -) +from ..utils import extract_attributes class AsianCrushIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?asiancrush\.com/video/(?:[^/]+/)?0+(?P\d+)v\b' + IE_NAME = 'asiancrush' + _DOMAINS = r'(?:asiancrush\.com|yuyutv\.com|midnightpulp\.com)' + _VALID_URL = r'https?://(?:www\.)?(?P%s)/video/(?:[^/]+/)?0+(?P\d+)v\b' % _DOMAINS _TESTS = [{ 'url': 'https://www.asiancrush.com/video/012869v/women-who-flirt/', 'md5': 'c3b740e48d0ba002a42c0b72857beae6', @@ -20,7 +19,7 @@ class AsianCrushIE(InfoExtractor): 'id': '1_y4tmjm5r', 'ext': 'mp4', 'title': 'Women Who Flirt', - 'description': 'md5:3db14e9186197857e7063522cb89a805', + 'description': 'md5:7e986615808bcfb11756eb503a751487', 'timestamp': 1496936429, 'upload_date': '20170608', 'uploader_id': 'craig@crifkin.com', @@ -28,11 +27,42 @@ class AsianCrushIE(InfoExtractor): }, { 'url': 'https://www.asiancrush.com/video/she-was-pretty/011886v-pretty-episode-3/', 'only_matching': True, + }, { + 'url': 'https://www.yuyutv.com/video/013886v/the-act-of-killing/', + 'md5': '90dc82dfcd52e1f44c0b065bc1dd3827', + 'info_dict': { + 'id': '1_66x4rg7o', + 'ext': 'mp4', + 'title': 'The Act of Killing', + 'description': 'md5:76d745019ef586b3dc59c2a344758741', + 'timestamp': 1557419432, + 'upload_date': '20190509', + 'uploader_id': 'admin@ottera.tv', + }, + }, { + 'url': 'https://www.yuyutv.com/video/peep-show/013922v-warring-factions/', + 'only_matching': True, + }, { + 'url': 'https://www.midnightpulp.com/video/010400v/drifters/', + 'md5': '44c7993934434afd7f1ee928ef0f734b', + 'info_dict': { + 'id': '1_sh88xpyz', + 'ext': 'mp4', + 'title': 'Drifters', + 'description': 'md5:beccb5d5d2955d6defbf88ea256607db', + 'timestamp': 1497470628, + 'upload_date': '20170614', + 'uploader_id': 'craig@crifkin.com', + }, + }, { + 'url': 'https://www.midnightpulp.com/video/mononoke/016378v-zashikiwarashi-part-1/', + 'only_matching': True, }] def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + host = mobj.group('host') video_id = self._match_id(url) - webpage = self._download_webpage(url, video_id) entry_id, partner_id, title = [None] * 3 @@ -51,7 +81,7 @@ def _real_extract(self, url): r'\bentry_id["\']\s*:\s*["\'](\d+)', webpage, 'entry id') player = self._download_webpage( - 'https://api.asiancrush.com/embeddedVideoPlayer', video_id, + 'https://api.%s/embeddedVideoPlayer' % host, video_id, query={'id': entry_id}) kaltura_id = self._search_regex( @@ -63,15 +93,23 @@ def _real_extract(self, url): r'/p(?:artner_id)?/(\d+)', player, 'partner id', default='513551') - return self.url_result( - 'kaltura:%s:%s' % (partner_id, kaltura_id), - ie=KalturaIE.ie_key(), video_id=kaltura_id, - video_title=title) + description = self._html_search_regex(r'
(.+?)
', webpage, 'description', fatal=False, flags=re.DOTALL) + + return { + '_type': 'url_transparent', + 'url': 'kaltura:%s:%s' % (partner_id, kaltura_id), + 'ie_key': KalturaIE.ie_key(), + 'id': video_id, + 'title': title, + 'description': description, + } class AsianCrushPlaylistIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?asiancrush\.com/series/0+(?P\d+)s\b' - _TEST = { + _VIDEO_IE = AsianCrushIE + _DOMAINS = r'(?:asiancrush\.com|yuyutv\.com|midnightpulp\.com)' + _VALID_URL = r'https?://(?:www\.)?(?P%s)/series/0+(?P\d+)s\b' % _DOMAINS + _TESTS = [{ 'url': 'https://www.asiancrush.com/series/012481s/scholar-walks-night/', 'info_dict': { 'id': '12481', @@ -79,7 +117,23 @@ class AsianCrushPlaylistIE(InfoExtractor): 'description': 'md5:7addd7c5132a09fd4741152d96cce886', }, 'playlist_count': 20, - } + }, { + 'url': 'https://www.yuyutv.com/series/013920s/peep-show/', + 'info_dict': { + 'id': '13920', + 'title': 'Peep Show', + 'description': 'md5:15b59e05eb41fd2082671577eaf1b02d', + }, + 'playlist_count': 54, + }, { + 'url': 'https://www.midnightpulp.com/series/016375s/mononoke/', + 'info_dict': { + 'id': '16375', + 'title': 'Mononoke', + 'description': 'md5:4a6d4f36eb1165f1854dbc9ac3b83c07', + }, + 'playlist_count': 12, + }] def _real_extract(self, url): playlist_id = self._match_id(url) @@ -89,22 +143,22 @@ def _real_extract(self, url): entries = [] for mobj in re.finditer( - r']+href=(["\'])(?P%s.*?)\1[^>]*>' % AsianCrushIE._VALID_URL, + r']+href=(["\'])(?P%s.*?)\1[^>]*>' % self._VIDEO_IE._VALID_URL, webpage): attrs = extract_attributes(mobj.group(0)) if attrs.get('class') == 'clearfix': entries.append(self.url_result( - mobj.group('url'), ie=AsianCrushIE.ie_key())) - - title = remove_end( - self._html_search_regex( - r'(?s)]\bid=["\']movieTitle[^>]+>(.+?)', webpage, - 'title', default=None) or self._og_search_title( - webpage, default=None) or self._html_search_meta( - 'twitter:title', webpage, 'title', - default=None) or self._search_regex( - r'([^<]+)', webpage, 'title', fatal=False), - ' | AsianCrush') + mobj.group('url'), ie=self._VIDEO_IE.ie_key())) + + title = re.sub(r'\s*\|\s*.+?$', '', + self._html_search_regex( + r'(?s)]\bid=["\']movieTitle[^>]+>(.+?)', webpage, + 'title', default=None) or self._og_search_title( + webpage, default=None) or self._html_search_meta( + 'twitter:title', webpage, 'title', + default=None) or self._search_regex( + r'([^<]+)', webpage, 'title', fatal=False) + ) description = self._og_search_description( webpage, default=None) or self._html_search_meta( From 5aaa26a9dc24ab5780d53155484558652dbb81ed Mon Sep 17 00:00:00 2001 From: Elliot Algase Date: Sun, 14 Jul 2019 00:13:50 -0400 Subject: [PATCH 2/2] [asiancrush] improve coding conventions --- youtube_dl/extractor/asiancrush.py | 35 +++++++++++++++--------------- 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/youtube_dl/extractor/asiancrush.py b/youtube_dl/extractor/asiancrush.py index de9bf2bdec2..8118c5d46bc 100644 --- a/youtube_dl/extractor/asiancrush.py +++ b/youtube_dl/extractor/asiancrush.py @@ -10,8 +10,7 @@ class AsianCrushIE(InfoExtractor): IE_NAME = 'asiancrush' - _DOMAINS = r'(?:asiancrush\.com|yuyutv\.com|midnightpulp\.com)' - _VALID_URL = r'https?://(?:www\.)?(?P%s)/video/(?:[^/]+/)?0+(?P\d+)v\b' % _DOMAINS + _VALID_URL = r'https?://(?:www\.)?(?P(?:asiancrush\.com|yuyutv\.com|midnightpulp\.com))/video/(?:[^/]+/)?0+(?P\d+)v\b' _TESTS = [{ 'url': 'https://www.asiancrush.com/video/012869v/women-who-flirt/', 'md5': 'c3b740e48d0ba002a42c0b72857beae6', @@ -93,7 +92,8 @@ def _real_extract(self, url): r'/p(?:artner_id)?/(\d+)', player, 'partner id', default='513551') - description = self._html_search_regex(r'
(.+?)
', webpage, 'description', fatal=False, flags=re.DOTALL) + description = self._html_search_regex( + r'
(.+?)
', webpage, 'description', fatal=False, flags=re.DOTALL) return { '_type': 'url_transparent', @@ -106,9 +106,7 @@ def _real_extract(self, url): class AsianCrushPlaylistIE(InfoExtractor): - _VIDEO_IE = AsianCrushIE - _DOMAINS = r'(?:asiancrush\.com|yuyutv\.com|midnightpulp\.com)' - _VALID_URL = r'https?://(?:www\.)?(?P%s)/series/0+(?P\d+)s\b' % _DOMAINS + _VALID_URL = r'https?://(?:www\.)?(?P(?:asiancrush\.com|yuyutv\.com|midnightpulp\.com))/series/0+(?P\d+)s\b' _TESTS = [{ 'url': 'https://www.asiancrush.com/series/012481s/scholar-walks-night/', 'info_dict': { @@ -143,22 +141,23 @@ def _real_extract(self, url): entries = [] for mobj in re.finditer( - r']+href=(["\'])(?P%s.*?)\1[^>]*>' % self._VIDEO_IE._VALID_URL, + r']+href=(["\'])(?P%s.*?)\1[^>]*>' % AsianCrushIE._VALID_URL, webpage): attrs = extract_attributes(mobj.group(0)) if attrs.get('class') == 'clearfix': entries.append(self.url_result( - mobj.group('url'), ie=self._VIDEO_IE.ie_key())) - - title = re.sub(r'\s*\|\s*.+?$', '', - self._html_search_regex( - r'(?s)]\bid=["\']movieTitle[^>]+>(.+?)', webpage, - 'title', default=None) or self._og_search_title( - webpage, default=None) or self._html_search_meta( - 'twitter:title', webpage, 'title', - default=None) or self._search_regex( - r'([^<]+)', webpage, 'title', fatal=False) - ) + mobj.group('url'), ie=AsianCrushIE.ie_key())) + + title = re.sub( + r'\s*\|\s*.+?$', '', + self._html_search_regex( + r'(?s)]\bid=["\']movieTitle[^>]+>(.+?)', webpage, + 'title', default=None) or self._og_search_title( + webpage, default=None) or self._html_search_meta( + 'twitter:title', webpage, 'title', + default=None) or self._search_regex( + r'([^<]+)', webpage, 'title', fatal=False) + ) description = self._og_search_description( webpage, default=None) or self._html_search_meta(