diff --git a/youtube_dl/extractor/cbs.py b/youtube_dl/extractor/cbs.py index c79e55a75..c4e901b42 100644 --- a/youtube_dl/extractor/cbs.py +++ b/youtube_dl/extractor/cbs.py @@ -1,3 +1,4 @@ +# coding: utf-8 from __future__ import unicode_literals from .theplatform import ThePlatformFeedIE @@ -8,6 +9,7 @@ from ..utils import ( xpath_element, xpath_text, update_url_query, + url_or_none, ) @@ -25,68 +27,27 @@ class CBSBaseIE(ThePlatformFeedIE): }) return subtitles - -class CBSIE(CBSBaseIE): - _VALID_URL = r'(?:cbs:|https?://(?:www\.)?(?:(?:cbs|paramountplus)\.com/shows/[^/]+/video|colbertlateshow\.com/(?:video|podcasts))/)(?P[\w-]+)' - - _TESTS = [{ - 'url': 'http://www.cbs.com/shows/garth-brooks/video/_u7W953k6la293J7EPTd9oHkSPs6Xn6_/connect-chat-feat-garth-brooks/', - 'info_dict': { - 'id': '_u7W953k6la293J7EPTd9oHkSPs6Xn6_', - 'ext': 'mp4', - 'title': 'Connect Chat feat. Garth Brooks', - 'description': 'Connect with country music singer Garth Brooks, as he chats with fans on Wednesday November 27, 2013. Be sure to tune in to Garth Brooks: Live from Las Vegas, Friday November 29, at 9/8c on CBS!', - 'duration': 1495, - 'timestamp': 1385585425, - 'upload_date': '20131127', - 'uploader': 'CBSI-NEW', - }, - 'params': { - # m3u8 download - 'skip_download': True, - }, - '_skip': 'Blocked outside the US', - }, { - 'url': 'http://colbertlateshow.com/video/8GmB0oY0McANFvp2aEffk9jZZZ2YyXxy/the-colbeard/', - 'only_matching': True, - }, { - 'url': 'http://www.colbertlateshow.com/podcasts/dYSwjqPs_X1tvbV_P2FcPWRa_qT6akTC/in-the-bad-room-with-stephen/', - 'only_matching': True, - }, { - 'url': 'https://www.paramountplus.com/shows/all-rise/video/QmR1WhNkh1a_IrdHZrbcRklm176X_rVc/all-rise-space/', - 'only_matching': True, - }] - - def _extract_video_info(self, content_id, site='cbs', mpx_acc=2198311517): - items_data = self._download_xml( - 'http://can.cbs.com/thunder/player/videoPlayerService.php', - content_id, query={'partner': site, 'contentId': content_id}) - video_data = xpath_element(items_data, './/item') - title = xpath_text(video_data, 'videoTitle', 'title', True) + def _extract_common_video_info(self, content_id, asset_types, mpx_acc, extra_info): tp_path = 'dJ5BDC/media/guid/%d/%s' % (mpx_acc, content_id) tp_release_url = 'http://link.theplatform.com/s/' + tp_path + info = self._extract_theplatform_metadata(tp_path, content_id) - asset_types = [] - subtitles = {} - formats = [] + formats, subtitles = [], {} last_e = None - for item in items_data.findall('.//item'): - asset_type = xpath_text(item, 'assetType') - if not asset_type or asset_type in asset_types or 'HLS_FPS' in asset_type or 'DASH_CENC' in asset_type: - continue - asset_types.append(asset_type) - query = { - 'mbr': 'true', - 'assetTypes': asset_type, - } - if asset_type.startswith('HLS') or asset_type in ('OnceURL', 'StreamPack'): - query['formats'] = 'MPEG4,M3U' - elif asset_type in ('RTMP', 'WIFI', '3G'): - query['formats'] = 'MPEG4,FLV' + for asset_type, query in asset_types.items(): try: tp_formats, tp_subtitles = self._extract_theplatform_smil( update_url_query(tp_release_url, query), content_id, 'Downloading %s SMIL data' % asset_type) + except ExtractorError as e: + last_e = e + if asset_type != 'fallback': + continue + query['formats'] = '' # blank query to check if expired + try: + tp_formats, tp_subtitles = self._extract_theplatform_smil( + update_url_query(tp_release_url, query), content_id, + 'Downloading %s SMIL data, trying again with another format' % asset_type) except ExtractorError as e: last_e = e continue @@ -96,20 +57,117 @@ class CBSIE(CBSBaseIE): raise last_e self._sort_formats(formats) - info = self._extract_theplatform_metadata(tp_path, content_id) - info.update({ + extra_info.update({ 'id': content_id, + 'formats': formats, + 'subtitles': subtitles, + }) + info.update({k: v for k, v in extra_info.items() if v is not None}) + return info + + def _extract_video_info(self, *args, **kwargs): + # Extract assets + metadata and call _extract_common_video_info + raise NotImplementedError('This method must be implemented by subclasses') + + def _real_extract(self, url): + return self._extract_video_info(self._match_id(url)) + + +class CBSIE(CBSBaseIE): + _VALID_URL = r'''(?x) + (?: + cbs:| + https?://(?:www\.)?(?: + cbs\.com/(?:shows|movies)/(?:video|[^/]+/video|[^/]+)/| + colbertlateshow\.com/(?:video|podcasts)/) + )(?P[\w-]+)''' + + # All tests are blocked outside US + _TESTS = [{ + 'url': 'https://www.cbs.com/shows/video/xrUyNLtl9wd8D_RWWAg9NU2F_V6QpB3R/', + 'info_dict': { + 'id': 'xrUyNLtl9wd8D_RWWAg9NU2F_V6QpB3R', + 'ext': 'mp4', + 'title': 'Tough As Nails - Dreams Never Die', + 'description': 'md5:a3535a62531cdd52b0364248a2c1ae33', + 'duration': 2588, + 'timestamp': 1639015200, + 'upload_date': '20211209', + 'uploader': 'CBSI-NEW', + }, + 'params': { + # m3u8 download + 'skip_download': True, + }, + '_skip': 'Blocked outside the US', + }, { + 'url': 'https://www.cbs.com/shows/video/sZH1MGgomIosZgxGJ1l263MFq16oMtW1/', + 'info_dict': { + 'id': 'sZH1MGgomIosZgxGJ1l263MFq16oMtW1', + 'title': 'The Late Show - 3/16/22 (Michael Buble, Rose Matafeo)', + 'timestamp': 1647488100, + 'description': 'md5:d0e6ec23c544b7fa8e39a8e6844d2439', + 'uploader': 'CBSI-NEW', + 'upload_date': '20220317', + }, + 'params': { + 'ignore_no_formats_error': True, + 'skip_download': True, + }, + 'expected_warnings': [ + 'This content expired on', 'No video formats found', 'Requested format is not available'], + }, { + 'url': 'http://colbertlateshow.com/video/8GmB0oY0McANFvp2aEffk9jZZZ2YyXxy/the-colbeard/', + 'only_matching': True, + }, { + 'url': 'http://www.colbertlateshow.com/podcasts/dYSwjqPs_X1tvbV_P2FcPWRa_qT6akTC/in-the-bad-room-with-stephen/', + 'only_matching': True, + }, { + # old format + 'url': 'http://www.cbs.com/shows/garth-brooks/video/_u7W953k6la293J7EPTd9oHkSPs6Xn6_/connect-chat-feat-garth-brooks/', + 'only_matching': True, + }] + + def _extract_video_info(self, content_id, site='cbs', mpx_acc=2198311517): + items_data = self._download_xml( + 'https://can.cbs.com/thunder/player/videoPlayerService.php', + content_id, query={'partner': site, 'contentId': content_id}) + video_data = xpath_element(items_data, './/item') + title = xpath_text(video_data, 'videoTitle', 'title') or xpath_text(video_data, 'videotitle', 'title') + + asset_types = {} + has_drm = False + for item in items_data.findall('.//item'): + asset_type = xpath_text(item, 'assetType') + query = { + 'mbr': 'true', + 'assetTypes': asset_type, + } + if not asset_type: + # fallback for content_ids that videoPlayerService doesn't return anything for + asset_type = 'fallback' + query['formats'] = 'M3U+none,MPEG4,M3U+appleHlsEncryption,MP3' + del query['assetTypes'] + if asset_type in asset_types: + continue + elif any(excluded in asset_type for excluded in ('HLS_FPS', 'DASH_CENC', 'OnceURL')): + if 'DASH_CENC' in asset_type: + has_drm = True + continue + if asset_type.startswith('HLS') or 'StreamPack' in asset_type: + query['formats'] = 'MPEG4,M3U' + elif asset_type in ('RTMP', 'WIFI', '3G'): + query['formats'] = 'MPEG4,FLV' + asset_types[asset_type] = query + + if not asset_types and has_drm: + raise ExtractorError('Only DRM formats found', video_id=content_id, expected=True) + + return self._extract_common_video_info(content_id, asset_types, mpx_acc, extra_info={ 'title': title, 'series': xpath_text(video_data, 'seriesTitle'), 'season_number': int_or_none(xpath_text(video_data, 'seasonNumber')), 'episode_number': int_or_none(xpath_text(video_data, 'episodeNumber')), 'duration': int_or_none(xpath_text(video_data, 'videoLength'), 1000), - 'thumbnail': xpath_text(video_data, 'previewImageURL'), - 'formats': formats, - 'subtitles': subtitles, + 'thumbnail': url_or_none(xpath_text(video_data, 'previewImageURL')), }) - return info - - def _real_extract(self, url): - content_id = self._match_id(url) - return self._extract_video_info(content_id) diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py index 3da5f8020..08292f88c 100644 --- a/youtube_dl/extractor/extractors.py +++ b/youtube_dl/extractor/extractors.py @@ -917,6 +917,10 @@ from .palcomp3 import ( PalcoMP3VideoIE, ) from .pandoratv import PandoraTVIE +from .paramountplus import ( + ParamountPlusIE, + ParamountPlusSeriesIE, +) from .parliamentliveuk import ParliamentLiveUKIE from .patreon import PatreonIE from .pbs import PBSIE diff --git a/youtube_dl/extractor/paramountplus.py b/youtube_dl/extractor/paramountplus.py new file mode 100644 index 000000000..8b1908906 --- /dev/null +++ b/youtube_dl/extractor/paramountplus.py @@ -0,0 +1,159 @@ +# coding: utf-8 +from __future__ import unicode_literals +import itertools + +from .common import InfoExtractor +from .cbs import CBSBaseIE +from ..utils import ( + int_or_none, + url_or_none, +) + + +class ParamountPlusIE(CBSBaseIE): + _VALID_URL = r'''(?x) + (?: + paramountplus:| + https?://(?:www\.)?(?: + paramountplus\.com/(?:shows|movies)/(?:video|[^/]+/video|[^/]+)/ + )(?P[\w-]+))''' + + # All tests are blocked outside US + _TESTS = [{ + 'url': 'https://www.paramountplus.com/shows/video/Oe44g5_NrlgiZE3aQVONleD6vXc8kP0k/', + 'info_dict': { + 'id': 'Oe44g5_NrlgiZE3aQVONleD6vXc8kP0k', + 'ext': 'mp4', + 'title': 'CatDog - Climb Every CatDog/The Canine Mutiny', + 'description': 'md5:7ac835000645a69933df226940e3c859', + 'duration': 1426, + 'timestamp': 920264400, + 'upload_date': '19990301', + 'uploader': 'CBSI-NEW', + }, + 'params': { + 'skip_download': 'm3u8', + }, + }, { + 'url': 'https://www.paramountplus.com/shows/video/6hSWYWRrR9EUTz7IEe5fJKBhYvSUfexd/', + 'info_dict': { + 'id': '6hSWYWRrR9EUTz7IEe5fJKBhYvSUfexd', + 'ext': 'mp4', + 'title': '7/23/21 WEEK IN REVIEW (Rep. Jahana Hayes/Howard Fineman/Sen. Michael Bennet/Sheera Frenkel & Cecilia Kang)', + 'description': 'md5:f4adcea3e8b106192022e121f1565bae', + 'duration': 2506, + 'timestamp': 1627063200, + 'upload_date': '20210723', + 'uploader': 'CBSI-NEW', + }, + 'params': { + 'skip_download': 'm3u8', + }, + }, { + 'url': 'https://www.paramountplus.com/movies/video/vM2vm0kE6vsS2U41VhMRKTOVHyQAr6pC/', + 'info_dict': { + 'id': 'vM2vm0kE6vsS2U41VhMRKTOVHyQAr6pC', + 'ext': 'mp4', + 'title': 'Daddy\'s Home', + 'upload_date': '20151225', + 'description': 'md5:9a6300c504d5e12000e8707f20c54745', + 'uploader': 'CBSI-NEW', + 'timestamp': 1451030400, + }, + 'params': { + 'skip_download': 'm3u8', + 'format': 'bestvideo', + }, + 'expected_warnings': ['Ignoring subtitle tracks'], # TODO: Investigate this + }, { + 'url': 'https://www.paramountplus.com/movies/video/5EKDXPOzdVf9voUqW6oRuocyAEeJGbEc/', + 'info_dict': { + 'id': '5EKDXPOzdVf9voUqW6oRuocyAEeJGbEc', + 'ext': 'mp4', + 'uploader': 'CBSI-NEW', + 'description': 'md5:bc7b6fea84ba631ef77a9bda9f2ff911', + 'timestamp': 1577865600, + 'title': 'Sonic the Hedgehog', + 'upload_date': '20200101', + }, + 'params': { + 'skip_download': 'm3u8', + 'format': 'bestvideo', + }, + 'expected_warnings': ['Ignoring subtitle tracks'], + }, { + 'url': 'https://www.paramountplus.com/shows/the-real-world/video/mOVeHeL9ub9yWdyzSZFYz8Uj4ZBkVzQg/the-real-world-reunion/', + 'only_matching': True, + }, { + 'url': 'https://www.paramountplus.com/shows/video/mOVeHeL9ub9yWdyzSZFYz8Uj4ZBkVzQg/', + 'only_matching': True, + }, { + 'url': 'https://www.paramountplus.com/movies/video/W0VyStQqUnqKzJkrpSAIARuCc9YuYGNy/', + 'only_matching': True, + }, { + 'url': 'https://www.paramountplus.com/movies/paw-patrol-the-movie/W0VyStQqUnqKzJkrpSAIARuCc9YuYGNy/', + 'only_matching': True, + }, { + # old format + 'url': 'https://www.paramountplus.com/shows/catdog/video/Oe44g5_NrlgiZE3aQVONleD6vXc8kP0k/catdog-climb-every-catdog-the-canine-mutiny/', + 'only_matching': True, + }] + + def _extract_video_info(self, content_id, mpx_acc=2198311517): + items_data = self._download_json( + 'https://www.paramountplus.com/apps-api/v2.0/androidtv/video/cid/%s.json' % content_id, + content_id, query={'locale': 'en-us', 'at': 'ABCqWNNSwhIqINWIIAG+DFzcFUvF8/vcN6cNyXFFfNzWAIvXuoVgX+fK4naOC7V8MLI='}, headers=self.geo_verification_headers()) + + asset_types = { + item.get('assetType'): { + 'format': 'SMIL', + 'formats': 'MPEG4,M3U', + } for item in items_data['itemList'] + } + item = items_data['itemList'][-1] + return self._extract_common_video_info(content_id, asset_types, mpx_acc, extra_info={ + 'title': item.get('title'), + 'series': item.get('seriesTitle'), + 'season_number': int_or_none(item.get('seasonNum')), + 'episode_number': int_or_none(item.get('episodeNum')), + 'duration': int_or_none(item.get('duration')), + 'thumbnail': url_or_none(item.get('thumbnail')), + }) + + +class ParamountPlusSeriesIE(InfoExtractor): + _VALID_URL = r'https?://(?:www\.)?paramountplus\.com/shows/(?P[a-zA-Z0-9-_]+)/?(?:[#?]|$)' + _TESTS = [{ + 'url': 'https://www.paramountplus.com/shows/drake-josh', + 'playlist_mincount': 45, + 'info_dict': { + 'id': 'drake-josh', + } + }, { + 'url': 'https://www.paramountplus.com/shows/hawaii_five_0/', + 'playlist_mincount': 240, + 'info_dict': { + 'id': 'hawaii_five_0', + } + }, { + 'url': 'https://www.paramountplus.com/shows/spongebob-squarepants/', + 'playlist_mincount': 248, + 'info_dict': { + 'id': 'spongebob-squarepants', + } + }] + + def _entries(self, show_name): + for page in itertools.count(): + show_json = self._download_json( + 'https://www.paramountplus.com/shows/{0}/xhr/episodes/page/{1}/size/50/xs/0/season/0'.format(show_name, page), show_name) + if not show_json.get('success'): + return + for episode in show_json['result']['data']: + yield self.url_result( + 'https://www.paramountplus.com%s' % episode['url'], + ie=ParamountPlusIE.ie_key(), video_id=episode['content_id']) + + def _real_extract(self, url): + show_name = self._match_id(url) + return self.playlist_result(self._entries(show_name), playlist_id=show_name) diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py index adfe11e31..65d583ce6 100644 --- a/youtube_dl/extractor/theplatform.py +++ b/youtube_dl/extractor/theplatform.py @@ -20,6 +20,7 @@ from ..utils import ( float_or_none, int_or_none, sanitized_Request, + try_get, unsmuggle_url, update_url_query, xpath_with_ns, @@ -34,6 +35,15 @@ _x = lambda p: xpath_with_ns(p, {'smil': default_ns}) class ThePlatformBaseIE(OnceIE): _TP_TLD = 'com' + @classmethod + def _match_valid_url(cls, url): + # This does not use has/getattr intentionally - we want to know whether + # we have cached the regexp for *this* class, whereas getattr would also + # match the superclass + if '_VALID_URL_RE' not in cls.__dict__: + cls._VALID_URL_RE = re.compile(cls._VALID_URL) + return cls._VALID_URL_RE.match(url) + def _extract_theplatform_smil(self, smil_url, video_id, note='Downloading SMIL data'): meta = self._download_xml( smil_url, video_id, note=note, query={'format': 'SMIL'}, @@ -238,7 +248,7 @@ class ThePlatformIE(ThePlatformBaseIE, AdobePassIE): 'countries': smuggled_data.get('geo_countries'), }) - mobj = re.match(self._VALID_URL, url) + mobj = self._match_valid_url(url) provider_id = mobj.group('provider_id') video_id = mobj.group('id') @@ -338,6 +348,7 @@ class ThePlatformFeedIE(ThePlatformBaseIE): 'categories': ['MSNBC/Issues/Democrats', 'MSNBC/Issues/Elections/Election 2016'], 'uploader': 'NBCU-NEWS', }, + 'expected_warnings': ('Empty metadata',), }, { 'url': 'http://feed.theplatform.com/f/2E2eJC/nnd_NBCNews?byGuid=nn_netcast_180306.Copy.01', 'only_matching': True, @@ -345,7 +356,11 @@ class ThePlatformFeedIE(ThePlatformBaseIE): def _extract_feed_info(self, provider_id, feed_id, filter_query, video_id, custom_fields=None, asset_types_query={}, account_id=None): real_url = self._URL_TEMPLATE % (self.http_scheme(), provider_id, feed_id, filter_query) - entry = self._download_json(real_url, video_id)['entries'][0] + entry = self._download_json(real_url, video_id) + entry = try_get(entry, lambda x: x['entries'][0], dict) + if not entry: + self.report_warning('Empty metadata', video_id) + return None main_smil_url = 'http://link.theplatform.com/s/%s/media/guid/%d/%s' % (provider_id, account_id, entry['guid']) if account_id else entry.get('plmedia$publicUrl') formats = [] @@ -404,7 +419,7 @@ class ThePlatformFeedIE(ThePlatformBaseIE): return ret def _real_extract(self, url): - mobj = re.match(self._VALID_URL, url) + mobj = self._match_valid_url(url) video_id = mobj.group('id') provider_id = mobj.group('provider_id')