From 552e30e98832df169b2b616e37ebde172b92bf6e Mon Sep 17 00:00:00 2001 From: Morgan Harris Date: Tue, 21 Mar 2023 15:27:31 +1100 Subject: [PATCH 1/6] Allow downloading videos requiring login on 10play.com.au As far as I can tell this is now essentially every episode of every show. --- youtube_dl/extractor/tenplay.py | 97 +++++++++++++++++++++++---------- 1 file changed, 67 insertions(+), 30 deletions(-) diff --git a/youtube_dl/extractor/tenplay.py b/youtube_dl/extractor/tenplay.py index cd30d57f4..14a0d4b91 100644 --- a/youtube_dl/extractor/tenplay.py +++ b/youtube_dl/extractor/tenplay.py @@ -5,66 +5,103 @@ from .common import InfoExtractor from ..utils import ( HEADRequest, parse_age_limit, - parse_iso8601, - # smuggle_url, + unescapeHTML, + str_to_int, ) +from datetime import datetime +import base64 +import json + class TenPlayIE(InfoExtractor): _VALID_URL = r'https?://(?:www\.)?10play\.com\.au/(?:[^/]+/)+(?Ptpv\d{6}[a-z]{5})' _TESTS = [{ - 'url': 'https://10play.com.au/masterchef/episodes/season-1/masterchef-s1-ep-1/tpv190718kwzga', + 'url': 'https://10play.com.au/masterchef/episodes/season-1/episode-1/tpv220408msjpb', 'info_dict': { - 'id': '6060533435001', + 'id': 'tpv220408msjpb', 'ext': 'mp4', 'title': 'MasterChef - S1 Ep. 1', 'description': 'md5:4fe7b78e28af8f2d900cd20d900ef95c', 'age_limit': 10, 'timestamp': 1240828200, 'upload_date': '20090427', - 'uploader_id': '2199827728001', }, 'params': { # 'format': 'bestvideo', 'skip_download': True, + 'usenetrc': True, } }, { 'url': 'https://10play.com.au/how-to-stay-married/web-extras/season-1/terrys-talks-ep-1-embracing-change/tpv190915ylupc', 'only_matching': True, }] - # BRIGHTCOVE_URL_TEMPLATE = 'https://players.brightcove.net/2199827728001/cN6vRtRQt_default/index.html?videoId=%s' + + _NETRC_MACHINE = '10play.com.au' _GEO_BYPASS = False - _FASTLY_URL_TEMPL = 'https://10-selector.global.ssl.fastly.net/s/kYEXFC/media/%s?mbr=true&manifest=m3u&format=redirect' + + def get_access_token(self, content_id): + # log in with username and password + username, password = self._get_login_info() + + if username is None or password is None: + self.raise_login_required() + + ten_auth_header = base64.b64encode(datetime.utcnow().strftime("%Y%m%d%H%M%S").encode()) + + auth_request_data = json.dumps({"email": username, "password": password}).encode() + token_data = self._download_json( + 'https://10play.com.au/api/user/auth', content_id, + note='Logging in to 10play', + data=auth_request_data, + headers={ + 'Content-Type': 'application/json;charset=utf-8', + 'X-Network-Ten-Auth': ten_auth_header + }) + + access_token = token_data['jwt']['accessToken'] + + return {'Authorization': f"Bearer {access_token}"} def _real_extract(self, url): content_id = self._match_id(url) - data = self._download_json( - 'https://10play.com.au/api/video/' + content_id, content_id) - video = data.get('video') or {} - metadata = data.get('metaData') or {} - brightcove_id = video.get('videoId') or metadata['showContentVideoId'] - # brightcove_url = smuggle_url( - # self.BRIGHTCOVE_URL_TEMPLATE % brightcove_id, - # {'geo_countries': ['AU']}) - m3u8_url = self._request_webpage(HEADRequest( - self._FASTLY_URL_TEMPL % brightcove_id), brightcove_id).geturl() + + video_info = self._download_json( + 'https://10play.com.au/api/v1/videos/' + content_id, content_id, note='Fetching video info') + + playback_url = video_info['playbackApiEndpoint'] + + headers = {} + + # Handle member-gated videos + if video_info.get('memberGated'): + extra_headers = self.get_access_token(content_id) + headers.update(**extra_headers) + + playback_info = self._download_json( + playback_url, content_id, + note='Fetching playback info', + headers=headers) + + m3u8_url = self._request_webpage(HEADRequest(playback_info['source']), content_id).geturl() if '10play-not-in-oz' in m3u8_url: self.raise_geo_restricted(countries=['AU']) - formats = self._extract_m3u8_formats(m3u8_url, brightcove_id, 'mp4') + + formats = self._extract_m3u8_formats(m3u8_url, content_id, 'mp4') self._sort_formats(formats) return { - # '_type': 'url_transparent', - # 'url': brightcove_url, 'formats': formats, - 'id': brightcove_id, - 'title': video.get('title') or metadata.get('pageContentName') or metadata['showContentName'], - 'description': video.get('description'), - 'age_limit': parse_age_limit(video.get('showRatingClassification') or metadata.get('showProgramClassification')), - 'series': metadata.get('showName'), - 'season': metadata.get('showContentSeason'), - 'timestamp': parse_iso8601(metadata.get('contentPublishDate') or metadata.get('pageContentPublishDate')), - 'thumbnail': video.get('poster'), - 'uploader_id': '2199827728001', - # 'ie_key': 'BrightcoveNew', + 'id': content_id, + 'title': unescapeHTML(video_info.get('subtitle')) or unescapeHTML(video_info.get('title')), + 'description': unescapeHTML(video_info.get('description')), + 'age_limit': parse_age_limit(video_info.get('classification')), + 'series': video_info.get('tvShow'), + 'season': video_info.get('season'), + 'season_number': str_to_int(video_info.get('season')), + 'episode': unescapeHTML(video_info.get('title')), + 'episode_number': str_to_int(video_info.get('episode')), + 'timestamp': video_info.get('published'), + 'duration': video_info.get('duration'), + 'thumbnail': video_info.get('posterImageUrl'), } From 8b11dbba0051da0c5f22576a2593b15abe6a2deb Mon Sep 17 00:00:00 2001 From: Morgan Harris Date: Tue, 21 Mar 2023 17:25:12 +1100 Subject: [PATCH 2/6] Add support for downloading an entire season as a playlist --- youtube_dl/extractor/tenplay.py | 126 +++++++++++++++++++++++++------- 1 file changed, 100 insertions(+), 26 deletions(-) diff --git a/youtube_dl/extractor/tenplay.py b/youtube_dl/extractor/tenplay.py index 14a0d4b91..59937400e 100644 --- a/youtube_dl/extractor/tenplay.py +++ b/youtube_dl/extractor/tenplay.py @@ -3,10 +3,10 @@ from __future__ import unicode_literals from .common import InfoExtractor from ..utils import ( - HEADRequest, parse_age_limit, unescapeHTML, str_to_int, + urljoin ) from datetime import datetime @@ -15,7 +15,11 @@ import json class TenPlayIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?10play\.com\.au/(?:[^/]+/)+(?Ptpv\d{6}[a-z]{5})' + _VALID_URL = r'''(?x)^ + https?://(?:www\.)?10play\.com\.au/(?: + (?:[^/]+/)+(?Ptpv\d{6}[a-z]{5})| (?# Individual show id) + (?P[^/]+)/episodes/(?P[^/]+)(?# Entire season playlist) + )''' _TESTS = [{ 'url': 'https://10play.com.au/masterchef/episodes/season-1/episode-1/tpv220408msjpb', 'info_dict': { @@ -28,44 +32,118 @@ class TenPlayIE(InfoExtractor): 'upload_date': '20090427', }, 'params': { - # 'format': 'bestvideo', 'skip_download': True, 'usenetrc': True, } }, { 'url': 'https://10play.com.au/how-to-stay-married/web-extras/season-1/terrys-talks-ep-1-embracing-change/tpv190915ylupc', 'only_matching': True, + }, { + 'info_dict': { + 'title': 'Season 2022' + }, + 'url': 'https://10play.com.au/the-bold-and-the-beautiful-fast-tracked/episodes/season-2022', + 'playlist_count': 256, + 'params': { + 'skip_download': True, + 'usenetrc': True, + } }] _NETRC_MACHINE = '10play.com.au' _GEO_BYPASS = False + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._access_token = None + def get_access_token(self, content_id): - # log in with username and password - username, password = self._get_login_info() + if self._access_token is None: + # log in with username and password + username, password = self._get_login_info() - if username is None or password is None: - self.raise_login_required() + if username is None or password is None: + self.raise_login_required() - ten_auth_header = base64.b64encode(datetime.utcnow().strftime("%Y%m%d%H%M%S").encode()) + ten_auth_header = base64.b64encode(datetime.utcnow().strftime("%Y%m%d%H%M%S").encode()) - auth_request_data = json.dumps({"email": username, "password": password}).encode() - token_data = self._download_json( - 'https://10play.com.au/api/user/auth', content_id, - note='Logging in to 10play', - data=auth_request_data, - headers={ - 'Content-Type': 'application/json;charset=utf-8', - 'X-Network-Ten-Auth': ten_auth_header - }) + auth_request_data = json.dumps({"email": username, "password": password}).encode() + token_data = self._download_json( + 'https://10play.com.au/api/user/auth', content_id, + note='Logging in to 10play', + data=auth_request_data, + headers={ + 'Content-Type': 'application/json;charset=utf-8', + 'X-Network-Ten-Auth': ten_auth_header + }) - access_token = token_data['jwt']['accessToken'] + self._access_token = token_data['jwt']['accessToken'] - return {'Authorization': f"Bearer {access_token}"} + return self._access_token + + def extract_playlist(self, url): + matches = self._VALID_URL_RE.match(url) + show = matches.group('show') + season = matches.group('season') + + # The api/v1 endpoint is throwing up 403 Forbidden, so we need to use the old API + season_info = self._download_json( + f'https://10play.com.au/api/shows/{show}/episodes/{season}', f"{show}/{season}", + note='Fetching playlist info') + + # Try to find a carousel with the title "episodes", otherwise default to the top one + episodes_carousel = next((c for c in season_info['content'][0]['components'] if c['title'].lower() == 'episodes'), + season_info['content'][0]['components'][0]) + + episodes = episodes_carousel['slides'] + + load_more_url = urljoin(url, episodes_carousel['loadMoreUrl']) + + while episodes_carousel['hasMore']: + skip_ids = [ep['id'] for ep in episodes] + + episodes_carousel = self._download_json( + load_more_url, f"{show}/{season}", + note=f'Fetching episodes {len(skip_ids)}+', + query={'skipIds[]': skip_ids}) + + episodes += episodes_carousel['items'] + + episodes_urls = [urljoin(url, ep['cardLink']) for ep in episodes] + + return self.playlist_from_matches(episodes_urls, playlist_title=season_info['content'][0].get('title')) + + # Altered version to check for geoblocking without extraneous HEAD request + def _extract_m3u8_formats(self, m3u8_url, video_id, ext=None, + entry_protocol='m3u8', preference=None, + m3u8_id=None, note=None, errnote=None, + fatal=True, live=False, data=None, headers={}, + query={}): + res = self._download_webpage_handle( + m3u8_url, video_id, + note=note or 'Downloading m3u8 information', + errnote=errnote or 'Failed to download m3u8 information', + fatal=fatal, data=data, headers=headers, query=query) + + if res is False: + return [] + + m3u8_doc, urlh = res + m3u8_url = urlh.geturl() + + if '10play-not-in-oz' in m3u8_url: + self.raise_geo_restricted(countries=['AU']) + + return self._parse_m3u8_formats( + m3u8_doc, m3u8_url, ext=ext, entry_protocol=entry_protocol, + preference=preference, m3u8_id=m3u8_id, live=live) def _real_extract(self, url): content_id = self._match_id(url) + if content_id is None or content_id == 'None': + return self.extract_playlist(url) + video_info = self._download_json( 'https://10play.com.au/api/v1/videos/' + content_id, content_id, note='Fetching video info') @@ -75,19 +153,15 @@ class TenPlayIE(InfoExtractor): # Handle member-gated videos if video_info.get('memberGated'): - extra_headers = self.get_access_token(content_id) - headers.update(**extra_headers) + access_token = self.get_access_token(content_id) + headers.update(Authorization=f"Bearer {access_token}") playback_info = self._download_json( playback_url, content_id, note='Fetching playback info', headers=headers) - m3u8_url = self._request_webpage(HEADRequest(playback_info['source']), content_id).geturl() - if '10play-not-in-oz' in m3u8_url: - self.raise_geo_restricted(countries=['AU']) - - formats = self._extract_m3u8_formats(m3u8_url, content_id, 'mp4') + formats = self._extract_m3u8_formats(playback_info['source'], content_id, 'mp4') self._sort_formats(formats) return { From def8836ca9e729e75beb4f3b5d07453e9361b85b Mon Sep 17 00:00:00 2001 From: dirkf Date: Wed, 11 Sep 2024 17:34:13 +0100 Subject: [PATCH 3/6] Back-port PR yt-dlp/yt-dlp#10928 * back-port PR * improve geo-block detection --- youtube_dl/extractor/tenplay.py | 328 +++++++++++++++++--------------- 1 file changed, 174 insertions(+), 154 deletions(-) diff --git a/youtube_dl/extractor/tenplay.py b/youtube_dl/extractor/tenplay.py index 59937400e..197c45a3a 100644 --- a/youtube_dl/extractor/tenplay.py +++ b/youtube_dl/extractor/tenplay.py @@ -1,181 +1,201 @@ # coding: utf-8 from __future__ import unicode_literals +import functools + from .common import InfoExtractor +from ..compat import ( + compat_itertools_count, + compat_kwargs, +) from ..utils import ( - parse_age_limit, - unescapeHTML, - str_to_int, - urljoin + get_elements_by_attribute, + HEADRequest, + float_or_none, + int_or_none, + merge_dicts, + NO_DEFAULT, + T, + traverse_obj, + txt_or_none, + url_or_none, + urljoin, ) -from datetime import datetime -import base64 -import json + +class TenPlayBase(InfoExtractor): + _GEO_COUNTRIES = ['AU'] + _GEO_BYPASS = False + + def raise_geo_restricted(self, *args, **kwargs): + countries = args[1] if len(args) > 1 else kwargs.get('countries', NO_DEFAULT) + if countries is NO_DEFAULT: + kwargs['countries'] = self._GEO_COUNTRIES + kwargs = compat_kwargs(kwargs) + super(TenPlayBase, self).raise_geo_restricted(*args, **kwargs) + + def _download_webpage_handle(self, url_or_request, video_id, *args, **kwargs): + res = super(TenPlayBase, self)._download_webpage_handle(url_or_request, video_id, *args, **kwargs) + if res and any('Sorry, 10 play is not available in your region.' in e + for e in get_elements_by_attribute('class', 'iserror__text', res[0])): + self.raise_geo_restricted() + return res -class TenPlayIE(InfoExtractor): - _VALID_URL = r'''(?x)^ - https?://(?:www\.)?10play\.com\.au/(?: - (?:[^/]+/)+(?Ptpv\d{6}[a-z]{5})| (?# Individual show id) - (?P[^/]+)/episodes/(?P[^/]+)(?# Entire season playlist) - )''' +class TenPlayIE(TenPlayBase): + _VALID_URL = r'https?://(?:www\.)?10play\.com\.au/(?:[^/]+/)+(?Ptpv\d{6}[a-z]{5})' + _NETRC_MACHINE = '10play' _TESTS = [{ - 'url': 'https://10play.com.au/masterchef/episodes/season-1/episode-1/tpv220408msjpb', + 'url': 'https://10play.com.au/neighbours/web-extras/season-41/heres-a-first-look-at-mischa-bartons-neighbours-debut/tpv230911hyxnz', 'info_dict': { - 'id': 'tpv220408msjpb', + 'id': '6336940246112', 'ext': 'mp4', - 'title': 'MasterChef - S1 Ep. 1', - 'description': 'md5:4fe7b78e28af8f2d900cd20d900ef95c', - 'age_limit': 10, - 'timestamp': 1240828200, - 'upload_date': '20090427', + 'title': 'Here\'s A First Look At Mischa Barton\'s Neighbours Debut', + 'alt_title': 'Here\'s A First Look At Mischa Barton\'s Neighbours Debut', + 'description': 'Neighbours Premieres Monday, September 18 At 4:30pm On 10 And 10 Play And 6:30pm On 10 Peach', + 'duration': 74, + 'season': 'Season 41', + 'season_number': 41, + 'series': 'Neighbours', + 'thumbnail': r're:https://.*\.jpg', + 'uploader': 'Channel 10', + 'age_limit': 15, + 'timestamp': 1694386800, + 'upload_date': '20230910', + 'uploader_id': '2199827728001', }, 'params': { 'skip_download': True, - 'usenetrc': True, - } + }, + 'skip': 'Only available in Australia', + }, { + 'url': 'https://10play.com.au/neighbours/episodes/season-42/episode-9107/tpv240902nzqyp', + 'info_dict': { + 'id': '9000000000091177', + 'ext': 'mp4', + 'title': 'Neighbours - S42 Ep. 9107', + 'alt_title': 'Thu 05 Sep', + 'description': 'md5:37a1f4271be34b9ee2b533426a5fbaef', + 'duration': 1388, + 'episode': 'Episode 9107', + 'episode_number': 9107, + 'season': 'Season 42', + 'season_number': 42, + 'series': 'Neighbours', + 'thumbnail': r're:https://.*\.jpg', + 'age_limit': 15, + 'timestamp': 1725517860, + 'upload_date': '20240905', + 'uploader': 'Channel 10', + 'uploader_id': '2199827728001', + }, + 'params': { + 'skip_download': True, + }, + 'skip': 'Only available in Australia', }, { 'url': 'https://10play.com.au/how-to-stay-married/web-extras/season-1/terrys-talks-ep-1-embracing-change/tpv190915ylupc', 'only_matching': True, - }, { - 'info_dict': { - 'title': 'Season 2022' - }, - 'url': 'https://10play.com.au/the-bold-and-the-beautiful-fast-tracked/episodes/season-2022', - 'playlist_count': 256, - 'params': { - 'skip_download': True, - 'usenetrc': True, - } }] - _NETRC_MACHINE = '10play.com.au' - _GEO_BYPASS = False - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self._access_token = None - - def get_access_token(self, content_id): - if self._access_token is None: - # log in with username and password - username, password = self._get_login_info() - - if username is None or password is None: - self.raise_login_required() - - ten_auth_header = base64.b64encode(datetime.utcnow().strftime("%Y%m%d%H%M%S").encode()) - - auth_request_data = json.dumps({"email": username, "password": password}).encode() - token_data = self._download_json( - 'https://10play.com.au/api/user/auth', content_id, - note='Logging in to 10play', - data=auth_request_data, - headers={ - 'Content-Type': 'application/json;charset=utf-8', - 'X-Network-Ten-Auth': ten_auth_header - }) - - self._access_token = token_data['jwt']['accessToken'] - - return self._access_token - - def extract_playlist(self, url): - matches = self._VALID_URL_RE.match(url) - show = matches.group('show') - season = matches.group('season') - - # The api/v1 endpoint is throwing up 403 Forbidden, so we need to use the old API - season_info = self._download_json( - f'https://10play.com.au/api/shows/{show}/episodes/{season}', f"{show}/{season}", - note='Fetching playlist info') - - # Try to find a carousel with the title "episodes", otherwise default to the top one - episodes_carousel = next((c for c in season_info['content'][0]['components'] if c['title'].lower() == 'episodes'), - season_info['content'][0]['components'][0]) - - episodes = episodes_carousel['slides'] - - load_more_url = urljoin(url, episodes_carousel['loadMoreUrl']) - - while episodes_carousel['hasMore']: - skip_ids = [ep['id'] for ep in episodes] - - episodes_carousel = self._download_json( - load_more_url, f"{show}/{season}", - note=f'Fetching episodes {len(skip_ids)}+', - query={'skipIds[]': skip_ids}) - - episodes += episodes_carousel['items'] - - episodes_urls = [urljoin(url, ep['cardLink']) for ep in episodes] - - return self.playlist_from_matches(episodes_urls, playlist_title=season_info['content'][0].get('title')) - - # Altered version to check for geoblocking without extraneous HEAD request - def _extract_m3u8_formats(self, m3u8_url, video_id, ext=None, - entry_protocol='m3u8', preference=None, - m3u8_id=None, note=None, errnote=None, - fatal=True, live=False, data=None, headers={}, - query={}): - res = self._download_webpage_handle( - m3u8_url, video_id, - note=note or 'Downloading m3u8 information', - errnote=errnote or 'Failed to download m3u8 information', - fatal=fatal, data=data, headers=headers, query=query) - - if res is False: - return [] - - m3u8_doc, urlh = res - m3u8_url = urlh.geturl() - - if '10play-not-in-oz' in m3u8_url: - self.raise_geo_restricted(countries=['AU']) - - return self._parse_m3u8_formats( - m3u8_doc, m3u8_url, ext=ext, entry_protocol=entry_protocol, - preference=preference, m3u8_id=m3u8_id, live=live) + _AUS_AGES = { + 'G': 0, + 'PG': 15, + 'M': 15, + 'MA': 15, + 'MA15+': 15, + 'R': 18, + 'X': 18, + } def _real_extract(self, url): content_id = self._match_id(url) + data = self._download_json( + 'https://10play.com.au/api/v1/videos/' + content_id, content_id) - if content_id is None or content_id == 'None': - return self.extract_playlist(url) - - video_info = self._download_json( - 'https://10play.com.au/api/v1/videos/' + content_id, content_id, note='Fetching video info') - - playback_url = video_info['playbackApiEndpoint'] - - headers = {} - - # Handle member-gated videos - if video_info.get('memberGated'): - access_token = self.get_access_token(content_id) - headers.update(Authorization=f"Bearer {access_token}") - - playback_info = self._download_json( - playback_url, content_id, - note='Fetching playback info', - headers=headers) - - formats = self._extract_m3u8_formats(playback_info['source'], content_id, 'mp4') + video_data = self._download_json( + 'https://vod.ten.com.au/api/videos/bcquery?command=find_videos_by_id&video_id={0}'.format(data['altId']), + content_id, 'Downloading video JSON') + m3u8_url = self._request_webpage( + HEADRequest(traverse_obj(video_data, ('items', 0, 'HLSURL', T(url_or_none)))), + content_id, 'Checking stream URL').url + if '10play-not-in-oz' in m3u8_url: + self.raise_geo_restricted() + # Attempt to get a higher quality stream + m3u8_url = m3u8_url.replace(',150,75,55,0000', ',300,150,75,55,0000') + formats = self._extract_m3u8_formats(m3u8_url, content_id, 'mp4') self._sort_formats(formats) - return { - 'formats': formats, + return merge_dicts({ 'id': content_id, - 'title': unescapeHTML(video_info.get('subtitle')) or unescapeHTML(video_info.get('title')), - 'description': unescapeHTML(video_info.get('description')), - 'age_limit': parse_age_limit(video_info.get('classification')), - 'series': video_info.get('tvShow'), - 'season': video_info.get('season'), - 'season_number': str_to_int(video_info.get('season')), - 'episode': unescapeHTML(video_info.get('title')), - 'episode_number': str_to_int(video_info.get('episode')), - 'timestamp': video_info.get('published'), - 'duration': video_info.get('duration'), - 'thumbnail': video_info.get('posterImageUrl'), - } + 'formats': formats, + 'uploader': 'Channel 10', + 'uploader_id': '2199827728001', + }, traverse_obj(data, { + 'subtitles': ('captionUrl', T(lambda x: None if x is None + else {'en': [{'url': x}]})), + 'id': ('altId', T(txt_or_none)), + 'duration': ('duration', T(float_or_none)), + 'title': ('subtitle', T(txt_or_none)), + 'alt_title': ('title', T(txt_or_none)), + 'description': ('description', T(txt_or_none)), + 'age_limit': ('classification', T(self._AUS_AGES.get)), + 'series': ('tvShow', T(txt_or_none)), + 'season_number': ('season', T(int_or_none)), + 'episode_number': ('episode', T(int_or_none)), + 'timestamp': ('published', T(int_or_none)), + 'thumbnail': ('imageUrl', T(url_or_none)), + }), rev=True) + + +class TenPlaySeasonIE(TenPlayBase): + _VALID_URL = r'https?://(?:www\.)?10play\.com\.au/(?P[^/?#]+)/episodes/(?P[^/?#]+)/?(?:$|[?#])' + _TESTS = [{ + 'url': 'https://10play.com.au/masterchef/episodes/season-14', + 'info_dict': { + 'title': 'Season 14', + 'id': 'MjMyOTIy', + }, + 'playlist_mincount': 64, + }, { + 'url': 'https://10play.com.au/the-bold-and-the-beautiful-fast-tracked/episodes/season-2022', + 'info_dict': { + 'title': 'Season 2022', + 'id': 'Mjc0OTIw', + }, + 'playlist_mincount': 256, + }] + + def _entries(self, load_more_url, display_id=None): + skip_ids = [] + for page in compat_itertools_count(1): + episodes_carousel = self._download_json( + load_more_url, display_id, query={'skipIds[]': skip_ids}, + note='Fetching episodes page {0}'.format(page)) + + episodes_chunk = episodes_carousel['items'] + skip_ids.extend(ep['id'] for ep in episodes_chunk) + + for ep in episodes_chunk: + yield ep['cardLink'] + if not episodes_carousel.get('hasMore'): + break + + def _real_extract(self, url): + show, season = self._match_valid_url(url).group('show', 'season') + season_info = self._download_json(*(s.format(show=show, season=season) for s in ( + 'https://10play.com.au/api/shows/{show}/episodes/{season}', '{show}/{season}') + )) + + episodes_carousel = traverse_obj(season_info, ( + 'content', 0, 'components', ( + lambda _, v: v['title'].lower() == 'episodes', + (Ellipsis, T(dict)), + )), any) or {} + + playlist_id = episodes_carousel['tpId'] + + return self.playlist_from_matches( + self._entries(urljoin(url, episodes_carousel['loadMoreUrl']), playlist_id), + playlist_id, traverse_obj(season_info, ('content', 0, 'title', T(txt_or_none))), + getter=functools.partial(urljoin, url)) From 3ae35e29335726f89eaf56c12fad2ba26127f9af Mon Sep 17 00:00:00 2001 From: dirkf Date: Wed, 11 Sep 2024 17:38:07 +0100 Subject: [PATCH 4/6] Support TenPlaySeasonIE --- youtube_dl/extractor/extractors.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py index 3a87f9e33..44cad7868 100644 --- a/youtube_dl/extractor/extractors.py +++ b/youtube_dl/extractor/extractors.py @@ -1260,7 +1260,10 @@ from .telequebec import ( from .teletask import TeleTaskIE from .telewebion import TelewebionIE from .tennistv import TennisTVIE -from .tenplay import TenPlayIE +from .tenplay import ( + TenPlayIE, + TenPlaySeasonIE, +) from .testurl import TestURLIE from .tf1 import TF1IE from .tfo import TFOIE From 3d122d40c179a0f9a526a000e91455d004dabb4c Mon Sep 17 00:00:00 2001 From: dirkf Date: Wed, 11 Sep 2024 18:06:00 +0100 Subject: [PATCH 5/6] Disable TenPlaySeasonIE tests by default --- youtube_dl/extractor/tenplay.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/youtube_dl/extractor/tenplay.py b/youtube_dl/extractor/tenplay.py index 197c45a3a..4f60d0225 100644 --- a/youtube_dl/extractor/tenplay.py +++ b/youtube_dl/extractor/tenplay.py @@ -157,6 +157,7 @@ class TenPlaySeasonIE(TenPlayBase): 'id': 'MjMyOTIy', }, 'playlist_mincount': 64, + 'skip': 'Only available in Australia', }, { 'url': 'https://10play.com.au/the-bold-and-the-beautiful-fast-tracked/episodes/season-2022', 'info_dict': { @@ -164,6 +165,7 @@ class TenPlaySeasonIE(TenPlayBase): 'id': 'Mjc0OTIw', }, 'playlist_mincount': 256, + 'skip': 'Only available in Australia', }] def _entries(self, load_more_url, display_id=None): From 7ac17d4b4469d23e575c3483695a790a309b2ff8 Mon Sep 17 00:00:00 2001 From: dirkf Date: Sat, 14 Sep 2024 23:18:32 +0100 Subject: [PATCH 6/6] Match merged yt-dlp extractor --- youtube_dl/extractor/tenplay.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/tenplay.py b/youtube_dl/extractor/tenplay.py index 4f60d0225..11c3d72ab 100644 --- a/youtube_dl/extractor/tenplay.py +++ b/youtube_dl/extractor/tenplay.py @@ -117,7 +117,7 @@ class TenPlayIE(TenPlayBase): 'https://vod.ten.com.au/api/videos/bcquery?command=find_videos_by_id&video_id={0}'.format(data['altId']), content_id, 'Downloading video JSON') m3u8_url = self._request_webpage( - HEADRequest(traverse_obj(video_data, ('items', 0, 'HLSURL', T(url_or_none)))), + HEADRequest(video_data['items'][0]['HLSURL']), content_id, 'Checking stream URL').url if '10play-not-in-oz' in m3u8_url: self.raise_geo_restricted()