lopar/youtube-dl
1
0
Fork 0
mirror of https://github.com/ytdl-org/youtube-dl.git synced 2025-02-07 02:29:48 +00:00
Code Releases Activity

Add further improvements and metadata extraction for SIGI-type pages

Browse Source 
See https://github.com/ytdl-org/youtube-dl/issues/30251#issuecomment-976947005
This commit is contained in:
dirkf 2022-01-07 01:41:11 +00:00
parent b4eb012d34
commit 37f01573a6
1 changed files with 37 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

68
youtube_dl/extractor/tiktok.py
View File

@ -15,10 +15,11 @@ from ..utils import (
class TikTokBaseIE(InfoExtractor): class TikTokBaseIE(InfoExtractor):
def _extract_video(self, data, video_id=None): def _extract_video(self, data, video_id=None):
video = data['video'] video = try_get(data, lambda x: x['video'], dict)
description = str_or_none(try_get(data, lambda x: x['desc'])) if not video:
width = int_or_none(try_get(data, lambda x: video['width'])) return
height = int_or_none(try_get(data, lambda x: video['height'])) width = int_or_none(video.get('width'))
height = int_or_none(video.get('height'))
format_urls = set() format_urls = set()
formats = [] formats = []
@ -43,30 +44,32 @@ class TikTokBaseIE(InfoExtractor):
thumbnail = url_or_none(video.get('cover')) thumbnail = url_or_none(video.get('cover'))
duration = float_or_none(video.get('duration')) duration = float_or_none(video.get('duration'))
uploader = try_get(data, lambda x: x['author']['nickname'], compat_str) author = data.get('author')
uploader_id = try_get(data, lambda x: x['author']['id'], compat_str) if isinstance(author, dict):
uploader_id = author.get('id')
else:
uploader_id = data.get('authorId')
author = data
uploader = str_or_none(author.get('nickname'))
timestamp = int_or_none(data.get('createTime')) timestamp = int_or_none(data.get('createTime'))
def stats(key): stats = try_get(data, lambda x: x['stats'], dict)
return int_or_none(try_get( view_count, like_count, comment_count, repost_count = [
data, lambda x: x['stats']['%sCount' % key])) stats and int_or_none(stats.get('%sCount' % key))
for key in ('play', 'digg', 'comment', 'share', )]
view_count = stats('play')
like_count = stats('digg')
comment_count = stats('comment')
repost_count = stats('share')
aweme_id = data.get('id') or video_id aweme_id = data.get('id') or video_id
return { return {
'id': aweme_id, 'id': aweme_id,
'display_id': video_id,
'title': uploader or aweme_id, 'title': uploader or aweme_id,
'description': description, 'description': str_or_none(data.get('desc')),
'thumbnail': thumbnail, 'thumbnail': thumbnail,
'duration': duration, 'duration': duration,
'uploader': uploader, 'uploader': uploader,
'uploader_id': uploader_id, 'uploader_id': str_or_none(uploader_id),
'timestamp': timestamp, 'timestamp': timestamp,
'view_count': view_count, 'view_count': view_count,
'like_count': like_count, 'like_count': like_count,
@ -84,11 +87,11 @@ class TikTokIE(TikTokBaseIE):
'info_dict': { 'info_dict': {
'id': '6606727368545406213', 'id': '6606727368545406213',
'ext': 'mp4', 'ext': 'mp4',
'title': 'Zureeal', 'title': 'md5:24acc456b62b938a7e2dd88e978b20d9',
'description': '#bowsette#mario#cosplay#uk#lgbt#gaming#asian#bowsettecosplay', 'description': '#bowsette#mario#cosplay#uk#lgbt#gaming#asian#bowsettecosplay',
'thumbnail': r're:^https?://.*', 'thumbnail': r're:^https?://.*',
'duration': 15, 'duration': 15,
'uploader': 'Zureeal', 'uploader': 'md5:24acc456b62b938a7e2dd88e978b20d9',
'uploader_id': '188294915489964032', 'uploader_id': '188294915489964032',
'timestamp': 1538248586, 'timestamp': 1538248586,
'upload_date': '20180929', 'upload_date': '20180929',
@ -98,7 +101,6 @@ class TikTokIE(TikTokBaseIE):
'repost_count': int, 'repost_count': int,
} }
}] }]
def _real_initialize(self): def _real_initialize(self):
# Setup session (will set necessary cookies) # Setup session (will set necessary cookies)
self._request_webpage( self._request_webpage(
@ -106,19 +108,23 @@ class TikTokIE(TikTokBaseIE):
def _real_extract(self, url): def _real_extract(self, url):
video_id = self._match_id(url) video_id = self._match_id(url)
webpage = self._download_webpage(url, video_id) webpage = self._download_webpage(url, video_id)
try:
page_props = self._parse_json(self._search_regex( page_props = self._parse_json(self._search_regex(
r'<script[^>]+\bid=["\']__NEXT_DATA__[^>]+>\s*({.+?})\s*</script', r'''(?s)<script\s[^>]*?\bid\s*=\s*(?P<q>"|'|\b)sigi-persisted-data(?P=q)[^>]*>[^=]*=\s*(?P<json>{.+?})\s*(?:;[^<]+)?</script''',
webpage, 'data'), video_id)['props']['pageProps'] webpage, 'sigi data', default='{}', group='json'), video_id)
data = try_get(page_props, lambda x: x['itemInfo']['itemStruct'], dict) data = try_get(page_props, lambda x: x['ItemModule'][video_id]['video'], dict)
except: if data:
page_props = self._parse_json(self._search_regex( data = page_props['ItemModule'][video_id]
r'<script[^>]+\bid=["\']sigi-persisted-data[^>]+>window\[\'SIGI_STATE\']=({.+?});window\[', if data.get('privateItem'):
webpage, 'data'), video_id) raise ExtractorError('This video is private', expected=True)
data = try_get(page_props, lambda x: x['ItemModule'][video_id], dict) return self._extract_video(data, video_id)
author = try_get(page_props, lambda x: x['UserModule']['users'][data['author']], dict)
data['author'] = author page_props = self._parse_json(self._search_regex(
r'<script[^>]+\bid=["\']__NEXT_DATA__[^>]+>\s*({.+?})\s*</script',
webpage, 'data'), video_id)['props']['pageProps']
data = try_get(page_props, lambda x: x['itemInfo']['itemStruct'], dict)
if not data and page_props.get('statusCode') == 10216: if not data and page_props.get('statusCode') == 10216:
raise ExtractorError('This video is private', expected=True) raise ExtractorError('This video is private', expected=True)
return self._extract_video(data, video_id) return self._extract_video(data, video_id)