Compare commits
29 Commits
7f0eb28ee9
...
49efb69e64
Author | SHA1 | Date |
---|---|---|
pgaig | 49efb69e64 | |
dirkf | 668332b973 | |
dirkf | 0b2ce3685e | |
dirkf | c2766cb80e | |
dirkf | eb38665438 | |
dirkf | e0727e4ab6 | |
Ori Avtalion | 4ea59c6107 | |
dirkf | 21792b88b7 | |
dirkf | d8f134a664 | |
dirkf | 31a15a7c8d | |
dirkf | 19dc10b986 | |
dirkf | 182f63e82a | |
gy-chen | 71211e7db7 | |
Zizheng Guo | a96a45b2cd | |
hatsomatt | 820fae3b3a | |
dirkf | aef24d97e9 | |
dirkf | f7b30e3f73 | |
dirkf | f66372403f | |
dirkf | 7216fa2ac4 | |
dirkf | acc383b9e3 | |
Hubert Hirtz | f0812d7848 | |
Aaron Tan | 40bd5c1815 | |
dirkf | 70f230f9cf | |
dirkf | 48ddab1f3a | |
dirkf | 7687389f08 | |
pgaig | e0556cd956 | |
pgaig | 944a8f2b0b | |
pgaig | 03d89133f3 | |
pgaig | 17dca698cf |
|
@ -10,6 +10,7 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
|||
|
||||
from test.helper import (
|
||||
assertGreaterEqual,
|
||||
assertLessEqual,
|
||||
expect_warnings,
|
||||
get_params,
|
||||
gettestcases,
|
||||
|
@ -122,7 +123,10 @@ def generator(test_case, tname):
|
|||
params['outtmpl'] = tname + '_' + params['outtmpl']
|
||||
if is_playlist and 'playlist' not in test_case:
|
||||
params.setdefault('extract_flat', 'in_playlist')
|
||||
params.setdefault('playlistend', test_case.get('playlist_mincount'))
|
||||
params.setdefault('playlistend',
|
||||
test_case['playlist_maxcount'] + 1
|
||||
if test_case.get('playlist_maxcount')
|
||||
else test_case.get('playlist_mincount'))
|
||||
params.setdefault('skip_download', True)
|
||||
|
||||
ydl = YoutubeDL(params, auto_init=False)
|
||||
|
@ -190,6 +194,14 @@ def generator(test_case, tname):
|
|||
'Expected at least %d in playlist %s, but got only %d' % (
|
||||
test_case['playlist_mincount'], test_case['url'],
|
||||
len(res_dict['entries'])))
|
||||
if 'playlist_maxcount' in test_case:
|
||||
assertLessEqual(
|
||||
self,
|
||||
len(res_dict['entries']),
|
||||
test_case['playlist_maxcount'],
|
||||
'Expected at most %d in playlist %s, but got %d' % (
|
||||
test_case['playlist_maxcount'], test_case['url'],
|
||||
len(res_dict['entries'])))
|
||||
if 'playlist_count' in test_case:
|
||||
self.assertEqual(
|
||||
len(res_dict['entries']),
|
||||
|
|
|
@ -18,6 +18,7 @@ from test.helper import (
|
|||
)
|
||||
from youtube_dl import YoutubeDL
|
||||
from youtube_dl.compat import (
|
||||
compat_contextlib_suppress,
|
||||
compat_http_cookiejar_Cookie,
|
||||
compat_http_server,
|
||||
compat_kwargs,
|
||||
|
@ -35,6 +36,9 @@ from youtube_dl.downloader.external import (
|
|||
HttpieFD,
|
||||
WgetFD,
|
||||
)
|
||||
from youtube_dl.postprocessor import (
|
||||
FFmpegPostProcessor,
|
||||
)
|
||||
import threading
|
||||
|
||||
TEST_SIZE = 10 * 1024
|
||||
|
@ -227,7 +231,17 @@ class TestAria2cFD(unittest.TestCase):
|
|||
self.assertIn('--load-cookies=%s' % downloader._cookies_tempfile, cmd)
|
||||
|
||||
|
||||
@ifExternalFDAvailable(FFmpegFD)
|
||||
# Handle delegated availability
|
||||
def ifFFmpegFDAvailable(externalFD):
|
||||
# raise SkipTest, or set False!
|
||||
avail = ifExternalFDAvailable(externalFD) and False
|
||||
with compat_contextlib_suppress(Exception):
|
||||
avail = FFmpegPostProcessor(downloader=None).available
|
||||
return unittest.skipUnless(
|
||||
avail, externalFD.get_basename() + ' not found')
|
||||
|
||||
|
||||
@ifFFmpegFDAvailable(FFmpegFD)
|
||||
class TestFFmpegFD(unittest.TestCase):
|
||||
_args = []
|
||||
|
||||
|
|
|
@ -81,6 +81,7 @@ from youtube_dl.utils import (
|
|||
sanitize_filename,
|
||||
sanitize_path,
|
||||
sanitize_url,
|
||||
sanitized_Request,
|
||||
shell_quote,
|
||||
smuggle_url,
|
||||
str_or_none,
|
||||
|
@ -255,6 +256,18 @@ class TestUtil(unittest.TestCase):
|
|||
self.assertEqual(sanitize_url('https://foo.bar'), 'https://foo.bar')
|
||||
self.assertEqual(sanitize_url('foo bar'), 'foo bar')
|
||||
|
||||
def test_sanitized_Request(self):
|
||||
self.assertFalse(sanitized_Request('http://foo.bar').has_header('Authorization'))
|
||||
self.assertFalse(sanitized_Request('http://:foo.bar').has_header('Authorization'))
|
||||
self.assertEqual(sanitized_Request('http://@foo.bar').get_header('Authorization'),
|
||||
'Basic Og==')
|
||||
self.assertEqual(sanitized_Request('http://:pass@foo.bar').get_header('Authorization'),
|
||||
'Basic OnBhc3M=')
|
||||
self.assertEqual(sanitized_Request('http://user:@foo.bar').get_header('Authorization'),
|
||||
'Basic dXNlcjo=')
|
||||
self.assertEqual(sanitized_Request('http://user:pass@foo.bar').get_header('Authorization'),
|
||||
'Basic dXNlcjpwYXNz')
|
||||
|
||||
def test_expand_path(self):
|
||||
def env(var):
|
||||
return '%{0}%'.format(var) if sys.platform == 'win32' else '${0}'.format(var)
|
||||
|
@ -512,11 +525,14 @@ class TestUtil(unittest.TestCase):
|
|||
self.assertEqual(float_or_none(set()), None)
|
||||
|
||||
def test_int_or_none(self):
|
||||
self.assertEqual(int_or_none(42), 42)
|
||||
self.assertEqual(int_or_none('42'), 42)
|
||||
self.assertEqual(int_or_none(''), None)
|
||||
self.assertEqual(int_or_none(None), None)
|
||||
self.assertEqual(int_or_none([]), None)
|
||||
self.assertEqual(int_or_none(set()), None)
|
||||
self.assertEqual(int_or_none('42', base=8), 34)
|
||||
self.assertRaises(TypeError, int_or_none(42, base=8))
|
||||
|
||||
def test_str_to_int(self):
|
||||
self.assertEqual(str_to_int('123,456'), 123456)
|
||||
|
|
|
@ -2421,29 +2421,26 @@ except ImportError: # Python 2
|
|||
compat_urllib_request_urlretrieve = compat_urlretrieve
|
||||
|
||||
try:
|
||||
from HTMLParser import (
|
||||
HTMLParser as compat_HTMLParser,
|
||||
HTMLParseError as compat_HTMLParseError)
|
||||
except ImportError: # Python 3
|
||||
from html.parser import HTMLParser as compat_HTMLParser
|
||||
except ImportError: # Python 2
|
||||
from HTMLParser import HTMLParser as compat_HTMLParser
|
||||
compat_html_parser_HTMLParser = compat_HTMLParser
|
||||
|
||||
try: # Python 2
|
||||
from HTMLParser import HTMLParseError as compat_HTMLParseError
|
||||
except ImportError: # Python <3.4
|
||||
try:
|
||||
from html.parser import HTMLParseError as compat_HTMLParseError
|
||||
except ImportError: # Python >3.4
|
||||
|
||||
# HTMLParseError has been deprecated in Python 3.3 and removed in
|
||||
# HTMLParseError was deprecated in Python 3.3 and removed in
|
||||
# Python 3.5. Introducing dummy exception for Python >3.5 for compatible
|
||||
# and uniform cross-version exception handling
|
||||
class compat_HTMLParseError(Exception):
|
||||
pass
|
||||
compat_html_parser_HTMLParser = compat_HTMLParser
|
||||
compat_html_parser_HTMLParseError = compat_HTMLParseError
|
||||
|
||||
try:
|
||||
from subprocess import DEVNULL
|
||||
compat_subprocess_get_DEVNULL = lambda: DEVNULL
|
||||
except ImportError:
|
||||
_DEVNULL = subprocess.DEVNULL
|
||||
compat_subprocess_get_DEVNULL = lambda: _DEVNULL
|
||||
except AttributeError:
|
||||
compat_subprocess_get_DEVNULL = lambda: open(os.path.devnull, 'w')
|
||||
|
||||
try:
|
||||
|
@ -2943,6 +2940,51 @@ else:
|
|||
compat_socket_create_connection = socket.create_connection
|
||||
|
||||
|
||||
try:
|
||||
from contextlib import suppress as compat_contextlib_suppress
|
||||
except ImportError:
|
||||
class compat_contextlib_suppress(object):
|
||||
_exceptions = None
|
||||
|
||||
def __init__(self, *exceptions):
|
||||
super(compat_contextlib_suppress, self).__init__()
|
||||
# TODO: [Base]ExceptionGroup (3.12+)
|
||||
self._exceptions = exceptions
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
return exc_val is not None and isinstance(exc_val, self._exceptions or tuple())
|
||||
|
||||
|
||||
# subprocess.Popen context manager
|
||||
# avoids leaking handles if .communicate() is not called
|
||||
try:
|
||||
_Popen = subprocess.Popen
|
||||
# check for required context manager attributes
|
||||
_Popen.__enter__ and _Popen.__exit__
|
||||
compat_subprocess_Popen = _Popen
|
||||
except AttributeError:
|
||||
# not a context manager - make one
|
||||
from contextlib import contextmanager
|
||||
|
||||
@contextmanager
|
||||
def compat_subprocess_Popen(*args, **kwargs):
|
||||
popen = None
|
||||
try:
|
||||
popen = _Popen(*args, **kwargs)
|
||||
yield popen
|
||||
finally:
|
||||
if popen:
|
||||
for f in (popen.stdin, popen.stdout, popen.stderr):
|
||||
if f:
|
||||
# repeated .close() is OK, but just in case
|
||||
with compat_contextlib_suppress(EnvironmentError):
|
||||
f.close()
|
||||
popen.wait()
|
||||
|
||||
|
||||
# Fix https://github.com/ytdl-org/youtube-dl/issues/4223
|
||||
# See http://bugs.python.org/issue9161 for what is broken
|
||||
def workaround_optparse_bug9161():
|
||||
|
@ -3263,6 +3305,7 @@ __all__ = [
|
|||
'compat_http_cookiejar_Cookie',
|
||||
'compat_http_cookies',
|
||||
'compat_http_cookies_SimpleCookie',
|
||||
'compat_contextlib_suppress',
|
||||
'compat_ctypes_WINFUNCTYPE',
|
||||
'compat_etree_fromstring',
|
||||
'compat_filter',
|
||||
|
@ -3298,6 +3341,7 @@ __all__ = [
|
|||
'compat_struct_pack',
|
||||
'compat_struct_unpack',
|
||||
'compat_subprocess_get_DEVNULL',
|
||||
'compat_subprocess_Popen',
|
||||
'compat_tokenize_tokenize',
|
||||
'compat_urllib_error',
|
||||
'compat_urllib_parse',
|
||||
|
|
|
@ -11,8 +11,14 @@ from .common import FileDownloader
|
|||
from ..compat import (
|
||||
compat_setenv,
|
||||
compat_str,
|
||||
compat_subprocess_Popen,
|
||||
)
|
||||
from ..postprocessor.ffmpeg import FFmpegPostProcessor, EXT_TO_OUT_FORMATS
|
||||
|
||||
try:
|
||||
from ..postprocessor.ffmpeg import FFmpegPostProcessor, EXT_TO_OUT_FORMATS
|
||||
except ImportError:
|
||||
FFmpegPostProcessor = None
|
||||
|
||||
from ..utils import (
|
||||
cli_option,
|
||||
cli_valueless_option,
|
||||
|
@ -206,7 +212,10 @@ class WgetFD(ExternalFD):
|
|||
retry[1] = '0'
|
||||
cmd += retry
|
||||
cmd += self._option('--bind-address', 'source_address')
|
||||
cmd += self._option('--proxy', 'proxy')
|
||||
proxy = self.params.get('proxy')
|
||||
if proxy:
|
||||
for var in ('http_proxy', 'https_proxy'):
|
||||
cmd += ['--execute', '%s=%s' % (var, proxy)]
|
||||
cmd += self._valueless_option('--no-check-certificate', 'nocheckcertificate')
|
||||
cmd += self._configuration_args()
|
||||
cmd += ['--', info_dict['url']]
|
||||
|
@ -358,13 +367,14 @@ class FFmpegFD(ExternalFD):
|
|||
|
||||
@classmethod
|
||||
def available(cls):
|
||||
return FFmpegPostProcessor().available
|
||||
# actual availability can only be confirmed for an instance
|
||||
return bool(FFmpegPostProcessor)
|
||||
|
||||
def _call_downloader(self, tmpfilename, info_dict):
|
||||
url = info_dict['url']
|
||||
ffpp = FFmpegPostProcessor(downloader=self)
|
||||
# `downloader` means the parent `YoutubeDL`
|
||||
ffpp = FFmpegPostProcessor(downloader=self.ydl)
|
||||
if not ffpp.available:
|
||||
self.report_error('m3u8 download detected but ffmpeg or avconv could not be found. Please install one.')
|
||||
self.report_error('ffmpeg required for download but no ffmpeg (nor avconv) executable could be found. Please install one.')
|
||||
return False
|
||||
ffpp.check_version()
|
||||
|
||||
|
@ -393,6 +403,7 @@ class FFmpegFD(ExternalFD):
|
|||
# if end_time:
|
||||
# args += ['-t', compat_str(end_time - start_time)]
|
||||
|
||||
url = info_dict['url']
|
||||
cookies = self.ydl.cookiejar.get_cookies_for_url(url)
|
||||
if cookies:
|
||||
args.extend(['-cookies', ''.join(
|
||||
|
@ -480,21 +491,25 @@ class FFmpegFD(ExternalFD):
|
|||
|
||||
self._debug_cmd(args)
|
||||
|
||||
proc = subprocess.Popen(args, stdin=subprocess.PIPE, env=env)
|
||||
try:
|
||||
retval = proc.wait()
|
||||
except BaseException as e:
|
||||
# subprocess.run would send the SIGKILL signal to ffmpeg and the
|
||||
# mp4 file couldn't be played, but if we ask ffmpeg to quit it
|
||||
# produces a file that is playable (this is mostly useful for live
|
||||
# streams). Note that Windows is not affected and produces playable
|
||||
# files (see https://github.com/ytdl-org/youtube-dl/issues/8300).
|
||||
if isinstance(e, KeyboardInterrupt) and sys.platform != 'win32':
|
||||
process_communicate_or_kill(proc, b'q')
|
||||
else:
|
||||
proc.kill()
|
||||
proc.wait()
|
||||
raise
|
||||
# From [1], a PIPE opened in Popen() should be closed, unless
|
||||
# .communicate() is called. Avoid leaking any PIPEs by using Popen
|
||||
# as a context manager (newer Python 3.x and compat)
|
||||
# Fixes "Resource Warning" in test/test_downloader_external.py
|
||||
# [1] https://devpress.csdn.net/python/62fde12d7e66823466192e48.html
|
||||
with compat_subprocess_Popen(args, stdin=subprocess.PIPE, env=env) as proc:
|
||||
try:
|
||||
retval = proc.wait()
|
||||
except BaseException as e:
|
||||
# subprocess.run would send the SIGKILL signal to ffmpeg and the
|
||||
# mp4 file couldn't be played, but if we ask ffmpeg to quit it
|
||||
# produces a file that is playable (this is mostly useful for live
|
||||
# streams). Note that Windows is not affected and produces playable
|
||||
# files (see https://github.com/ytdl-org/youtube-dl/issues/8300).
|
||||
if isinstance(e, KeyboardInterrupt) and sys.platform != 'win32':
|
||||
process_communicate_or_kill(proc, b'q')
|
||||
else:
|
||||
proc.kill()
|
||||
raise
|
||||
return retval
|
||||
|
||||
|
||||
|
|
|
@ -0,0 +1,79 @@
|
|||
# coding: utf-8
|
||||
from __future__ import unicode_literals
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
determine_ext,
|
||||
int_or_none,
|
||||
merge_dicts,
|
||||
parse_iso8601,
|
||||
T,
|
||||
traverse_obj,
|
||||
txt_or_none,
|
||||
urljoin,
|
||||
)
|
||||
|
||||
|
||||
class CaffeineTVIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://(?:www\.)?caffeine\.tv/[^/]+/video/(?P<id>[0-9a-f-]+)'
|
||||
_TESTS = [{
|
||||
'url': 'https://www.caffeine.tv/TsuSurf/video/cffc0a00-e73f-11ec-8080-80017d29f26e',
|
||||
'info_dict': {
|
||||
'id': 'cffc0a00-e73f-11ec-8080-80017d29f26e',
|
||||
'ext': 'mp4',
|
||||
'title': 'GOOOOD MORNINNNNN #highlights',
|
||||
'timestamp': 1654702180,
|
||||
'upload_date': '20220608',
|
||||
'uploader': 'TsuSurf',
|
||||
'duration': 3145,
|
||||
'age_limit': 17,
|
||||
},
|
||||
'params': {
|
||||
'format': 'bestvideo',
|
||||
},
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id = self._match_id(url)
|
||||
json_data = self._download_json(
|
||||
'https://api.caffeine.tv/social/public/activity/' + video_id,
|
||||
video_id)
|
||||
broadcast_info = traverse_obj(json_data, ('broadcast_info', T(dict))) or {}
|
||||
title = broadcast_info['broadcast_title']
|
||||
video_url = broadcast_info['video_url']
|
||||
|
||||
ext = determine_ext(video_url)
|
||||
if ext == 'm3u8':
|
||||
formats = self._extract_m3u8_formats(
|
||||
video_url, video_id, 'mp4', entry_protocol='m3u8',
|
||||
fatal=False)
|
||||
else:
|
||||
formats = [{'url': video_url}]
|
||||
self._sort_formats(formats)
|
||||
|
||||
return merge_dicts({
|
||||
'id': video_id,
|
||||
'title': title,
|
||||
'formats': formats,
|
||||
}, traverse_obj(json_data, {
|
||||
'uploader': ((None, 'user'), 'username'),
|
||||
}, get_all=False), traverse_obj(json_data, {
|
||||
'like_count': ('like_count', T(int_or_none)),
|
||||
'view_count': ('view_count', T(int_or_none)),
|
||||
'comment_count': ('comment_count', T(int_or_none)),
|
||||
'tags': ('tags', Ellipsis, T(txt_or_none)),
|
||||
'is_live': 'is_live',
|
||||
'uploader': ('user', 'name'),
|
||||
}), traverse_obj(broadcast_info, {
|
||||
'duration': ('content_duration', T(int_or_none)),
|
||||
'timestamp': ('broadcast_start_time', T(parse_iso8601)),
|
||||
'thumbnail': ('preview_image_path', T(lambda u: urljoin(url, u))),
|
||||
'age_limit': ('content_rating', T(lambda r: r and {
|
||||
# assume Apple Store ratings [1]
|
||||
# 1. https://en.wikipedia.org/wiki/Mobile_software_content_rating_system
|
||||
'FOUR_PLUS': 0,
|
||||
'NINE_PLUS': 9,
|
||||
'TWELVE_PLUS': 12,
|
||||
'SEVENTEEN_PLUS': 17,
|
||||
}.get(r, 17))),
|
||||
}))
|
|
@ -1,7 +1,6 @@
|
|||
from __future__ import unicode_literals
|
||||
|
||||
import re
|
||||
import json
|
||||
|
||||
from .common import InfoExtractor
|
||||
from .gigya import GigyaBaseIE
|
||||
|
@ -17,6 +16,7 @@ from ..utils import (
|
|||
str_or_none,
|
||||
strip_or_none,
|
||||
url_or_none,
|
||||
urlencode_postdata
|
||||
)
|
||||
|
||||
|
||||
|
@ -259,7 +259,7 @@ class VrtNUIE(GigyaBaseIE):
|
|||
'expected_warnings': ['Unable to download asset JSON', 'is not a supported codec', 'Unknown MIME type'],
|
||||
}]
|
||||
_NETRC_MACHINE = 'vrtnu'
|
||||
_APIKEY = '3_0Z2HujMtiWq_pkAjgnS2Md2E11a1AwZjYiBETtwNE-EoEHDINgtnvcAOpNgmrVGy'
|
||||
_APIKEY = '3_qhEcPa5JGFROVwu5SWKqJ4mVOIkwlFNMSKwzPDAh8QZOtHqu6L4nD5Q7lk0eXOOG'
|
||||
_CONTEXT_ID = 'R3595707040'
|
||||
|
||||
def _real_initialize(self):
|
||||
|
@ -270,35 +270,38 @@ class VrtNUIE(GigyaBaseIE):
|
|||
if username is None:
|
||||
return
|
||||
|
||||
auth_data = {
|
||||
'APIKey': self._APIKEY,
|
||||
'targetEnv': 'jssdk',
|
||||
'loginID': username,
|
||||
'password': password,
|
||||
'authMode': 'cookie',
|
||||
}
|
||||
|
||||
auth_info = self._gigya_login(auth_data)
|
||||
auth_info = self._download_json(
|
||||
'https://accounts.vrt.be/accounts.login', None,
|
||||
note='Login data', errnote='Could not get Login data',
|
||||
headers={}, data=urlencode_postdata({
|
||||
'loginID': username,
|
||||
'password': password,
|
||||
'sessionExpiration': '-2',
|
||||
'APIKey': self._APIKEY,
|
||||
'targetEnv': 'jssdk',
|
||||
}))
|
||||
|
||||
# Sometimes authentication fails for no good reason, retry
|
||||
login_attempt = 1
|
||||
while login_attempt <= 3:
|
||||
try:
|
||||
# When requesting a token, no actual token is returned, but the
|
||||
# necessary cookies are set.
|
||||
self._request_webpage('https://token.vrt.be/vrtnuinitlogin',
|
||||
None, note='Requesting XSRF Token', errnote='Could not get XSRF Token',
|
||||
query={'provider': 'site', 'destination': 'https://www.vrt.be/vrtnu/'})
|
||||
|
||||
post_data = {
|
||||
'UID': auth_info['UID'],
|
||||
'UIDSignature': auth_info['UIDSignature'],
|
||||
'signatureTimestamp': auth_info['signatureTimestamp'],
|
||||
'client_id': 'vrtnu-site',
|
||||
'_csrf': self._get_cookies('https://login.vrt.be/').get('OIDCXSRF').value
|
||||
}
|
||||
|
||||
self._request_webpage(
|
||||
'https://token.vrt.be',
|
||||
'https://login.vrt.be/perform_login',
|
||||
None, note='Requesting a token', errnote='Could not get a token',
|
||||
headers={
|
||||
'Content-Type': 'application/json',
|
||||
'Referer': 'https://www.vrt.be/vrtnu/',
|
||||
},
|
||||
data=json.dumps({
|
||||
'uid': auth_info['UID'],
|
||||
'uidsig': auth_info['UIDSignature'],
|
||||
'ts': auth_info['signatureTimestamp'],
|
||||
'email': auth_info['profile']['email'],
|
||||
}).encode('utf-8'))
|
||||
headers={}, data=urlencode_postdata(post_data))
|
||||
|
||||
except ExtractorError as e:
|
||||
if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:
|
||||
login_attempt += 1
|
||||
|
|
|
@ -25,6 +25,7 @@ from ..compat import (
|
|||
compat_getpass,
|
||||
compat_integer_types,
|
||||
compat_http_client,
|
||||
compat_kwargs,
|
||||
compat_map as map,
|
||||
compat_open as open,
|
||||
compat_os_name,
|
||||
|
@ -1102,6 +1103,60 @@ class InfoExtractor(object):
|
|||
self._downloader.report_warning('unable to extract %s' % _name + bug_reports_message())
|
||||
return None
|
||||
|
||||
def _search_json(self, start_pattern, string, name, video_id, **kwargs):
|
||||
"""Searches string for the JSON object specified by start_pattern"""
|
||||
|
||||
# self, start_pattern, string, name, video_id, *, end_pattern='',
|
||||
# contains_pattern=r'{(?s:.+)}', fatal=True, default=NO_DEFAULT
|
||||
# NB: end_pattern is only used to reduce the size of the initial match
|
||||
end_pattern = kwargs.pop('end_pattern', '')
|
||||
# (?:[\s\S]) simulates (?(s):.) (eg)
|
||||
contains_pattern = kwargs.pop('contains_pattern', r'{[\s\S]+}')
|
||||
fatal = kwargs.pop('fatal', True)
|
||||
default = kwargs.pop('default', NO_DEFAULT)
|
||||
|
||||
if default is NO_DEFAULT:
|
||||
default, has_default = {}, False
|
||||
else:
|
||||
fatal, has_default = False, True
|
||||
|
||||
json_string = self._search_regex(
|
||||
r'(?:{0})\s*(?P<json>{1})\s*(?:{2})'.format(
|
||||
start_pattern, contains_pattern, end_pattern),
|
||||
string, name, group='json', fatal=fatal, default=None if has_default else NO_DEFAULT)
|
||||
if not json_string:
|
||||
return default
|
||||
|
||||
# yt-dlp has a special JSON parser that allows trailing text.
|
||||
# Until that arrives here, the diagnostic from the exception
|
||||
# raised by json.loads() is used to extract the wanted text.
|
||||
# Either way, it's a problem if a transform_source() can't
|
||||
# handle the trailing text.
|
||||
|
||||
# force an exception
|
||||
kwargs['fatal'] = True
|
||||
|
||||
# self._downloader._format_err(name, self._downloader.Styles.EMPHASIS)
|
||||
for _ in range(2):
|
||||
try:
|
||||
# return self._parse_json(json_string, video_id, ignore_extra=True, **kwargs)
|
||||
transform_source = kwargs.pop('transform_source', None)
|
||||
if transform_source:
|
||||
json_string = transform_source(json_string)
|
||||
return self._parse_json(json_string, video_id, **compat_kwargs(kwargs))
|
||||
except ExtractorError as e:
|
||||
end = int_or_none(self._search_regex(r'\(char\s+(\d+)', error_to_compat_str(e), 'end', default=None))
|
||||
if end is not None:
|
||||
json_string = json_string[:end]
|
||||
continue
|
||||
msg = 'Unable to extract {0} - Failed to parse JSON'.format(name)
|
||||
if fatal:
|
||||
raise ExtractorError(msg, cause=e.cause, video_id=video_id)
|
||||
elif not has_default:
|
||||
self.report_warning(
|
||||
'{0}: {1}'.format(msg, error_to_compat_str(e)), video_id=video_id)
|
||||
return default
|
||||
|
||||
def _html_search_regex(self, pattern, string, name, default=NO_DEFAULT, fatal=True, flags=0, group=None):
|
||||
"""
|
||||
Like _search_regex, but strips HTML tags and unescapes entities.
|
||||
|
@ -2966,25 +3021,22 @@ class InfoExtractor(object):
|
|||
return formats
|
||||
|
||||
def _find_jwplayer_data(self, webpage, video_id=None, transform_source=js_to_json):
|
||||
mobj = re.search(
|
||||
r'''(?s)jwplayer\s*\(\s*(?P<q>'|")(?!(?P=q)).+(?P=q)\s*\)(?!</script>).*?\.\s*setup\s*\(\s*(?P<options>(?:\([^)]*\)|[^)])+)\s*\)''',
|
||||
webpage)
|
||||
if mobj:
|
||||
try:
|
||||
jwplayer_data = self._parse_json(mobj.group('options'),
|
||||
video_id=video_id,
|
||||
transform_source=transform_source)
|
||||
except ExtractorError:
|
||||
pass
|
||||
else:
|
||||
if isinstance(jwplayer_data, dict):
|
||||
return jwplayer_data
|
||||
return self._search_json(
|
||||
r'''(?<!-)\bjwplayer\s*\(\s*(?P<q>'|")(?!(?P=q)).+(?P=q)\s*\)(?:(?!</script>).)*?\.\s*(?:setup\s*\(|(?P<load>load)\s*\(\s*\[)''',
|
||||
webpage, 'JWPlayer data', video_id,
|
||||
# must be a {...} or sequence, ending
|
||||
contains_pattern=r'\{[\s\S]*}(?(load)(?:\s*,\s*\{[\s\S]*})*)', end_pattern=r'(?(load)\]|\))',
|
||||
transform_source=transform_source, default=None)
|
||||
|
||||
def _extract_jwplayer_data(self, webpage, video_id, *args, **kwargs):
|
||||
jwplayer_data = self._find_jwplayer_data(
|
||||
webpage, video_id, transform_source=js_to_json)
|
||||
return self._parse_jwplayer_data(
|
||||
jwplayer_data, video_id, *args, **kwargs)
|
||||
|
||||
# allow passing `transform_source` through to _find_jwplayer_data()
|
||||
transform_source = kwargs.pop('transform_source', None)
|
||||
kwfind = compat_kwargs({'transform_source': transform_source}) if transform_source else {}
|
||||
|
||||
jwplayer_data = self._find_jwplayer_data(webpage, video_id, **kwfind)
|
||||
|
||||
return self._parse_jwplayer_data(jwplayer_data, video_id, *args, **kwargs)
|
||||
|
||||
def _parse_jwplayer_data(self, jwplayer_data, video_id=None, require_title=True,
|
||||
m3u8_id=None, mpd_id=None, rtmp_params=None, base_url=None):
|
||||
|
@ -3018,22 +3070,14 @@ class InfoExtractor(object):
|
|||
mpd_id=mpd_id, rtmp_params=rtmp_params, base_url=base_url)
|
||||
|
||||
subtitles = {}
|
||||
tracks = video_data.get('tracks')
|
||||
if tracks and isinstance(tracks, list):
|
||||
for track in tracks:
|
||||
if not isinstance(track, dict):
|
||||
continue
|
||||
track_kind = track.get('kind')
|
||||
if not track_kind or not isinstance(track_kind, compat_str):
|
||||
continue
|
||||
if track_kind.lower() not in ('captions', 'subtitles'):
|
||||
continue
|
||||
track_url = urljoin(base_url, track.get('file'))
|
||||
if not track_url:
|
||||
continue
|
||||
subtitles.setdefault(track.get('label') or 'en', []).append({
|
||||
'url': self._proto_relative_url(track_url)
|
||||
})
|
||||
for track in traverse_obj(video_data, (
|
||||
'tracks', lambda _, t: t.get('kind').lower() in ('captions', 'subtitles'))):
|
||||
track_url = urljoin(base_url, track.get('file'))
|
||||
if not track_url:
|
||||
continue
|
||||
subtitles.setdefault(track.get('label') or 'en', []).append({
|
||||
'url': self._proto_relative_url(track_url)
|
||||
})
|
||||
|
||||
entry = {
|
||||
'id': this_video_id,
|
||||
|
|
|
@ -159,6 +159,7 @@ from .businessinsider import BusinessInsiderIE
|
|||
from .buzzfeed import BuzzFeedIE
|
||||
from .byutv import BYUtvIE
|
||||
from .c56 import C56IE
|
||||
from .caffeine import CaffeineTVIE
|
||||
from .callin import CallinIE
|
||||
from .camdemy import (
|
||||
CamdemyIE,
|
||||
|
@ -382,7 +383,6 @@ from .fc2 import (
|
|||
FC2EmbedIE,
|
||||
)
|
||||
from .fczenit import FczenitIE
|
||||
from .filemoon import FileMoonIE
|
||||
from .fifa import FifaIE
|
||||
from .filmon import (
|
||||
FilmOnIE,
|
||||
|
@ -443,6 +443,7 @@ from .gamespot import GameSpotIE
|
|||
from .gamestar import GameStarIE
|
||||
from .gaskrank import GaskrankIE
|
||||
from .gazeta import GazetaIE
|
||||
from .gbnews import GBNewsIE
|
||||
from .gdcvault import GDCVaultIE
|
||||
from .gedidigital import GediDigitalIE
|
||||
from .generic import GenericIE
|
||||
|
@ -1652,7 +1653,15 @@ from .younow import (
|
|||
YouNowChannelIE,
|
||||
YouNowMomentIE,
|
||||
)
|
||||
from .youporn import YouPornIE
|
||||
from .youporn import (
|
||||
YouPornIE,
|
||||
YouPornCategoryIE,
|
||||
YouPornChannelIE,
|
||||
YouPornCollectionIE,
|
||||
YouPornStarIE,
|
||||
YouPornTagIE,
|
||||
YouPornVideosIE,
|
||||
)
|
||||
from .yourporn import YourPornIE
|
||||
from .yourupload import YourUploadIE
|
||||
from .youtube import (
|
||||
|
|
|
@ -1,43 +0,0 @@
|
|||
# coding: utf-8
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import re
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
decode_packed_codes,
|
||||
js_to_json,
|
||||
)
|
||||
|
||||
|
||||
class FileMoonIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://(?:www\.)?filemoon\.sx/./(?P<id>\w+)'
|
||||
_TEST = {
|
||||
'url': 'https://filemoon.sx/e/dw40rxrzruqz',
|
||||
'md5': '5a713742f57ac4aef29b74733e8dda01',
|
||||
'info_dict': {
|
||||
'id': 'dw40rxrzruqz',
|
||||
'title': 'dw40rxrzruqz',
|
||||
'ext': 'mp4'
|
||||
}
|
||||
}
|
||||
|
||||
def _real_extract(self, url):
|
||||
video_id = self._match_id(url)
|
||||
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
matches = re.findall(r'(?s)(eval.*?)</script>', webpage)
|
||||
packed = matches[-1]
|
||||
unpacked = decode_packed_codes(packed)
|
||||
jwplayer_sources = self._parse_json(
|
||||
self._search_regex(
|
||||
r'(?s)player\s*\.\s*setup\s*\(\s*\{\s*sources\s*:\s*(.*?])', unpacked, 'jwplayer sources'),
|
||||
video_id, transform_source=js_to_json)
|
||||
|
||||
formats = self._parse_jwplayer_formats(jwplayer_sources, video_id)
|
||||
|
||||
return {
|
||||
'id': video_id,
|
||||
'title': self._generic_title(url) or video_id,
|
||||
'formats': formats
|
||||
}
|
|
@ -0,0 +1,139 @@
|
|||
# coding: utf-8
|
||||
from __future__ import unicode_literals
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
extract_attributes,
|
||||
ExtractorError,
|
||||
T,
|
||||
traverse_obj,
|
||||
txt_or_none,
|
||||
url_or_none,
|
||||
)
|
||||
|
||||
|
||||
class GBNewsIE(InfoExtractor):
|
||||
IE_DESC = 'GB News clips, features and live stream'
|
||||
|
||||
# \w+ is normally shows or news, but apparently any word redirects to the correct URL
|
||||
_VALID_URL = r'https?://(?:www\.)?gbnews\.(?:uk|com)/(?:\w+/)?(?P<id>[^#?]+)'
|
||||
|
||||
_PLATFORM = 'safari'
|
||||
_SSMP_URL = 'https://mm-v2.simplestream.com/ssmp/api.php'
|
||||
_TESTS = [{
|
||||
'url': 'https://www.gbnews.uk/shows/andrew-neils-message-to-companies-choosing-to-boycott-gb-news/106889',
|
||||
'info_dict': {
|
||||
'id': '106889',
|
||||
'ext': 'mp4',
|
||||
'title': "Andrew Neil's message to companies choosing to boycott GB News",
|
||||
'description': 'md5:b281f5d22fd6d5eda64a4e3ba771b351',
|
||||
},
|
||||
'skip': '404 not found',
|
||||
}, {
|
||||
'url': 'https://www.gbnews.com/news/bbc-claudine-gay-harvard-university-antisemitism-row',
|
||||
'info_dict': {
|
||||
'id': '52264136',
|
||||
'display_id': 'bbc-claudine-gay-harvard-university-antisemitism-row',
|
||||
'ext': 'mp4',
|
||||
'title': 'BBC deletes post after furious backlash over headline downplaying antisemitism',
|
||||
'description': 'The post was criticised by former employers of the broadcaster',
|
||||
},
|
||||
}, {
|
||||
'url': 'https://www.gbnews.uk/watchlive',
|
||||
'info_dict': {
|
||||
'id': '1069',
|
||||
'display_id': 'watchlive',
|
||||
'ext': 'mp4',
|
||||
'title': 'GB News Live',
|
||||
'is_live': True,
|
||||
},
|
||||
'params': {
|
||||
'skip_download': 'm3u8',
|
||||
},
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
display_id = self._match_id(url).split('/')[-1]
|
||||
|
||||
webpage = self._download_webpage(url, display_id)
|
||||
# extraction based on https://github.com/ytdl-org/youtube-dl/issues/29341
|
||||
'''
|
||||
<div id="video-106908"
|
||||
class="simplestream"
|
||||
data-id="GB001"
|
||||
data-type="vod"
|
||||
data-key="3Li3Nt2Qs8Ct3Xq9Fi5Uy0Mb2Bj0Qs"
|
||||
data-token="f9c317c727dc07f515b20036c8ef14a6"
|
||||
data-expiry="1624300052"
|
||||
data-uvid="37900558"
|
||||
data-poster="https://thumbnails.simplestreamcdn.com/gbnews/ondemand/37900558.jpg?width=700&"
|
||||
data-npaw="false"
|
||||
data-env="production">
|
||||
'''
|
||||
# exception if no match
|
||||
video_data = self._search_regex(
|
||||
r'(<div\s[^>]*\bclass\s*=\s*(\'|")(?!.*sidebar\b)simplestream(?:\s[\s\w$-]*)?\2[^>]*>)',
|
||||
webpage, 'video data')
|
||||
|
||||
video_data = extract_attributes(video_data)
|
||||
ss_id = video_data.get('data-id')
|
||||
if not ss_id:
|
||||
raise ExtractorError('Simplestream ID not found')
|
||||
|
||||
json_data = self._download_json(
|
||||
self._SSMP_URL, display_id,
|
||||
note='Downloading Simplestream JSON metadata',
|
||||
errnote='Unable to download Simplestream JSON metadata',
|
||||
query={
|
||||
'id': ss_id,
|
||||
'env': video_data.get('data-env', 'production'),
|
||||
}, fatal=False)
|
||||
|
||||
meta_url = traverse_obj(json_data, ('response', 'api_hostname'))
|
||||
if not meta_url:
|
||||
raise ExtractorError('No API host found')
|
||||
|
||||
uvid = video_data['data-uvid']
|
||||
dtype = video_data.get('data-type')
|
||||
stream_data = self._download_json(
|
||||
'%s/api/%s/stream/%s' % (meta_url, 'show' if dtype == 'vod' else dtype, uvid),
|
||||
uvid,
|
||||
query={
|
||||
'key': video_data.get('data-key'),
|
||||
'platform': self._PLATFORM,
|
||||
},
|
||||
headers={
|
||||
'Token': video_data.get('data-token'),
|
||||
'Token-Expiry': video_data.get('data-expiry'),
|
||||
'Uvid': uvid,
|
||||
}, fatal=False)
|
||||
|
||||
stream_url = traverse_obj(stream_data, (
|
||||
'response', 'stream', T(url_or_none)))
|
||||
if not stream_url:
|
||||
raise ExtractorError('No stream data/URL')
|
||||
|
||||
# now known to be a dict
|
||||
stream_data = stream_data['response']
|
||||
drm = stream_data.get('drm')
|
||||
if drm:
|
||||
self.report_drm(uvid)
|
||||
|
||||
formats = self._extract_m3u8_formats(
|
||||
stream_url, uvid, ext='mp4', entry_protocol='m3u8_native',
|
||||
fatal=False)
|
||||
# exception if no formats
|
||||
self._sort_formats(formats)
|
||||
|
||||
return {
|
||||
'id': uvid,
|
||||
'display_id': display_id,
|
||||
'title': (traverse_obj(stream_data, ('title', T(txt_or_none)))
|
||||
or self._og_search_title(webpage, default=None)
|
||||
or display_id.replace('-', ' ').capitalize()),
|
||||
'description': self._og_search_description(webpage, default=None),
|
||||
'thumbnail': (traverse_obj(video_data, ('data-poster', T(url_or_none)))
|
||||
or self._og_search_thumbnail(webpage)),
|
||||
'formats': formats,
|
||||
'is_live': (dtype == 'live') or None,
|
||||
}
|
|
@ -7,6 +7,7 @@ import time
|
|||
from .common import InfoExtractor
|
||||
from ..compat import compat_kwargs
|
||||
from ..utils import (
|
||||
base_url,
|
||||
determine_ext,
|
||||
ExtractorError,
|
||||
float_or_none,
|
||||
|
@ -14,6 +15,7 @@ from ..utils import (
|
|||
T,
|
||||
traverse_obj,
|
||||
txt_or_none,
|
||||
url_basename,
|
||||
url_or_none,
|
||||
)
|
||||
|
||||
|
@ -33,8 +35,8 @@ class Vbox7IE(InfoExtractor):
|
|||
'''
|
||||
_EMBED_REGEX = [r'<iframe[^>]+src=(?P<q>["\'])(?P<url>(?:https?:)?//vbox7\.com/emb/external\.php.+?)(?P=q)']
|
||||
_GEO_COUNTRIES = ['BG']
|
||||
_GEO_BYPASS = False
|
||||
_TESTS = [{
|
||||
# the http: URL just redirects here
|
||||
'url': 'https://vbox7.com/play:0946fff23c',
|
||||
'md5': '50ca1f78345a9c15391af47d8062d074',
|
||||
'info_dict': {
|
||||
|
@ -42,17 +44,19 @@ class Vbox7IE(InfoExtractor):
|
|||
'ext': 'mp4',
|
||||
'title': 'Борисов: Притеснен съм за бъдещето на България',
|
||||
'description': 'По думите му е опасно страната ни да бъде обявена за "сигурна"',
|
||||
'thumbnail': r're:^https?://.*\.jpg$',
|
||||
'timestamp': 1470982814,
|
||||
'upload_date': '20160812',
|
||||
'uploader': 'zdraveibulgaria',
|
||||
'thumbnail': r're:^https?://.*\.jpg$',
|
||||
'view_count': int,
|
||||
'duration': 2640,
|
||||
},
|
||||
'expected_warnings': [
|
||||
'Unable to download webpage',
|
||||
],
|
||||
}, {
|
||||
'url': 'http://vbox7.com/play:249bb972c2',
|
||||
'md5': 'aaf19465e37ec0b30b918df83ec32c50',
|
||||
'md5': '99f65c0c9ef9b682b97313e052734c3f',
|
||||
'info_dict': {
|
||||
'id': '249bb972c2',
|
||||
'ext': 'mp4',
|
||||
|
@ -61,7 +65,11 @@ class Vbox7IE(InfoExtractor):
|
|||
'timestamp': 1360215023,
|
||||
'upload_date': '20130207',
|
||||
'uploader': 'svideteliat_ot_varshava',
|
||||
'thumbnail': 'https://i49.vbox7.com/o/249/249bb972c20.jpg',
|
||||
'view_count': int,
|
||||
'duration': 83,
|
||||
},
|
||||
'expected_warnings': ['Failed to download m3u8 information'],
|
||||
}, {
|
||||
'url': 'http://vbox7.com/emb/external.php?vid=a240d20f9c&autoplay=1',
|
||||
'only_matching': True,
|
||||
|
@ -76,6 +84,9 @@ class Vbox7IE(InfoExtractor):
|
|||
if mobj:
|
||||
return mobj.group('url')
|
||||
|
||||
# specialisation to transform what looks like ld+json that
|
||||
# may contain invalid character combinations
|
||||
|
||||
# transform_source=None, fatal=True
|
||||
def _parse_json(self, json_string, video_id, *args, **kwargs):
|
||||
if '"@context"' in json_string[:30]:
|
||||
|
@ -103,49 +114,64 @@ class Vbox7IE(InfoExtractor):
|
|||
|
||||
now = time.time()
|
||||
response = self._download_json(
|
||||
'https://www.vbox7.com/aj/player/item/options?vid=%s' % (video_id,),
|
||||
video_id, headers={'Referer': url})
|
||||
'https://www.vbox7.com/aj/player/item/options', video_id,
|
||||
query={'vid': video_id}, headers={'Referer': url})
|
||||
# estimate time to which possible `ago` member is relative
|
||||
now = now + 0.5 * (time.time() - now)
|
||||
|
||||
if 'error' in response:
|
||||
if traverse_obj(response, 'error'):
|
||||
raise ExtractorError(
|
||||
'%s said: %s' % (self.IE_NAME, response['error']), expected=True)
|
||||
|
||||
video_url = traverse_obj(response, ('options', 'src', T(url_or_none)))
|
||||
src_url = traverse_obj(response, ('options', 'src', T(url_or_none))) or ''
|
||||
|
||||
if '/na.mp4' in video_url or '':
|
||||
fmt_base = url_basename(src_url).rsplit('.', 1)[0].rsplit('_', 1)[0]
|
||||
if fmt_base in ('na', 'vn'):
|
||||
self.raise_geo_restricted(countries=self._GEO_COUNTRIES)
|
||||
|
||||
ext = determine_ext(video_url)
|
||||
ext = determine_ext(src_url)
|
||||
if ext == 'mpd':
|
||||
# In case MPD cannot be parsed, or anyway, get mp4 combined
|
||||
# formats usually provided to Safari, iOS, and old Windows
|
||||
# extract MPD
|
||||
try:
|
||||
formats, subtitles = self._extract_mpd_formats_and_subtitles(
|
||||
video_url, video_id, 'dash', fatal=False)
|
||||
except KeyError:
|
||||
src_url, video_id, 'dash', fatal=False)
|
||||
except KeyError: # fatal doesn't catch this
|
||||
self.report_warning('Failed to parse MPD manifest')
|
||||
formats, subtitles = [], {}
|
||||
elif ext != 'm3u8':
|
||||
formats = [{
|
||||
'url': src_url,
|
||||
}] if src_url else []
|
||||
subtitles = {}
|
||||
|
||||
if src_url:
|
||||
# possibly extract HLS, based on https://github.com/yt-dlp/yt-dlp/pull/9100
|
||||
fmt_base = base_url(src_url) + fmt_base
|
||||
# prepare for _extract_m3u8_formats_and_subtitles()
|
||||
# hls_formats, hls_subs = self._extract_m3u8_formats_and_subtitles(
|
||||
hls_formats = self._extract_m3u8_formats(
|
||||
'{0}.m3u8'.format(fmt_base), video_id, m3u8_id='hls', fatal=False)
|
||||
formats.extend(hls_formats)
|
||||
# self._merge_subtitles(hls_subs, target=subtitles)
|
||||
|
||||
# In case MPD/HLS cannot be parsed, or anyway, get mp4 combined
|
||||
# formats usually provided to Safari, iOS, and old Windows
|
||||
video = response['options']
|
||||
resolutions = (1080, 720, 480, 240, 144)
|
||||
highest_res = traverse_obj(video, ('highestRes', T(int))) or resolutions[0]
|
||||
for res in traverse_obj(video, ('resolutions', lambda _, r: int(r) > 0)) or resolutions:
|
||||
if res > highest_res:
|
||||
continue
|
||||
formats.append({
|
||||
'url': video_url.replace('.mpd', '_%d.mp4' % res),
|
||||
'format_id': '%dp' % res,
|
||||
highest_res = traverse_obj(video, (
|
||||
'highestRes', T(int))) or resolutions[0]
|
||||
resolutions = traverse_obj(video, (
|
||||
'resolutions', lambda _, r: highest_res >= int(r) > 0)) or resolutions
|
||||
mp4_formats = traverse_obj(resolutions, (
|
||||
Ellipsis, T(lambda res: {
|
||||
'url': '{0}_{1}.mp4'.format(fmt_base, res),
|
||||
'format_id': 'http-{0}'.format(res),
|
||||
'height': res,
|
||||
})
|
||||
})))
|
||||
# if above formats are flaky, enable the line below
|
||||
# self._check_formats(formats, video_id)
|
||||
else:
|
||||
formats = [{
|
||||
'url': video_url,
|
||||
}]
|
||||
subtitles = {}
|
||||
# self._check_formats(mp4_formats, video_id)
|
||||
formats.extend(mp4_formats)
|
||||
|
||||
self._sort_formats(formats)
|
||||
|
||||
webpage = self._download_webpage(url, video_id, fatal=False) or ''
|
||||
|
|
|
@ -6,22 +6,31 @@ import re
|
|||
import string
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..compat import (
|
||||
compat_b64decode,
|
||||
compat_ord,
|
||||
compat_struct_pack,
|
||||
)
|
||||
from ..utils import (
|
||||
ExtractorError,
|
||||
int_or_none,
|
||||
mimetype2ext,
|
||||
parse_codecs,
|
||||
parse_qs,
|
||||
update_url_query,
|
||||
urljoin,
|
||||
xpath_element,
|
||||
xpath_text,
|
||||
)
|
||||
from ..compat import (
|
||||
compat_b64decode,
|
||||
compat_ord,
|
||||
compat_struct_pack,
|
||||
compat_urlparse,
|
||||
)
|
||||
|
||||
|
||||
def compat_random_choices(population, *args, **kwargs):
|
||||
# weights=None, *, cum_weights=None, k=1
|
||||
# limited implementation needed here
|
||||
weights = args[0] if args else kwargs.get('weights')
|
||||
assert all(w is None for w in (weights, kwargs.get('cum_weights')))
|
||||
k = kwargs.get('k', 1)
|
||||
return ''.join(random.choice(population) for _ in range(k))
|
||||
|
||||
|
||||
class VideaIE(InfoExtractor):
|
||||
|
@ -35,6 +44,7 @@ class VideaIE(InfoExtractor):
|
|||
)
|
||||
(?P<id>[^?#&]+)
|
||||
'''
|
||||
_EMBED_REGEX = [r'<iframe[^>]+src=(["\'])(?P<url>(?:https?:)?//videa\.hu/player\?.*?\bv=.+?)\1']
|
||||
_TESTS = [{
|
||||
'url': 'http://videa.hu/videok/allatok/az-orult-kigyasz-285-kigyot-kigyo-8YfIAjxwWGwT8HVQ',
|
||||
'md5': '97a7af41faeaffd9f1fc864a7c7e7603',
|
||||
|
@ -44,6 +54,7 @@ class VideaIE(InfoExtractor):
|
|||
'title': 'Az őrült kígyász 285 kígyót enged szabadon',
|
||||
'thumbnail': r're:^https?://.*',
|
||||
'duration': 21,
|
||||
'age_limit': 0,
|
||||
},
|
||||
}, {
|
||||
'url': 'http://videa.hu/videok/origo/jarmuvek/supercars-elozes-jAHDWfWSJH5XuFhH',
|
||||
|
@ -54,6 +65,7 @@ class VideaIE(InfoExtractor):
|
|||
'title': 'Supercars előzés',
|
||||
'thumbnail': r're:^https?://.*',
|
||||
'duration': 64,
|
||||
'age_limit': 0,
|
||||
},
|
||||
}, {
|
||||
'url': 'http://videa.hu/player?v=8YfIAjxwWGwT8HVQ',
|
||||
|
@ -64,6 +76,7 @@ class VideaIE(InfoExtractor):
|
|||
'title': 'Az őrült kígyász 285 kígyót enged szabadon',
|
||||
'thumbnail': r're:^https?://.*',
|
||||
'duration': 21,
|
||||
'age_limit': 0,
|
||||
},
|
||||
}, {
|
||||
'url': 'http://videa.hu/player/v/8YfIAjxwWGwT8HVQ?autoplay=1',
|
||||
|
@ -80,11 +93,14 @@ class VideaIE(InfoExtractor):
|
|||
}]
|
||||
_STATIC_SECRET = 'xHb0ZvME5q8CBcoQi6AngerDu3FGO9fkUlwPmLVY_RTzj2hJIS4NasXWKy1td7p'
|
||||
|
||||
@staticmethod
|
||||
def _extract_urls(webpage):
|
||||
return [url for _, url in re.findall(
|
||||
r'<iframe[^>]+src=(["\'])(?P<url>(?:https?:)?//videa\.hu/player\?.*?\bv=.+?)\1',
|
||||
webpage)]
|
||||
@classmethod
|
||||
def _extract_urls(cls, webpage):
|
||||
def yield_urls():
|
||||
for pattern in cls._EMBED_REGEX:
|
||||
for m in re.finditer(pattern, webpage):
|
||||
yield m.group('url')
|
||||
|
||||
return list(yield_urls())
|
||||
|
||||
@staticmethod
|
||||
def rc4(cipher_text, key):
|
||||
|
@ -130,13 +146,13 @@ class VideaIE(InfoExtractor):
|
|||
for i in range(0, 32):
|
||||
result += s[i - (self._STATIC_SECRET.index(l[i]) - 31)]
|
||||
|
||||
query = compat_urlparse.parse_qs(compat_urlparse.urlparse(player_url).query)
|
||||
random_seed = ''.join(random.choice(string.ascii_letters + string.digits) for _ in range(8))
|
||||
query = parse_qs(player_url)
|
||||
random_seed = ''.join(compat_random_choices(string.ascii_letters + string.digits, k=8))
|
||||
query['_s'] = random_seed
|
||||
query['_t'] = result[:16]
|
||||
|
||||
b64_info, handle = self._download_webpage_handle(
|
||||
'http://videa.hu/videaplayer_get_xml.php', video_id, query=query)
|
||||
'http://videa.hu/player/xml', video_id, query=query)
|
||||
if b64_info.startswith('<?xml'):
|
||||
info = self._parse_xml(b64_info, video_id)
|
||||
else:
|
||||
|
|
|
@ -673,8 +673,8 @@ class VimeoIE(VimeoBaseInfoExtractor):
|
|||
raise
|
||||
|
||||
if '//player.vimeo.com/video/' in url:
|
||||
config = self._parse_json(self._search_regex(
|
||||
r'(?s)\b(?:playerC|c)onfig\s*=\s*({.+?})\s*[;\n]', webpage, 'info section'), video_id)
|
||||
config = self._search_json(
|
||||
r'\b(?:playerC|c)onfig\s*=', webpage, 'info section', video_id)
|
||||
if config.get('view') == 4:
|
||||
config = self._verify_player_video_password(
|
||||
redirect_url, video_id, headers)
|
||||
|
|
|
@ -4,20 +4,28 @@ from __future__ import unicode_literals
|
|||
import re
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..compat import compat_chr
|
||||
from ..compat import (
|
||||
compat_chr,
|
||||
compat_zip as zip,
|
||||
)
|
||||
from ..utils import (
|
||||
clean_html,
|
||||
decode_packed_codes,
|
||||
determine_ext,
|
||||
ExtractorError,
|
||||
get_element_by_id,
|
||||
int_or_none,
|
||||
js_to_json,
|
||||
merge_dicts,
|
||||
T,
|
||||
traverse_obj,
|
||||
url_or_none,
|
||||
urlencode_postdata,
|
||||
)
|
||||
|
||||
|
||||
# based on openload_decode from 2bfeee69b976fe049761dd3012e30b637ee05a58
|
||||
def aa_decode(aa_code):
|
||||
symbol_table = [
|
||||
symbol_table = (
|
||||
('7', '((゚ー゚) + (o^_^o))'),
|
||||
('6', '((o^_^o) +(o^_^o))'),
|
||||
('5', '((゚ー゚) + (゚Θ゚))'),
|
||||
|
@ -26,84 +34,180 @@ def aa_decode(aa_code):
|
|||
('3', '(o^_^o)'),
|
||||
('1', '(゚Θ゚)'),
|
||||
('0', '(c^_^o)'),
|
||||
]
|
||||
('+', ''),
|
||||
)
|
||||
delim = '(゚Д゚)[゚ε゚]+'
|
||||
ret = ''
|
||||
for aa_char in aa_code.split(delim):
|
||||
|
||||
def chr_from_code(c):
|
||||
for val, pat in symbol_table:
|
||||
aa_char = aa_char.replace(pat, val)
|
||||
aa_char = aa_char.replace('+ ', '')
|
||||
m = re.match(r'^\d+', aa_char)
|
||||
if m:
|
||||
ret += compat_chr(int(m.group(0), 8))
|
||||
c = c.replace(pat, val)
|
||||
if c.startswith(('u', 'U')):
|
||||
base = 16
|
||||
c = c[1:]
|
||||
else:
|
||||
m = re.match(r'^u([\da-f]+)', aa_char)
|
||||
if m:
|
||||
ret += compat_chr(int(m.group(1), 16))
|
||||
return ret
|
||||
base = 10
|
||||
c = int_or_none(c, base=base)
|
||||
return '' if c is None else compat_chr(c)
|
||||
|
||||
return ''.join(
|
||||
chr_from_code(aa_char)
|
||||
for aa_char in aa_code.split(delim))
|
||||
|
||||
|
||||
class XFileShareIE(InfoExtractor):
|
||||
_SITES = (
|
||||
(r'aparat\.cam', 'Aparat'),
|
||||
(r'clipwatching\.com', 'ClipWatching'),
|
||||
(r'gounlimited\.to', 'GoUnlimited'),
|
||||
(r'govid\.me', 'GoVid'),
|
||||
(r'holavid\.com', 'HolaVid'),
|
||||
(r'streamty\.com', 'Streamty'),
|
||||
(r'thevideobee\.to', 'TheVideoBee'),
|
||||
(r'uqload\.com', 'Uqload'),
|
||||
(r'vidbom\.com', 'VidBom'),
|
||||
(r'vidlo\.us', 'vidlo'),
|
||||
(r'vidlocker\.xyz', 'VidLocker'),
|
||||
(r'vidshare\.tv', 'VidShare'),
|
||||
(r'vup\.to', 'VUp'),
|
||||
# status check 2024-02: site availability, G site: search
|
||||
(r'aparat\.cam', 'Aparat'), # Cloudflare says host error 522, apparently changed to wolfstreeam.tv
|
||||
(r'filemoon\.sx/.', 'FileMoon'),
|
||||
(r'gounlimited\.to', 'GoUnlimited'), # no media pages listed
|
||||
(r'govid\.me', 'GoVid'), # no media pages listed
|
||||
(r'highstream\.tv', 'HighStream'), # clipwatching.com redirects here
|
||||
(r'holavid\.com', 'HolaVid'), # Cloudflare says host error 522
|
||||
# (r'streamty\.com', 'Streamty'), # no media pages listed, connection timeout
|
||||
# (r'thevideobee\.to', 'TheVideoBee'), # no pages listed, refuses connection
|
||||
(r'uqload\.to', 'Uqload'), # .com, .co redirect here
|
||||
(r'(?:vedbam\.xyz|vadbam.net)', 'V?dB?m'), # vidbom.com redirects here, but no valid media pages listed
|
||||
(r'vidlo\.us', 'vidlo'), # no valid media pages listed
|
||||
(r'vidlocker\.xyz', 'VidLocker'), # no media pages listed
|
||||
(r'(?:w\d\.)?viidshar\.com', 'VidShare'), # vidshare.tv redirects here
|
||||
# (r'vup\.to', 'VUp'), # domain not found
|
||||
(r'wolfstream\.tv', 'WolfStream'),
|
||||
(r'xvideosharing\.com', 'XVideoSharing'),
|
||||
(r'xvideosharing\.com', 'XVideoSharing'), # just started showing 'maintenance mode'
|
||||
)
|
||||
|
||||
IE_DESC = 'XFileShare based sites: %s' % ', '.join(list(zip(*_SITES))[1])
|
||||
IE_DESC = 'XFileShare-based sites: %s' % ', '.join(list(zip(*_SITES))[1])
|
||||
_VALID_URL = (r'https?://(?:www\.)?(?P<host>%s)/(?:embed-)?(?P<id>[0-9a-zA-Z]+)'
|
||||
% '|'.join(site for site in list(zip(*_SITES))[0]))
|
||||
_EMBED_REGEX = [r'<iframe\b[^>]+\bsrc=(["\'])(?P<url>(?:https?:)?//(?:%s)/embed-[0-9a-zA-Z]+.*?)\1' % '|'.join(site for site in list(zip(*_SITES))[0])]
|
||||
|
||||
_FILE_NOT_FOUND_REGEXES = (
|
||||
r'>(?:404 - )?File Not Found<',
|
||||
r'>The file was removed by administrator<',
|
||||
)
|
||||
_TITLE_REGEXES = (
|
||||
r'style="z-index: [0-9]+;">([^<]+)</span>',
|
||||
r'<td nowrap>([^<]+)</td>',
|
||||
r'h4-fine[^>]*>([^<]+)<',
|
||||
r'>Watch (.+)[ <]',
|
||||
r'<h2 class="video-page-head">([^<]+)</h2>',
|
||||
r'<h2 style="[^"]*color:#403f3d[^"]*"[^>]*>([^<]+)<', # streamin.to (dead)
|
||||
r'title\s*:\s*"([^"]+)"', # govid.me
|
||||
)
|
||||
_SOURCE_URL_REGEXES = (
|
||||
r'(?:file|src)\s*:\s*(["\'])(?P<url>http(?:(?!\1).)+\.(?:m3u8|mp4|flv)(?:(?!\1).)*)\1',
|
||||
r'file_link\s*=\s*(["\'])(?P<url>http(?:(?!\1).)+)\1',
|
||||
r'addVariable\((\\?["\'])file\1\s*,\s*(\\?["\'])(?P<url>http(?:(?!\2).)+)\2\)',
|
||||
r'<embed[^>]+src=(["\'])(?P<url>http(?:(?!\1).)+\.(?:m3u8|mp4|flv)(?:(?!\1).)*)\1',
|
||||
)
|
||||
_THUMBNAIL_REGEXES = (
|
||||
r'<video[^>]+poster="([^"]+)"',
|
||||
r'(?:image|poster)\s*:\s*["\'](http[^"\']+)["\'],',
|
||||
)
|
||||
|
||||
_TESTS = [{
|
||||
'url': 'http://xvideosharing.com/fq65f94nd2ve',
|
||||
'md5': '4181f63957e8fe90ac836fa58dc3c8a6',
|
||||
'note': 'link in `sources`',
|
||||
'url': 'https://uqload.to/dcsu06gdb45o',
|
||||
'md5': '7f8db187b254379440bf4fcad094ae86',
|
||||
'info_dict': {
|
||||
'id': 'fq65f94nd2ve',
|
||||
'id': 'dcsu06gdb45o',
|
||||
'ext': 'mp4',
|
||||
'title': 'sample',
|
||||
'thumbnail': r're:http://.*\.jpg',
|
||||
'title': 'f2e31015957e74c8c8427982e161c3fc mp4',
|
||||
'thumbnail': r're:https://.*\.jpg'
|
||||
},
|
||||
'params': {
|
||||
'nocheckcertificate': True,
|
||||
},
|
||||
'expected_warnings': ['Unable to extract JWPlayer data'],
|
||||
}, {
|
||||
'note': 'link in decoded `sources`',
|
||||
'url': 'https://xvideosharing.com/1tlg6agrrdgc',
|
||||
'md5': '2608ce41932c1657ae56258a64e647d9',
|
||||
'info_dict': {
|
||||
'id': '1tlg6agrrdgc',
|
||||
'ext': 'mp4',
|
||||
'title': '0121',
|
||||
'thumbnail': r're:https?://.*\.jpg',
|
||||
},
|
||||
'skip': 'This server is in maintenance mode.',
|
||||
}, {
|
||||
'note': 'JWPlayer link in un-p,a,c,k,e,d JS',
|
||||
'url': 'https://filemoon.sx/e/dw40rxrzruqz',
|
||||
'md5': '5a713742f57ac4aef29b74733e8dda01',
|
||||
'info_dict': {
|
||||
'id': 'dw40rxrzruqz',
|
||||
'title': 'dw40rxrzruqz',
|
||||
'ext': 'mp4'
|
||||
},
|
||||
}, {
|
||||
'note': 'JWPlayer link in un-p,a,c,k,e,d JS',
|
||||
'url': 'https://vadbam.net/6lnbkci96wly.html',
|
||||
'md5': 'a1616800076177e2ac769203957c54bc',
|
||||
'info_dict': {
|
||||
'id': '6lnbkci96wly',
|
||||
'title': 'Heart Crime S01 E03 weciima autos',
|
||||
'ext': 'mp4'
|
||||
},
|
||||
}, {
|
||||
'note': 'JWPlayer link in clear',
|
||||
'url': 'https://w1.viidshar.com/nnibe0xf0h79.html',
|
||||
'md5': 'f0a580ce9df06cc61b4a5c979d672367',
|
||||
'info_dict': {
|
||||
'id': 'nnibe0xf0h79',
|
||||
'title': 'JaGa 68ar',
|
||||
'ext': 'mp4'
|
||||
},
|
||||
'params': {
|
||||
'skip_download': 'ffmpeg',
|
||||
},
|
||||
'expected_warnings': ['hlsnative has detected features it does not support'],
|
||||
}, {
|
||||
'note': 'JWPlayer link in clear',
|
||||
'url': 'https://wolfstream.tv/a3drtehyrg52.html',
|
||||
'md5': '1901d86a79c5e0c6a51bdc9a4cfd3769',
|
||||
'info_dict': {
|
||||
'id': 'a3drtehyrg52',
|
||||
'title': 'NFL 2023 W04 DET@GB',
|
||||
'ext': 'mp4'
|
||||
},
|
||||
}, {
|
||||
'url': 'https://aparat.cam/n4d6dh0wvlpr',
|
||||
'only_matching': True,
|
||||
}, {
|
||||
'url': 'https://wolfstream.tv/nthme29v9u2x',
|
||||
'url': 'https://uqload.to/ug5somm0ctnk.html',
|
||||
'only_matching': True,
|
||||
}, {
|
||||
'url': 'https://highstream.tv/2owiyz3sjoux',
|
||||
'only_matching': True,
|
||||
}, {
|
||||
'url': 'https://vedbam.xyz/6lnbkci96wly.html',
|
||||
'only_matching': True,
|
||||
}]
|
||||
|
||||
@staticmethod
|
||||
def _extract_urls(webpage):
|
||||
return [
|
||||
mobj.group('url')
|
||||
for mobj in re.finditer(
|
||||
r'<iframe\b[^>]+\bsrc=(["\'])(?P<url>(?:https?:)?//(?:%s)/embed-[0-9a-zA-Z]+.*?)\1'
|
||||
% '|'.join(site for site in list(zip(*XFileShareIE._SITES))[0]),
|
||||
webpage)]
|
||||
@classmethod
|
||||
def _extract_urls(cls, webpage):
|
||||
|
||||
def yield_urls():
|
||||
for regex in cls._EMBED_REGEX:
|
||||
for mobj in re.finditer(regex, webpage):
|
||||
yield mobj.group('url')
|
||||
|
||||
return list(yield_urls())
|
||||
|
||||
def _real_extract(self, url):
|
||||
host, video_id = re.match(self._VALID_URL, url).groups()
|
||||
host, video_id = self._match_valid_url(url).group('host', 'id')
|
||||
|
||||
url = 'https://%s/' % host + ('embed-%s.html' % video_id if host in ('govid.me', 'vidlo.us') else video_id)
|
||||
url = 'https://%s/%s' % (
|
||||
host,
|
||||
'embed-%s.html' % video_id if host in ('govid.me', 'vidlo.us') else video_id)
|
||||
webpage = self._download_webpage(url, video_id)
|
||||
|
||||
if any(re.search(p, webpage) for p in self._FILE_NOT_FOUND_REGEXES):
|
||||
container_div = get_element_by_id('container', webpage) or webpage
|
||||
if self._search_regex(
|
||||
r'>This server is in maintenance mode\.', container_div,
|
||||
'maint error', group=0, default=None):
|
||||
raise ExtractorError(clean_html(container_div), expected=True)
|
||||
if self._search_regex(
|
||||
self._FILE_NOT_FOUND_REGEXES, container_div,
|
||||
'missing video error', group=0, default=None):
|
||||
raise ExtractorError('Video %s does not exist' % video_id, expected=True)
|
||||
|
||||
fields = self._hidden_inputs(webpage)
|
||||
|
@ -122,59 +226,43 @@ class XFileShareIE(InfoExtractor):
|
|||
'Content-type': 'application/x-www-form-urlencoded',
|
||||
})
|
||||
|
||||
title = (self._search_regex(
|
||||
(r'style="z-index: [0-9]+;">([^<]+)</span>',
|
||||
r'<td nowrap>([^<]+)</td>',
|
||||
r'h4-fine[^>]*>([^<]+)<',
|
||||
r'>Watch (.+)[ <]',
|
||||
r'<h2 class="video-page-head">([^<]+)</h2>',
|
||||
r'<h2 style="[^"]*color:#403f3d[^"]*"[^>]*>([^<]+)<', # streamin.to
|
||||
r'title\s*:\s*"([^"]+)"'), # govid.me
|
||||
webpage, 'title', default=None) or self._og_search_title(
|
||||
webpage, default=None) or video_id).strip()
|
||||
title = (
|
||||
self._search_regex(self._TITLE_REGEXES, webpage, 'title', default=None)
|
||||
or self._og_search_title(webpage, default=None)
|
||||
or video_id).strip()
|
||||
|
||||
for regex, func in (
|
||||
(r'(eval\(function\(p,a,c,k,e,d\){.+)', decode_packed_codes),
|
||||
(r'(゚.+)', aa_decode)):
|
||||
obf_code = self._search_regex(regex, webpage, 'obfuscated code', default=None)
|
||||
if obf_code:
|
||||
webpage = webpage.replace(obf_code, func(obf_code))
|
||||
obf_code = True
|
||||
while obf_code:
|
||||
for regex, func in (
|
||||
(r'(?s)(?<!-)\b(eval\(function\(p,a,c,k,e,d\)\{(?:(?!</script>).)+\)\))',
|
||||
decode_packed_codes),
|
||||
(r'(゚.+)', aa_decode)):
|
||||
obf_code = self._search_regex(regex, webpage, 'obfuscated code', default=None)
|
||||
if obf_code:
|
||||
webpage = webpage.replace(obf_code, func(obf_code))
|
||||
break
|
||||
|
||||
formats = []
|
||||
jwplayer_data = self._find_jwplayer_data(
|
||||
webpage.replace(r'\'', '\''), video_id)
|
||||
result = self._parse_jwplayer_data(
|
||||
jwplayer_data, video_id, require_title=False,
|
||||
m3u8_id='hls', mpd_id='dash')
|
||||
|
||||
jwplayer_data = self._search_regex(
|
||||
[
|
||||
r'jwplayer\("[^"]+"\)\.load\(\[({.+?})\]\);',
|
||||
r'jwplayer\("[^"]+"\)\.setup\(({.+?})\);',
|
||||
], webpage,
|
||||
'jwplayer data', default=None)
|
||||
if jwplayer_data:
|
||||
jwplayer_data = self._parse_json(
|
||||
jwplayer_data.replace(r"\'", "'"), video_id, js_to_json)
|
||||
if not traverse_obj(result, 'formats'):
|
||||
if jwplayer_data:
|
||||
formats = self._parse_jwplayer_data(
|
||||
jwplayer_data, video_id, False,
|
||||
m3u8_id='hls', mpd_id='dash')['formats']
|
||||
|
||||
if not formats:
|
||||
urls = []
|
||||
for regex in (
|
||||
r'(?:file|src)\s*:\s*(["\'])(?P<url>http(?:(?!\1).)+\.(?:m3u8|mp4|flv)(?:(?!\1).)*)\1',
|
||||
r'file_link\s*=\s*(["\'])(?P<url>http(?:(?!\1).)+)\1',
|
||||
r'addVariable\((\\?["\'])file\1\s*,\s*(\\?["\'])(?P<url>http(?:(?!\2).)+)\2\)',
|
||||
r'<embed[^>]+src=(["\'])(?P<url>http(?:(?!\1).)+\.(?:m3u8|mp4|flv)(?:(?!\1).)*)\1'):
|
||||
self.report_warning(
|
||||
'Failed to extract JWPlayer formats', video_id=video_id)
|
||||
urls = set()
|
||||
for regex in self._SOURCE_URL_REGEXES:
|
||||
for mobj in re.finditer(regex, webpage):
|
||||
video_url = mobj.group('url')
|
||||
if video_url not in urls:
|
||||
urls.append(video_url)
|
||||
urls.add(mobj.group('url'))
|
||||
|
||||
sources = self._search_regex(
|
||||
r'sources\s*:\s*(\[(?!{)[^\]]+\])', webpage, 'sources', default=None)
|
||||
if sources:
|
||||
urls.extend(self._parse_json(sources, video_id))
|
||||
urls.update(traverse_obj(sources, (T(lambda s: self._parse_json(s, video_id)), Ellipsis)))
|
||||
|
||||
formats = []
|
||||
for video_url in urls:
|
||||
for video_url in traverse_obj(urls, (Ellipsis, T(url_or_none))):
|
||||
if determine_ext(video_url) == 'm3u8':
|
||||
formats.extend(self._extract_m3u8_formats(
|
||||
video_url, video_id, 'mp4',
|
||||
|
@ -185,17 +273,19 @@ class XFileShareIE(InfoExtractor):
|
|||
'url': video_url,
|
||||
'format_id': 'sd',
|
||||
})
|
||||
self._sort_formats(formats)
|
||||
result = {'formats': formats}
|
||||
|
||||
self._sort_formats(result['formats'])
|
||||
|
||||
thumbnail = self._search_regex(
|
||||
[
|
||||
r'<video[^>]+poster="([^"]+)"',
|
||||
r'(?:image|poster)\s*:\s*["\'](http[^"\']+)["\'],',
|
||||
], webpage, 'thumbnail', default=None)
|
||||
self._THUMBNAIL_REGEXES, webpage, 'thumbnail', default=None)
|
||||
|
||||
return {
|
||||
if not (title or result.get('title')):
|
||||
title = self._generic_title(url) or video_id
|
||||
|
||||
return merge_dicts(result, {
|
||||
'id': video_id,
|
||||
'title': title,
|
||||
'title': title or None,
|
||||
'thumbnail': thumbnail,
|
||||
'formats': formats,
|
||||
}
|
||||
'http_headers': {'Referer': url}
|
||||
})
|
||||
|
|
|
@ -1,20 +1,38 @@
|
|||
# coding: utf-8
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import itertools
|
||||
import re
|
||||
from time import sleep
|
||||
|
||||
from .common import InfoExtractor
|
||||
from ..utils import (
|
||||
clean_html,
|
||||
extract_attributes,
|
||||
ExtractorError,
|
||||
get_element_by_class,
|
||||
get_element_by_id,
|
||||
int_or_none,
|
||||
str_to_int,
|
||||
merge_dicts,
|
||||
parse_count,
|
||||
parse_qs,
|
||||
T,
|
||||
traverse_obj,
|
||||
unified_strdate,
|
||||
url_or_none,
|
||||
urljoin,
|
||||
)
|
||||
|
||||
|
||||
class YouPornIE(InfoExtractor):
|
||||
_VALID_URL = r'https?://(?:www\.)?youporn\.com/(?:watch|embed)/(?P<id>\d+)(?:/(?P<display_id>[^/?#&]+))?'
|
||||
_VALID_URL = (
|
||||
r'youporn:(?P<id>\d+)',
|
||||
r'''(?x)
|
||||
https?://(?:www\.)?youporn\.com/(?:watch|embed)/(?P<id>\d+)
|
||||
(?:/(?:(?P<display_id>[^/?#&]+)/?)?)?(?:[#?]|$)
|
||||
'''
|
||||
)
|
||||
_EMBED_REGEX = [r'<iframe[^>]+\bsrc=["\'](?P<url>(?:https?:)?//(?:www\.)?youporn\.com/embed/\d+)']
|
||||
_TESTS = [{
|
||||
'url': 'http://www.youporn.com/watch/505835/sex-ed-is-it-safe-to-masturbate-daily/',
|
||||
'md5': '3744d24c50438cf5b6f6d59feb5055c2',
|
||||
|
@ -34,7 +52,7 @@ class YouPornIE(InfoExtractor):
|
|||
'tags': list,
|
||||
'age_limit': 18,
|
||||
},
|
||||
'skip': 'This video has been disabled',
|
||||
'skip': 'This video has been deactivated',
|
||||
}, {
|
||||
# Unknown uploader
|
||||
'url': 'http://www.youporn.com/watch/561726/big-tits-awesome-brunette-on-amazing-webcam-show/?from=related3&al=2&from_id=561726&pos=4',
|
||||
|
@ -66,57 +84,104 @@ class YouPornIE(InfoExtractor):
|
|||
}, {
|
||||
'url': 'https://www.youporn.com/watch/13922959/femdom-principal/',
|
||||
'only_matching': True,
|
||||
}, {
|
||||
'url': 'https://www.youporn.com/watch/16290308/tinderspecial-trailer1/',
|
||||
'info_dict': {
|
||||
'id': '16290308',
|
||||
'age_limit': 18,
|
||||
'categories': [],
|
||||
'description': None, # SEO spam using title removed
|
||||
'display_id': 'tinderspecial-trailer1',
|
||||
'duration': 298.0,
|
||||
'ext': 'mp4',
|
||||
'upload_date': '20201123',
|
||||
'uploader': 'Ersties',
|
||||
'tags': [],
|
||||
'thumbnail': 'https://fi1.ypncdn.com/m=eaSaaTbWx/202011/23/16290308/original/3.jpg',
|
||||
'timestamp': 1606147564,
|
||||
'title': 'Tinder In Real Life',
|
||||
'view_count': int,
|
||||
}
|
||||
}]
|
||||
|
||||
@staticmethod
|
||||
def _extract_urls(webpage):
|
||||
return re.findall(
|
||||
r'<iframe[^>]+\bsrc=["\']((?:https?:)?//(?:www\.)?youporn\.com/embed/\d+)',
|
||||
webpage)
|
||||
@classmethod
|
||||
def _extract_urls(cls, webpage):
|
||||
def yield_urls():
|
||||
for p in cls._EMBED_REGEX:
|
||||
for m in re.finditer(p, webpage):
|
||||
yield m.group('url')
|
||||
|
||||
return list(yield_urls())
|
||||
|
||||
def _real_extract(self, url):
|
||||
mobj = re.match(self._VALID_URL, url)
|
||||
video_id = mobj.group('id')
|
||||
display_id = mobj.group('display_id') or video_id
|
||||
# A different video ID (data-video-id) is hidden in the page but
|
||||
# never seems to be used
|
||||
video_id, display_id = self._match_valid_url(url).group('id', 'display_id')
|
||||
url = 'http://www.youporn.com/watch/%s' % (video_id,)
|
||||
webpage = self._download_webpage(
|
||||
url, video_id, headers={'Cookie': 'age_verified=1'})
|
||||
|
||||
definitions = self._download_json(
|
||||
'https://www.youporn.com/api/video/media_definitions/%s/' % video_id,
|
||||
display_id)
|
||||
watchable = self._search_regex(
|
||||
r'''(<div\s[^>]*\bid\s*=\s*('|")?watch-container(?(2)\2|(?!-)\b)[^>]*>)''',
|
||||
webpage, 'watchability', default=None)
|
||||
if not watchable:
|
||||
msg = re.split(r'\s{4}', clean_html(get_element_by_id(
|
||||
'mainContent', webpage)) or '')[0]
|
||||
raise ExtractorError(
|
||||
('%s says: %s' % (self.IE_NAME, msg))
|
||||
if msg else 'Video unavailable: no reason found',
|
||||
expected=True)
|
||||
# internal ID ?
|
||||
# video_id = extract_attributes(watchable).get('data-video-id')
|
||||
|
||||
playervars = self._search_json(
|
||||
r'\bplayervars\s*:', webpage, 'playervars', video_id)
|
||||
|
||||
def get_fmt(x):
|
||||
v_url = url_or_none(x.get('videoUrl'))
|
||||
if v_url:
|
||||
x['videoUrl'] = v_url
|
||||
return (x['format'], x)
|
||||
|
||||
defs_by_format = dict(traverse_obj(playervars, (
|
||||
'mediaDefinitions', lambda _, v: v.get('format'), T(get_fmt))))
|
||||
|
||||
def get_format_data(f):
|
||||
if f not in defs_by_format:
|
||||
return []
|
||||
return self._download_json(
|
||||
defs_by_format[f]['videoUrl'], video_id, '{0}-formats'.format(f))
|
||||
|
||||
formats = []
|
||||
for definition in definitions:
|
||||
if not isinstance(definition, dict):
|
||||
continue
|
||||
video_url = url_or_none(definition.get('videoUrl'))
|
||||
if not video_url:
|
||||
continue
|
||||
f = {
|
||||
'url': video_url,
|
||||
'filesize': int_or_none(definition.get('videoSize')),
|
||||
}
|
||||
height = int_or_none(definition.get('quality'))
|
||||
# Try to extract only the actual master m3u8 first, avoiding the duplicate single resolution "master" m3u8s
|
||||
for hls_url in traverse_obj(
|
||||
get_format_data('hls'),
|
||||
(lambda _, v: not isinstance(v['defaultQuality'], bool), 'videoUrl'),
|
||||
(Ellipsis, 'videoUrl')):
|
||||
formats.extend(self._extract_m3u8_formats(
|
||||
hls_url, video_id, 'mp4', fatal=False, m3u8_id='hls',
|
||||
entry_protocol='m3u8_native'))
|
||||
|
||||
for f in traverse_obj(get_format_data('mp4'), (
|
||||
lambda _, v: v.get('videoUrl'), {
|
||||
'url': ('videoUrl', T(url_or_none)),
|
||||
'filesize': ('videoSize', T(int_or_none)),
|
||||
'height': ('quality', T(int_or_none)),
|
||||
}, T(lambda x: x.get('videoUrl') and x))):
|
||||
# Video URL's path looks like this:
|
||||
# /201012/17/505835/720p_1500k_505835/YouPorn%20-%20Sex%20Ed%20Is%20It%20Safe%20To%20Masturbate%20Daily.mp4
|
||||
# /201012/17/505835/vl_240p_240k_505835/YouPorn%20-%20Sex%20Ed%20Is%20It%20Safe%20To%20Masturbate%20Daily.mp4
|
||||
# /videos/201703/11/109285532/1080P_4000K_109285532.mp4
|
||||
# We will benefit from it by extracting some metadata
|
||||
mobj = re.search(r'(?P<height>\d{3,4})[pP]_(?P<bitrate>\d+)[kK]_\d+', video_url)
|
||||
mobj = re.search(r'(?P<height>\d{3,4})[pP]_(?P<bitrate>\d+)[kK]_\d+', f['videoUrl'])
|
||||
if mobj:
|
||||
if not height:
|
||||
height = int(mobj.group('height'))
|
||||
bitrate = int(mobj.group('bitrate'))
|
||||
f.update({
|
||||
'format_id': '%dp-%dk' % (height, bitrate),
|
||||
'tbr': bitrate,
|
||||
})
|
||||
f['height'] = height
|
||||
if not f.get('height'):
|
||||
f['height'] = int(mobj.group('height'))
|
||||
f['tbr'] = int(mobj.group('bitrate'))
|
||||
f['format_id'] = '%dp-%dk' % (f['height'], f['tbr'])
|
||||
formats.append(f)
|
||||
self._sort_formats(formats)
|
||||
|
||||
webpage = self._download_webpage(
|
||||
'http://www.youporn.com/watch/%s' % video_id, display_id,
|
||||
headers={'Cookie': 'age_verified=1'})
|
||||
|
||||
title = self._html_search_regex(
|
||||
r'(?s)<div[^>]+class=["\']watchVideoTitle[^>]+>(.+?)</div>',
|
||||
webpage, 'title', default=None) or self._og_search_title(
|
||||
|
@ -131,8 +196,10 @@ class YouPornIE(InfoExtractor):
|
|||
thumbnail = self._search_regex(
|
||||
r'(?:imageurl\s*=|poster\s*:)\s*(["\'])(?P<thumbnail>.+?)\1',
|
||||
webpage, 'thumbnail', fatal=False, group='thumbnail')
|
||||
duration = int_or_none(self._html_search_meta(
|
||||
'video:duration', webpage, 'duration', fatal=False))
|
||||
duration = traverse_obj(playervars, ('duration', T(int_or_none)))
|
||||
if duration is None:
|
||||
duration = int_or_none(self._html_search_meta(
|
||||
'video:duration', webpage, 'duration', fatal=False))
|
||||
|
||||
uploader = self._html_search_regex(
|
||||
r'(?s)<div[^>]+class=["\']submitByLink["\'][^>]*>(.+?)</div>',
|
||||
|
@ -148,11 +215,11 @@ class YouPornIE(InfoExtractor):
|
|||
|
||||
view_count = None
|
||||
views = self._search_regex(
|
||||
r'(<div[^>]+\bclass=["\']js_videoInfoViews["\']>)', webpage,
|
||||
'views', default=None)
|
||||
r'(<div\s[^>]*\bdata-value\s*=[^>]+>)\s*<label>Views:</label>',
|
||||
webpage, 'views', default=None)
|
||||
if views:
|
||||
view_count = str_to_int(extract_attributes(views).get('data-value'))
|
||||
comment_count = str_to_int(self._search_regex(
|
||||
view_count = parse_count(extract_attributes(views).get('data-value'))
|
||||
comment_count = parse_count(self._search_regex(
|
||||
r'>All [Cc]omments? \(([\d,.]+)\)',
|
||||
webpage, 'comment count', default=None))
|
||||
|
||||
|
@ -168,7 +235,10 @@ class YouPornIE(InfoExtractor):
|
|||
r'(?s)Tags:.*?</div>\s*<div[^>]+class=["\']tagBoxContent["\'][^>]*>(.+?)</div>',
|
||||
'tags')
|
||||
|
||||
return {
|
||||
data = self._search_json_ld(webpage, video_id, expected_type='VideoObject', fatal=False) or {}
|
||||
data.pop('url', None)
|
||||
|
||||
result = merge_dicts(data, {
|
||||
'id': video_id,
|
||||
'display_id': display_id,
|
||||
'title': title,
|
||||
|
@ -183,4 +253,442 @@ class YouPornIE(InfoExtractor):
|
|||
'tags': tags,
|
||||
'age_limit': age_limit,
|
||||
'formats': formats,
|
||||
}
|
||||
})
|
||||
# Remove promotional non-description
|
||||
if result.get('description', '').startswith(
|
||||
'Watch %s online' % (result['title'],)):
|
||||
del result['description']
|
||||
return result
|
||||
|
||||
|
||||
class YouPornListBase(InfoExtractor):
|
||||
# pattern in '.title-text' element of page section containing videos
|
||||
_PLAYLIST_TITLEBAR_RE = r'\s+[Vv]ideos\s*$'
|
||||
_PAGE_RETRY_COUNT = 0 # ie, no retry
|
||||
_PAGE_RETRY_DELAY = 2 # seconds
|
||||
|
||||
def _get_next_url(self, url, pl_id, html):
|
||||
return urljoin(url, self._search_regex(
|
||||
r'''<a\s[^>]*?\bhref\s*=\s*("|')(?P<url>(?:(?!\1)[^>])+)\1''',
|
||||
get_element_by_id('next', html) or '', 'next page',
|
||||
group='url', default=None))
|
||||
|
||||
@classmethod
|
||||
def _get_title_from_slug(cls, title_slug):
|
||||
return re.sub(r'[_-]', ' ', title_slug)
|
||||
|
||||
def _entries(self, url, pl_id, html=None, page_num=None):
|
||||
|
||||
# separates page sections
|
||||
PLAYLIST_SECTION_RE = (
|
||||
r'''<div\s[^>]*\bclass\s*=\s*('|")(?:[\w$-]+\s+|\s)*?title-bar(?:\s+[\w$-]+|\s)*\1[^>]*>'''
|
||||
)
|
||||
# contains video link
|
||||
VIDEO_URL_RE = r'''(?x)
|
||||
<div\s[^>]*\bdata-video-id\s*=\s*('|")\d+\1[^>]*>\s*
|
||||
(?:<div\b[\s\S]+?</div>\s*)*
|
||||
<a\s[^>]*\bhref\s*=\s*('|")(?P<url>(?:(?!\2)[^>])+)\2
|
||||
'''
|
||||
|
||||
def yield_pages(url, html=html, page_num=page_num):
|
||||
fatal = not html
|
||||
for pnum in itertools.count(start=page_num or 1):
|
||||
if not html:
|
||||
html = self._download_webpage(
|
||||
url, pl_id, note='Downloading page %d' % pnum,
|
||||
fatal=fatal)
|
||||
if not html:
|
||||
break
|
||||
fatal = False
|
||||
yield (url, html, pnum)
|
||||
# explicit page: extract just that page
|
||||
if page_num is not None:
|
||||
break
|
||||
next_url = self._get_next_url(url, pl_id, html)
|
||||
if not next_url or next_url == url:
|
||||
break
|
||||
url, html = next_url, None
|
||||
|
||||
def retry_page(msg, tries_left, page_data):
|
||||
if tries_left <= 0:
|
||||
return
|
||||
self.report_warning(msg, pl_id)
|
||||
sleep(self._PAGE_RETRY_DELAY)
|
||||
return next(
|
||||
yield_pages(page_data[0], page_num=page_data[2]), None)
|
||||
|
||||
def yield_entries(html):
|
||||
for frag in re.split(PLAYLIST_SECTION_RE, html):
|
||||
if not frag:
|
||||
continue
|
||||
t_text = get_element_by_class('title-text', frag or '')
|
||||
if not (t_text and re.search(self._PLAYLIST_TITLEBAR_RE, t_text)):
|
||||
continue
|
||||
for m in re.finditer(VIDEO_URL_RE, frag):
|
||||
video_url = urljoin(url, m.group('url'))
|
||||
if video_url:
|
||||
yield self.url_result(video_url)
|
||||
|
||||
last_first_url = None
|
||||
for page_data in yield_pages(url, html=html, page_num=page_num):
|
||||
# page_data: url, html, page_num
|
||||
first_url = None
|
||||
tries_left = self._PAGE_RETRY_COUNT + 1
|
||||
while tries_left > 0:
|
||||
tries_left -= 1
|
||||
for from_ in yield_entries(page_data[1]):
|
||||
# may get the same page twice instead of empty page
|
||||
# or (site bug) intead of actual next page
|
||||
if not first_url:
|
||||
first_url = from_['url']
|
||||
if first_url == last_first_url:
|
||||
# sometimes (/porntags/) the site serves the previous page
|
||||
# instead but may provide the correct page after a delay
|
||||
page_data = retry_page(
|
||||
'Retrying duplicate page...', tries_left, page_data)
|
||||
if page_data:
|
||||
first_url = None
|
||||
break
|
||||
continue
|
||||
yield from_
|
||||
else:
|
||||
if not first_url and 'no-result-paragarph1' in page_data[1]:
|
||||
page_data = retry_page(
|
||||
'Retrying empty page...', tries_left, page_data)
|
||||
if page_data:
|
||||
continue
|
||||
else:
|
||||
# success/failure
|
||||
break
|
||||
# may get an infinite (?) sequence of empty pages
|
||||
if not first_url:
|
||||
break
|
||||
last_first_url = first_url
|
||||
|
||||
def _real_extract(self, url, html=None):
|
||||
# exceptionally, id may be None
|
||||
m_dict = self._match_valid_url(url).groupdict()
|
||||
pl_id, page_type, sort = (m_dict.get(k) for k in ('id', 'type', 'sort'))
|
||||
|
||||
qs = parse_qs(url)
|
||||
for q, v in qs.items():
|
||||
if v:
|
||||
qs[q] = v[-1]
|
||||
else:
|
||||
del qs[q]
|
||||
|
||||
base_id = pl_id or 'YouPorn'
|
||||
title = self._get_title_from_slug(base_id)
|
||||
if page_type:
|
||||
title = '%s %s' % (page_type.capitalize(), title)
|
||||
base_id = [base_id.lower()]
|
||||
if sort is None:
|
||||
title += ' videos'
|
||||
else:
|
||||
title = '%s videos by %s' % (title, re.sub(r'[_-]', ' ', sort))
|
||||
base_id.append(sort)
|
||||
if qs:
|
||||
ps = ['%s=%s' % item for item in sorted(qs.items())]
|
||||
title += ' (%s)' % ','.join(ps)
|
||||
base_id.extend(ps)
|
||||
pl_id = '/'.join(base_id)
|
||||
|
||||
return self.playlist_result(
|
||||
self._entries(url, pl_id, html=html,
|
||||
page_num=int_or_none(qs.get('page'))),
|
||||
playlist_id=pl_id, playlist_title=title)
|
||||
|
||||
|
||||
class YouPornCategoryIE(YouPornListBase):
|
||||
IE_DESC = 'YouPorn category, with sorting, filtering and pagination'
|
||||
_VALID_URL = r'''(?x)
|
||||
https?://(?:www\.)?youporn\.com/
|
||||
(?P<type>category)/(?P<id>[^/?#&]+)
|
||||
(?:/(?P<sort>popular|views|rating|time|duration))?/?(?:[#?]|$)
|
||||
'''
|
||||
_TESTS = [{
|
||||
'note': 'Full list with pagination',
|
||||
'url': 'https://www.youporn.com/category/lingerie/popular/',
|
||||
'info_dict': {
|
||||
'id': 'lingerie/popular',
|
||||
'title': 'Category lingerie videos by popular',
|
||||
},
|
||||
'playlist_mincount': 39,
|
||||
}, {
|
||||
'note': 'Filtered paginated list with single page result',
|
||||
'url': 'https://www.youporn.com/category/lingerie/duration/?min_minutes=10',
|
||||
'info_dict': {
|
||||
'id': 'lingerie/duration/min_minutes=10',
|
||||
'title': 'Category lingerie videos by duration (min_minutes=10)',
|
||||
},
|
||||
'playlist_maxcount': 30,
|
||||
}, {
|
||||
'note': 'Single page of full list',
|
||||
'url': 'https://www.youporn.com/category/lingerie/popular?page=1',
|
||||
'info_dict': {
|
||||
'id': 'lingerie/popular/page=1',
|
||||
'title': 'Category lingerie videos by popular (page=1)',
|
||||
},
|
||||
'playlist_count': 30,
|
||||
}]
|
||||
|
||||
|
||||
class YouPornChannelIE(YouPornListBase):
|
||||
IE_DESC = 'YouPorn channel, with sorting and pagination'
|
||||
_VALID_URL = r'''(?x)
|
||||
https?://(?:www\.)?youporn\.com/
|
||||
(?P<type>channel)/(?P<id>[^/?#&]+)
|
||||
(?:/(?P<sort>rating|views|duration))?/?(?:[#?]|$)
|
||||
'''
|
||||
_TESTS = [{
|
||||
'note': 'Full list with pagination',
|
||||
'url': 'https://www.youporn.com/channel/x-feeds/',
|
||||
'info_dict': {
|
||||
'id': 'x-feeds',
|
||||
'title': 'Channel X-Feeds videos',
|
||||
},
|
||||
'playlist_mincount': 37,
|
||||
}, {
|
||||
'note': 'Single page of full list (no filters here)',
|
||||
'url': 'https://www.youporn.com/channel/x-feeds/duration?page=1',
|
||||
'info_dict': {
|
||||
'id': 'x-feeds/duration/page=1',
|
||||
'title': 'Channel X-Feeds videos by duration (page=1)',
|
||||
},
|
||||
'playlist_count': 24,
|
||||
}]
|
||||
|
||||
@staticmethod
|
||||
def _get_title_from_slug(title_slug):
|
||||
return re.sub(r'_', ' ', title_slug).title()
|
||||
|
||||
|
||||
class YouPornCollectionIE(YouPornListBase):
|
||||
IE_DESC = 'YouPorn collection (user playlist), with sorting and pagination'
|
||||
_VALID_URL = r'''(?x)
|
||||
https?://(?:www\.)?youporn\.com/
|
||||
(?P<type>collection)s/videos/(?P<id>\d+)
|
||||
(?:/(?P<sort>rating|views|time|duration))?/?(?:[#?]|$)
|
||||
'''
|
||||
_PLAYLIST_TITLEBAR_RE = r'^\s*Videos\s+in\s'
|
||||
_TESTS = [{
|
||||
'note': 'Full list with pagination',
|
||||
'url': 'https://www.youporn.com/collections/videos/33044251/',
|
||||
'info_dict': {
|
||||
'id': '33044251',
|
||||
'title': 'Collection Sexy Lips videos',
|
||||
'uploader': 'ph-littlewillyb',
|
||||
},
|
||||
'playlist_mincount': 50,
|
||||
}, {
|
||||
'note': 'Single page of full list (no filters here)',
|
||||
'url': 'https://www.youporn.com/collections/videos/33044251/time?page=1',
|
||||
'info_dict': {
|
||||
'id': '33044251/time/page=1',
|
||||
'title': 'Collection Sexy Lips videos by time (page=1)',
|
||||
'uploader': 'ph-littlewillyb',
|
||||
},
|
||||
'playlist_count': 20,
|
||||
}]
|
||||
|
||||
def _real_extract(self, url):
|
||||
pl_id = self._match_id(url)
|
||||
html = self._download_webpage(url, pl_id)
|
||||
playlist = super(YouPornCollectionIE, self)._real_extract(url, html=html)
|
||||
infos = re.sub(r'\s+', ' ', clean_html(get_element_by_class(
|
||||
'collection-infos', html)) or '')
|
||||
title, uploader = self._search_regex(
|
||||
r'^\s*Collection: (?P<title>.+?) \d+ VIDEOS \d+ VIEWS \d+ days LAST UPDATED From: (?P<uploader>[\w_-]+)',
|
||||
infos, 'title/uploader', group=('title', 'uploader'), default=(None, None))
|
||||
|
||||
return merge_dicts({
|
||||
'title': playlist['title'].replace(playlist['id'].split('/')[0], title),
|
||||
'uploader': uploader,
|
||||
}, playlist) if title else playlist
|
||||
|
||||
|
||||
class YouPornTagIE(YouPornListBase):
|
||||
IE_DESC = 'YouPorn tag (porntags), with sorting, filtering and pagination'
|
||||
_VALID_URL = r'''(?x)
|
||||
https?://(?:www\.)?youporn\.com/
|
||||
porn(?P<type>tag)s/(?P<id>[^/?#&]+)
|
||||
(?:/(?P<sort>views|rating|time|duration))?/?(?:[#?]|$)
|
||||
'''
|
||||
_PLAYLIST_TITLEBAR_RE = r'^\s*Videos\s+tagged\s'
|
||||
_PAGE_RETRY_COUNT = 1
|
||||
_TESTS = [{
|
||||
'note': 'Full list with pagination',
|
||||
'url': 'https://www.youporn.com/porntags/austrian',
|
||||
'info_dict': {
|
||||
'id': 'austrian',
|
||||
'title': 'Tag austrian videos',
|
||||
},
|
||||
'playlist_mincount': 35,
|
||||
'expected_warnings': ['Retrying duplicate page'],
|
||||
}, {
|
||||
'note': 'Filtered paginated list with single page result',
|
||||
'url': 'https://www.youporn.com/porntags/austrian/duration/?min_minutes=10',
|
||||
'info_dict': {
|
||||
'id': 'austrian/duration/min_minutes=10',
|
||||
'title': 'Tag austrian videos by duration (min_minutes=10)',
|
||||
},
|
||||
# number of videos per page is (row x col) 2x3 + 6x4 + 2, or + 3,
|
||||
# or more, varying with number of ads; let's set max as 9x4
|
||||
# NB col 1 may not be shown in non-JS page with site CSS and zoom 100%
|
||||
'playlist_maxcount': 32,
|
||||
'expected_warnings': ['Retrying duplicate page', 'Retrying empty page'],
|
||||
}, {
|
||||
'note': 'Single page of full list',
|
||||
'url': 'https://www.youporn.com/porntags/austrian/?page=1',
|
||||
'info_dict': {
|
||||
'id': 'austrian/page=1',
|
||||
'title': 'Tag austrian videos (page=1)',
|
||||
},
|
||||
'playlist_mincount': 32,
|
||||
'playlist_maxcount': 34,
|
||||
'expected_warnings': ['Retrying duplicate page', 'Retrying empty page'],
|
||||
}]
|
||||
|
||||
# YP tag navigation is broken, loses sort
|
||||
def _get_next_url(self, url, pl_id, html):
|
||||
next_url = super(YouPornTagIE, self)._get_next_url(url, pl_id, html)
|
||||
if next_url:
|
||||
n = self._match_valid_url(next_url)
|
||||
if n:
|
||||
s = n.groupdict().get('sort')
|
||||
if s:
|
||||
u = self._match_valid_url(url)
|
||||
if u:
|
||||
u = u.groupdict().get('sort')
|
||||
if s and not u:
|
||||
n = n.end('sort')
|
||||
next_url = next_url[:n] + '/' + u + next_url[n:]
|
||||
return next_url
|
||||
|
||||
|
||||
class YouPornStarIE(YouPornListBase):
|
||||
IE_DESC = 'YouPorn Pornstar, with description, sorting and pagination'
|
||||
_VALID_URL = r'''(?x)
|
||||
https?://(?:www\.)?youporn\.com/
|
||||
(?P<type>pornstar)/(?P<id>[^/?#&]+)
|
||||
(?:/(?P<sort>rating|views|duration))?/?(?:[#?]|$)
|
||||
'''
|
||||
_PLAYLIST_TITLEBAR_RE = r'^\s*Videos\s+[fF]eaturing\s'
|
||||
_TESTS = [{
|
||||
'note': 'Full list with pagination',
|
||||
'url': 'https://www.youporn.com/pornstar/daynia/',
|
||||
'info_dict': {
|
||||
'id': 'daynia',
|
||||
'title': 'Pornstar Daynia videos',
|
||||
'description': r're:Daynia Rank \d+ Videos \d+ Views [\d,.]+ .+ Subscribers \d+',
|
||||
},
|
||||
'playlist_mincount': 45,
|
||||
}, {
|
||||
'note': 'Single page of full list (no filters here)',
|
||||
'url': 'https://www.youporn.com/pornstar/daynia/?page=1',
|
||||
'info_dict': {
|
||||
'id': 'daynia/page=1',
|
||||
'title': 'Pornstar Daynia videos (page=1)',
|
||||
'description': 're:.{180,}',
|
||||
},
|
||||
'playlist_count': 26,
|
||||
}]
|
||||
|
||||
@staticmethod
|
||||
def _get_title_from_slug(title_slug):
|
||||
return re.sub(r'_', ' ', title_slug).title()
|
||||
|
||||
def _real_extract(self, url):
|
||||
pl_id = self._match_id(url)
|
||||
html = self._download_webpage(url, pl_id)
|
||||
playlist = super(YouPornStarIE, self)._real_extract(url, html=html)
|
||||
INFO_ELEMENT_RE = r'''(?x)
|
||||
<div\s[^>]*\bclass\s*=\s*('|")(?:[\w$-]+\s+|\s)*?pornstar-info-wrapper(?:\s+[\w$-]+|\s)*\1[^>]*>
|
||||
(?P<info>[\s\S]+?)(?:</div>\s*){6,}
|
||||
'''
|
||||
|
||||
infos = self._search_regex(INFO_ELEMENT_RE, html, 'infos', group='info', default='')
|
||||
if infos:
|
||||
infos = re.sub(
|
||||
r'(?:\s*nl=nl)+\s*', ' ',
|
||||
re.sub(r'(?u)\s+', ' ', clean_html(
|
||||
re.sub('\n', 'nl=nl', infos)))).replace('ribe Subsc', '')
|
||||
|
||||
return merge_dicts({
|
||||
'description': infos.strip() or None,
|
||||
}, playlist)
|
||||
|
||||
|
||||
class YouPornVideosIE(YouPornListBase):
|
||||
IE_DESC = 'YouPorn video (browse) playlists, with sorting, filtering and pagination'
|
||||
_VALID_URL = r'''(?x)
|
||||
https?://(?:www\.)?youporn\.com/
|
||||
(?:(?P<id>browse)/)?
|
||||
(?P<sort>(?(id)
|
||||
(?:duration|rating|time|views)|
|
||||
(?:most_(?:favou?rit|view)ed|recommended|top_rated)?))
|
||||
(?:[/#?]|$)
|
||||
'''
|
||||
_PLAYLIST_TITLEBAR_RE = r'\s+(?:[Vv]ideos|VIDEOS)\s*$'
|
||||
_TESTS = [{
|
||||
'note': 'Full list with pagination (too long for test)',
|
||||
'url': 'https://www.youporn.com/',
|
||||
'info_dict': {
|
||||
'id': 'youporn',
|
||||
'title': 'YouPorn videos',
|
||||
},
|
||||
'only_matching': True,
|
||||
}, {
|
||||
'note': 'Full list with pagination (too long for test)',
|
||||
'url': 'https://www.youporn.com/recommended',
|
||||
'info_dict': {
|
||||
'id': 'youporn/recommended',
|
||||
'title': 'YouPorn videos by recommended',
|
||||
},
|
||||
'only_matching': True,
|
||||
}, {
|
||||
'note': 'Full list with pagination (too long for test)',
|
||||
'url': 'https://www.youporn.com/top_rated',
|
||||
'info_dict': {
|
||||
'id': 'youporn/top_rated',
|
||||
'title': 'YouPorn videos by top rated',
|
||||
},
|
||||
'only_matching': True,
|
||||
}, {
|
||||
'note': 'Full list with pagination (too long for test)',
|
||||
'url': 'https://www.youporn.com/browse/time',
|
||||
'info_dict': {
|
||||
'id': 'browse/time',
|
||||
'title': 'YouPorn videos by time',
|
||||
},
|
||||
'only_matching': True,
|
||||
}, {
|
||||
'note': 'Filtered paginated list with single page result',
|
||||
'url': 'https://www.youporn.com/most_favorited/?res=VR&max_minutes=2',
|
||||
'info_dict': {
|
||||
'id': 'youporn/most_favorited/max_minutes=2/res=VR',
|
||||
'title': 'YouPorn videos by most favorited (max_minutes=2,res=VR)',
|
||||
},
|
||||
'playlist_mincount': 10,
|
||||
'playlist_maxcount': 28,
|
||||
}, {
|
||||
'note': 'Filtered paginated list with several pages',
|
||||
'url': 'https://www.youporn.com/most_favorited/?res=VR&max_minutes=5',
|
||||
'info_dict': {
|
||||
'id': 'youporn/most_favorited/max_minutes=5/res=VR',
|
||||
'title': 'YouPorn videos by most favorited (max_minutes=5,res=VR)',
|
||||
},
|
||||
'playlist_mincount': 45,
|
||||
}, {
|
||||
'note': 'Single page of full list',
|
||||
'url': 'https://www.youporn.com/browse/time?page=1',
|
||||
'info_dict': {
|
||||
'id': 'browse/time/page=1',
|
||||
'title': 'YouPorn videos by time (page=1)',
|
||||
},
|
||||
'playlist_count': 36,
|
||||
}]
|
||||
|
||||
@staticmethod
|
||||
def _get_title_from_slug(title_slug):
|
||||
return 'YouPorn' if title_slug == 'browse' else title_slug
|
||||
|
|
|
@ -1647,10 +1647,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
|
|||
except JSInterpreter.Exception as e:
|
||||
self.report_warning(
|
||||
'%s (%s %s)' % (
|
||||
self.__ie_msg(
|
||||
'Unable to decode n-parameter: download likely to be throttled'),
|
||||
'Unable to decode n-parameter: download likely to be throttled',
|
||||
error_to_compat_str(e),
|
||||
traceback.format_exc()))
|
||||
traceback.format_exc()),
|
||||
video_id=video_id)
|
||||
return
|
||||
|
||||
self.write_debug('Decrypted nsig {0} => {1}'.format(n, ret))
|
||||
|
|
|
@ -74,8 +74,11 @@ class FFmpegPostProcessor(PostProcessor):
|
|||
return FFmpegPostProcessor(downloader)._versions
|
||||
|
||||
def _determine_executables(self):
|
||||
programs = ['avprobe', 'avconv', 'ffmpeg', 'ffprobe']
|
||||
# ordered to match prefer_ffmpeg!
|
||||
convs = ['ffmpeg', 'avconv']
|
||||
probes = ['ffprobe', 'avprobe']
|
||||
prefer_ffmpeg = True
|
||||
programs = convs + probes
|
||||
|
||||
def get_ffmpeg_version(path):
|
||||
ver = get_exe_version(path, args=['-version'])
|
||||
|
@ -96,6 +99,7 @@ class FFmpegPostProcessor(PostProcessor):
|
|||
|
||||
self._paths = None
|
||||
self._versions = None
|
||||
location = None
|
||||
if self._downloader:
|
||||
prefer_ffmpeg = self._downloader.params.get('prefer_ffmpeg', True)
|
||||
location = self._downloader.params.get('ffmpeg_location')
|
||||
|
@ -118,33 +122,21 @@ class FFmpegPostProcessor(PostProcessor):
|
|||
location = os.path.dirname(os.path.abspath(location))
|
||||
if basename in ('ffmpeg', 'ffprobe'):
|
||||
prefer_ffmpeg = True
|
||||
self._paths = dict(
|
||||
(p, p if location is None else os.path.join(location, p))
|
||||
for p in programs)
|
||||
self._versions = dict(
|
||||
x for x in (
|
||||
(p, get_ffmpeg_version(self._paths[p])) for p in programs)
|
||||
if x[1] is not None)
|
||||
|
||||
self._paths = dict(
|
||||
(p, os.path.join(location, p)) for p in programs)
|
||||
self._versions = dict(
|
||||
(p, get_ffmpeg_version(self._paths[p])) for p in programs)
|
||||
if self._versions is None:
|
||||
self._versions = dict(
|
||||
(p, get_ffmpeg_version(p)) for p in programs)
|
||||
self._paths = dict((p, p) for p in programs)
|
||||
|
||||
if prefer_ffmpeg is False:
|
||||
prefs = ('avconv', 'ffmpeg')
|
||||
else:
|
||||
prefs = ('ffmpeg', 'avconv')
|
||||
for p in prefs:
|
||||
if self._versions[p]:
|
||||
self.basename = p
|
||||
break
|
||||
|
||||
if prefer_ffmpeg is False:
|
||||
prefs = ('avprobe', 'ffprobe')
|
||||
else:
|
||||
prefs = ('ffprobe', 'avprobe')
|
||||
for p in prefs:
|
||||
if self._versions[p]:
|
||||
self.probe_basename = p
|
||||
break
|
||||
basenames = [None, None]
|
||||
for i, progs in enumerate((convs, probes)):
|
||||
for p in progs[::-1 if prefer_ffmpeg is False else 1]:
|
||||
if self._versions.get(p):
|
||||
basenames[i] = p
|
||||
break
|
||||
self.basename, self.probe_basename = basenames
|
||||
|
||||
@property
|
||||
def available(self):
|
||||
|
|
|
@ -45,6 +45,7 @@ from .compat import (
|
|||
compat_casefold,
|
||||
compat_chr,
|
||||
compat_collections_abc,
|
||||
compat_contextlib_suppress,
|
||||
compat_cookiejar,
|
||||
compat_ctypes_WINFUNCTYPE,
|
||||
compat_datetime_timedelta_total_seconds,
|
||||
|
@ -1855,25 +1856,18 @@ def write_json_file(obj, fn):
|
|||
try:
|
||||
with tf:
|
||||
json.dump(obj, tf)
|
||||
if sys.platform == 'win32':
|
||||
# Need to remove existing file on Windows, else os.rename raises
|
||||
# WindowsError or FileExistsError.
|
||||
try:
|
||||
with compat_contextlib_suppress(OSError):
|
||||
if sys.platform == 'win32':
|
||||
# Need to remove existing file on Windows, else os.rename raises
|
||||
# WindowsError or FileExistsError.
|
||||
os.unlink(fn)
|
||||
except OSError:
|
||||
pass
|
||||
try:
|
||||
mask = os.umask(0)
|
||||
os.umask(mask)
|
||||
os.chmod(tf.name, 0o666 & ~mask)
|
||||
except OSError:
|
||||
pass
|
||||
os.rename(tf.name, fn)
|
||||
except Exception:
|
||||
try:
|
||||
with compat_contextlib_suppress(OSError):
|
||||
os.remove(tf.name)
|
||||
except OSError:
|
||||
pass
|
||||
raise
|
||||
|
||||
|
||||
|
@ -2033,14 +2027,13 @@ def extract_attributes(html_element):
|
|||
NB HTMLParser is stricter in Python 2.6 & 3.2 than in later versions,
|
||||
but the cases in the unit test will work for all of 2.6, 2.7, 3.2-3.5.
|
||||
"""
|
||||
parser = HTMLAttributeParser()
|
||||
try:
|
||||
parser.feed(html_element)
|
||||
parser.close()
|
||||
# Older Python may throw HTMLParseError in case of malformed HTML
|
||||
except compat_HTMLParseError:
|
||||
pass
|
||||
return parser.attrs
|
||||
ret = None
|
||||
# Older Python may throw HTMLParseError in case of malformed HTML (and on .close()!)
|
||||
with compat_contextlib_suppress(compat_HTMLParseError):
|
||||
with contextlib.closing(HTMLAttributeParser()) as parser:
|
||||
parser.feed(html_element)
|
||||
ret = parser.attrs
|
||||
return ret or {}
|
||||
|
||||
|
||||
def clean_html(html):
|
||||
|
@ -2182,8 +2175,28 @@ def sanitize_url(url):
|
|||
return url
|
||||
|
||||
|
||||
def extract_basic_auth(url):
|
||||
parts = compat_urllib_parse.urlsplit(url)
|
||||
if parts.username is None:
|
||||
return url, None
|
||||
url = compat_urllib_parse.urlunsplit(parts._replace(netloc=(
|
||||
parts.hostname if parts.port is None
|
||||
else '%s:%d' % (parts.hostname, parts.port))))
|
||||
auth_payload = base64.b64encode(
|
||||
('%s:%s' % (parts.username, parts.password or '')).encode('utf-8'))
|
||||
return url, 'Basic {0}'.format(auth_payload.decode('ascii'))
|
||||
|
||||
|
||||
def sanitized_Request(url, *args, **kwargs):
|
||||
return compat_urllib_request.Request(escape_url(sanitize_url(url)), *args, **kwargs)
|
||||
url, auth_header = extract_basic_auth(escape_url(sanitize_url(url)))
|
||||
if auth_header is not None:
|
||||
headers = args[1] if len(args) > 1 else kwargs.get('headers')
|
||||
headers = headers or {}
|
||||
headers['Authorization'] = auth_header
|
||||
if len(args) <= 1 and kwargs.get('headers') is None:
|
||||
kwargs['headers'] = headers
|
||||
kwargs = compat_kwargs(kwargs)
|
||||
return compat_urllib_request.Request(url, *args, **kwargs)
|
||||
|
||||
|
||||
def expand_path(s):
|
||||
|
@ -2221,7 +2234,8 @@ def _htmlentity_transform(entity_with_semicolon):
|
|||
numstr = '0%s' % numstr
|
||||
else:
|
||||
base = 10
|
||||
# See https://github.com/ytdl-org/youtube-dl/issues/7518
|
||||
# See https://github.com/ytdl-org/youtube-dl/issues/7518\
|
||||
# Also, weirdly, compat_contextlib_suppress fails here in 2.6
|
||||
try:
|
||||
return compat_chr(int(numstr, base))
|
||||
except ValueError:
|
||||
|
@ -2328,11 +2342,9 @@ def make_HTTPS_handler(params, **kwargs):
|
|||
# Some servers may (wrongly) reject requests if ALPN extension is not sent. See:
|
||||
# https://github.com/python/cpython/issues/85140
|
||||
# https://github.com/yt-dlp/yt-dlp/issues/3878
|
||||
try:
|
||||
with compat_contextlib_suppress(AttributeError, NotImplementedError):
|
||||
# fails for Python < 2.7.10, not ssl.HAS_ALPN
|
||||
ctx.set_alpn_protocols(ALPN_PROTOCOLS)
|
||||
except (AttributeError, NotImplementedError):
|
||||
# Python < 2.7.10, not ssl.HAS_ALPN
|
||||
pass
|
||||
|
||||
opts_no_check_certificate = params.get('nocheckcertificate', False)
|
||||
if hasattr(ssl, 'create_default_context'): # Python >= 3.4 or 2.7.9
|
||||
|
@ -2342,12 +2354,10 @@ def make_HTTPS_handler(params, **kwargs):
|
|||
context.check_hostname = False
|
||||
context.verify_mode = ssl.CERT_NONE
|
||||
|
||||
try:
|
||||
with compat_contextlib_suppress(TypeError):
|
||||
# Fails with Python 2.7.8 (create_default_context present
|
||||
# but HTTPSHandler has no context=)
|
||||
return YoutubeDLHTTPSHandler(params, context=context, **kwargs)
|
||||
except TypeError:
|
||||
# Python 2.7.8
|
||||
# (create_default_context present but HTTPSHandler has no context=)
|
||||
pass
|
||||
|
||||
if sys.version_info < (3, 2):
|
||||
return YoutubeDLHTTPSHandler(params, **kwargs)
|
||||
|
@ -2361,15 +2371,24 @@ def make_HTTPS_handler(params, **kwargs):
|
|||
return YoutubeDLHTTPSHandler(params, context=context, **kwargs)
|
||||
|
||||
|
||||
def bug_reports_message():
|
||||
def bug_reports_message(before=';'):
|
||||
if ytdl_is_updateable():
|
||||
update_cmd = 'type youtube-dl -U to update'
|
||||
else:
|
||||
update_cmd = 'see https://yt-dl.org/update on how to update'
|
||||
msg = '; please report this issue on https://yt-dl.org/bug .'
|
||||
msg += ' Make sure you are using the latest version; %s.' % update_cmd
|
||||
msg += ' Be sure to call youtube-dl with the --verbose flag and include its complete output.'
|
||||
return msg
|
||||
update_cmd = 'see https://github.com/ytdl-org/youtube-dl/#user-content-installation on how to update'
|
||||
|
||||
msg = (
|
||||
'please report this issue on https://github.com/ytdl-org/youtube-dl/issues ,'
|
||||
' using the appropriate issue template.'
|
||||
' Make sure you are using the latest version; %s.'
|
||||
' Be sure to call youtube-dl with the --verbose option and include the complete output.'
|
||||
) % update_cmd
|
||||
|
||||
before = (before or '').rstrip()
|
||||
if not before or before.endswith(('.', '!', '?')):
|
||||
msg = msg[0].title() + msg[1:]
|
||||
|
||||
return (before + ' ' if before else '') + msg
|
||||
|
||||
|
||||
class YoutubeDLError(Exception):
|
||||
|
@ -3156,12 +3175,10 @@ def parse_iso8601(date_str, delimiter='T', timezone=None):
|
|||
if timezone is None:
|
||||
timezone, date_str = extract_timezone(date_str)
|
||||
|
||||
try:
|
||||
with compat_contextlib_suppress(ValueError):
|
||||
date_format = '%Y-%m-%d{0}%H:%M:%S'.format(delimiter)
|
||||
dt = datetime.datetime.strptime(date_str, date_format) - timezone
|
||||
return calendar.timegm(dt.timetuple())
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
|
||||
def date_formats(day_first=True):
|
||||
|
@ -3181,17 +3198,13 @@ def unified_strdate(date_str, day_first=True):
|
|||
_, date_str = extract_timezone(date_str)
|
||||
|
||||
for expression in date_formats(day_first):
|
||||
try:
|
||||
with compat_contextlib_suppress(ValueError):
|
||||
upload_date = datetime.datetime.strptime(date_str, expression).strftime('%Y%m%d')
|
||||
except ValueError:
|
||||
pass
|
||||
if upload_date is None:
|
||||
timetuple = email.utils.parsedate_tz(date_str)
|
||||
if timetuple:
|
||||
try:
|
||||
with compat_contextlib_suppress(ValueError):
|
||||
upload_date = datetime.datetime(*timetuple[:6]).strftime('%Y%m%d')
|
||||
except ValueError:
|
||||
pass
|
||||
if upload_date is not None:
|
||||
return compat_str(upload_date)
|
||||
|
||||
|
@ -3220,11 +3233,9 @@ def unified_timestamp(date_str, day_first=True):
|
|||
date_str = m.group(1)
|
||||
|
||||
for expression in date_formats(day_first):
|
||||
try:
|
||||
with compat_contextlib_suppress(ValueError):
|
||||
dt = datetime.datetime.strptime(date_str, expression) - timezone + datetime.timedelta(hours=pm_delta)
|
||||
return calendar.timegm(dt.timetuple())
|
||||
except ValueError:
|
||||
pass
|
||||
timetuple = email.utils.parsedate_tz(date_str)
|
||||
if timetuple:
|
||||
return calendar.timegm(timetuple) + pm_delta * 3600 - compat_datetime_timedelta_total_seconds(timezone)
|
||||
|
@ -3832,14 +3843,15 @@ class PUTRequest(compat_urllib_request.Request):
|
|||
return 'PUT'
|
||||
|
||||
|
||||
def int_or_none(v, scale=1, default=None, get_attr=None, invscale=1):
|
||||
def int_or_none(v, scale=1, default=None, get_attr=None, invscale=1, base=None):
|
||||
if get_attr:
|
||||
if v is not None:
|
||||
v = getattr(v, get_attr, None)
|
||||
if v in (None, ''):
|
||||
return default
|
||||
try:
|
||||
return int(v) * invscale // scale
|
||||
# like int, raise if base is specified and v is not a string
|
||||
return (int(v) if base is None else int(v, base=base)) * invscale // scale
|
||||
except (ValueError, TypeError, OverflowError):
|
||||
return default
|
||||
|
||||
|
|
Loading…
Reference in New Issue