From f870544302f75bee0d96f6a8623c8ff270beca89 Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Mon, 13 Jul 2015 07:41:38 -0500
Subject: [PATCH 001/415] Add support for democracynow.org
Supports downloading clips or entire shows. Subtitle support
---
youtube_dl/extractor/__init__.py | 1 +
youtube_dl/extractor/democracynow.py | 100 +++++++++++++++++++++++++++
2 files changed, 101 insertions(+)
create mode 100644 youtube_dl/extractor/democracynow.py
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index cbaa07391..5cc03b875 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -112,6 +112,7 @@ from .daum import DaumIE
from .dbtv import DBTVIE
from .dctp import DctpTvIE
from .deezer import DeezerPlaylistIE
+from .democracynow import DemocracynowIE
from .dfb import DFBIE
from .dhm import DHMIE
from .dotsub import DotsubIE
diff --git a/youtube_dl/extractor/democracynow.py b/youtube_dl/extractor/democracynow.py
new file mode 100644
index 000000000..1c9b36052
--- /dev/null
+++ b/youtube_dl/extractor/democracynow.py
@@ -0,0 +1,100 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import json
+import time
+import hmac
+import hashlib
+import itertools
+import re
+from ..utils import (
+ ExtractorError,
+ int_or_none,
+ parse_age_limit,
+ parse_iso8601,
+)
+from ..compat import compat_urllib_request
+from .common import InfoExtractor
+
+
+class DemocracynowIE(InfoExtractor):
+ _VALID_URL = r'https?://(?:www\.)?democracynow.org/?(?P<id>[^\?]*)'
+ IE_NAME = 'democracynow'
+ _TESTS = [{
+ 'url': 'http://www.democracynow.org/shows/2015/7/3',
+ 'info_dict': {
+ 'id': '2015-0703-001',
+ 'ext': 'mp4',
+ 'title': 'July 03, 2015 - Democracy Now!',
+ 'description': 'A daily independent global news hour with Amy Goodman & Juan Gonz\xe1lez "What to the Slave is 4th of July?": James Earl Jones Reads Frederick Douglass\u2019 Historic Speech : "This Flag Comes Down Today": Bree Newsome Scales SC Capitol Flagpole, Takes Down Confederate Flag : "We Shall Overcome": Remembering Folk Icon, Activist Pete Seeger in His Own Words & Songs',
+ 'uploader': 'Democracy Now',
+ 'upload_date': None,
+ },
+ },{
+ 'url': 'http://www.democracynow.org/2015/7/3/this_flag_comes_down_today_bree',
+ 'info_dict': {
+ 'id': '2015-0703-001',
+ 'ext': 'mp4',
+ 'title': '"This Flag Comes Down Today": Bree Newsome Scales SC Capitol Flagpole, Takes Down Confederate Flag',
+ 'description': 'md5:4d2bc4f0d29f5553c2210a4bc7761a21',
+ 'uploader': 'Democracy Now',
+ 'upload_date': None,
+ },
+
+ }]
+
+ def _real_extract(self, url):
+ display_id = self._match_id(url)
+ base_host = re.search(r'^(.+?://[^/]+)', url).group(1)
+ if display_id == '':
+ display_id = 'home'
+ webpage = self._download_webpage(url, display_id)
+ re_desc = re.search(r'<meta property=.og:description. content=(["\'])(.+?)\1',webpage,re.DOTALL)
+ description = re_desc.group(2) if re_desc else ''
+
+ jstr = self._search_regex(r'({.+?"related_video_xml".+?})', webpage, 'json', default=None)
+ js = self._parse_json(jstr, display_id)
+ video_id = None
+ formats = []
+ subtitles = {}
+ for key in ('caption_file','.......'):
+ # ....... = pending vtt support that doesn't clobber srt 'chapter_file':
+ url = js.get(key,'')
+ if url == '' or url == None:
+ continue
+ if not re.match(r'^https?://',url):
+ url = base_host + url
+ ext = re.search(r'\.([^\.]+)$',url).group(1)
+ subtitles['eng'] = [{
+ 'ext': ext,
+ 'url': url,
+ }]
+ for key in ('file', 'audio'):
+ url = js.get(key,'')
+ if url == '' or url == None:
+ continue
+ if not re.match(r'^https?://',url):
+ url = base_host + url
+ purl = re.search(r'/(?P<dir>[^/]+)/(?:dn)?(?P<fn>[^/]+?)\.(?P<ext>[^\.\?]+)(?P<hasparams>\?|$)',url)
+ if video_id == None:
+ video_id = purl.group('fn')
+ if js.get('start') != None:
+ url += '&' if purl.group('hasparams') == '?' else '?'
+ url = url + 'start='+str(js.get('start'))
+ formats.append({
+ 'format_id': purl.group('dir'),
+ 'ext': purl.group('ext'),
+ 'url': url,
+ })
+ self._sort_formats(formats)
+ ret = {
+ 'id': video_id,
+ 'title': js.get('title'),
+ 'description': description,
+ 'uploader': 'Democracy Now',
+# 'thumbnails': thumbnails,
+ 'subtitles': subtitles,
+ 'formats': formats,
+ }
+ return ret
+#
\ No newline at end of file
From eb08081330f5ef52d66140589137ae1bb05eee5f Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Fri, 17 Jul 2015 02:57:08 -0500
Subject: [PATCH 002/415] democracynow: correct syntax
---
youtube_dl/extractor/democracynow.py | 43 +++++++++-------------------
1 file changed, 14 insertions(+), 29 deletions(-)
diff --git a/youtube_dl/extractor/democracynow.py b/youtube_dl/extractor/democracynow.py
index 1c9b36052..973bb437b 100644
--- a/youtube_dl/extractor/democracynow.py
+++ b/youtube_dl/extractor/democracynow.py
@@ -1,19 +1,7 @@
# coding: utf-8
from __future__ import unicode_literals
-import json
-import time
-import hmac
-import hashlib
-import itertools
import re
-from ..utils import (
- ExtractorError,
- int_or_none,
- parse_age_limit,
- parse_iso8601,
-)
-from ..compat import compat_urllib_request
from .common import InfoExtractor
@@ -30,7 +18,7 @@ class DemocracynowIE(InfoExtractor):
'uploader': 'Democracy Now',
'upload_date': None,
},
- },{
+ }, {
'url': 'http://www.democracynow.org/2015/7/3/this_flag_comes_down_today_bree',
'info_dict': {
'id': '2015-0703-001',
@@ -40,7 +28,6 @@ class DemocracynowIE(InfoExtractor):
'uploader': 'Democracy Now',
'upload_date': None,
},
-
}]
def _real_extract(self, url):
@@ -49,7 +36,7 @@ class DemocracynowIE(InfoExtractor):
if display_id == '':
display_id = 'home'
webpage = self._download_webpage(url, display_id)
- re_desc = re.search(r'<meta property=.og:description. content=(["\'])(.+?)\1',webpage,re.DOTALL)
+ re_desc = re.search(r'<meta property=.og:description. content=(["\'])(.+?)\1', webpage, re.DOTALL)
description = re_desc.group(2) if re_desc else ''
jstr = self._search_regex(r'({.+?"related_video_xml".+?})', webpage, 'json', default=None)
@@ -57,30 +44,30 @@ class DemocracynowIE(InfoExtractor):
video_id = None
formats = []
subtitles = {}
- for key in ('caption_file','.......'):
+ for key in ('caption_file', '.......'):
# ....... = pending vtt support that doesn't clobber srt 'chapter_file':
- url = js.get(key,'')
- if url == '' or url == None:
+ url = js.get(key, '')
+ if url == '' or url is None:
continue
- if not re.match(r'^https?://',url):
+ if not re.match(r'^https?://', url):
url = base_host + url
- ext = re.search(r'\.([^\.]+)$',url).group(1)
+ ext = re.search(r'\.([^\.]+)$', url).group(1)
subtitles['eng'] = [{
'ext': ext,
'url': url,
}]
for key in ('file', 'audio'):
- url = js.get(key,'')
- if url == '' or url == None:
+ url = js.get(key, '')
+ if url == '' or url is None:
continue
- if not re.match(r'^https?://',url):
+ if not re.match(r'^https?://', url):
url = base_host + url
- purl = re.search(r'/(?P<dir>[^/]+)/(?:dn)?(?P<fn>[^/]+?)\.(?P<ext>[^\.\?]+)(?P<hasparams>\?|$)',url)
- if video_id == None:
+ purl = re.search(r'/(?P<dir>[^/]+)/(?:dn)?(?P<fn>[^/]+?)\.(?P<ext>[^\.\?]+)(?P<hasparams>\?|$)', url)
+ if video_id is None:
video_id = purl.group('fn')
- if js.get('start') != None:
+ if js.get('start') is not None:
url += '&' if purl.group('hasparams') == '?' else '?'
- url = url + 'start='+str(js.get('start'))
+ url = url + 'start=' + str(js.get('start'))
formats.append({
'format_id': purl.group('dir'),
'ext': purl.group('ext'),
@@ -92,9 +79,7 @@ class DemocracynowIE(InfoExtractor):
'title': js.get('title'),
'description': description,
'uploader': 'Democracy Now',
-# 'thumbnails': thumbnails,
'subtitles': subtitles,
'formats': formats,
}
return ret
-#
\ No newline at end of file
From f57f84f606b246db4f102fc5bc55e64e4f7a3d60 Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Tue, 21 Jul 2015 16:38:40 -0500
Subject: [PATCH 003/415] Twitter: get and describe video from status urls
---
youtube_dl/extractor/twitter.py | 44 +++++++++++++++++++++++++++++++++
1 file changed, 44 insertions(+)
diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index 1aaa06305..a65252cc6 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -70,3 +70,47 @@ class TwitterCardIE(InfoExtractor):
'duration': duration,
'formats': formats,
}
+
+
+class TwitterIE(TwitterCardIE):
+ _VALID_URL = r'https?://(?:www|m|mobile)?\.?twitter\.com/(?P<id>[^/]+/status/\d+)'
+
+ _TESTS = [{
+ 'url': 'https://m.twitter.com/thereaIbanksy/status/614301758345490432',
+ 'md5': '8bbccb487bd7a31349b775915fcd412f',
+ 'info_dict': {
+ 'id': '614301758345490432',
+ 'ext': 'mp4',
+ 'title': 'thereaIbanksy - This time lapse is so pretty \U0001f60d\U0001f60d',
+ 'thumbnail': 're:^https?://.*\.jpg',
+ 'duration': 29.5,
+ 'description': 'banksy on Twitter: "This time lapse is so pretty \U0001f60d\U0001f60d http://t.co/QB8DDbqiR1"',
+ 'uploader': 'banksy',
+ 'uploader_id': 'thereaIbanksy',
+ },
+ }]
+
+ def _real_extract(self, url):
+ id = self._match_id(url)
+ username, twid = re.match(r'([^/]+)/status/(\d+)', id).groups()
+ name = username
+ url = re.sub(r'https?://(m|mobile)\.', 'https://', url)
+ webpage = self._download_webpage(url, 'tweet: ' + url)
+ description = unescapeHTML(self._search_regex('<title>\s*(.+?)\s*</title>', webpage, 'title'))
+ title = description.replace('\n', ' ')
+ splitdesc = re.match(r'^(.+?)\s*on Twitter:\s* "(.+?)"$', title)
+ if splitdesc:
+ name, title = splitdesc.groups()
+ title = re.sub(r'\s*https?://[^ ]+', '', title) # strip 'https -_t.co_BJYgOjSeGA' junk from filenames
+ card_id = self._search_regex(r'["\']/i/cards/tfw/v1/(\d+)', webpage, '/i/card/...')
+ card_url = 'https://twitter.com/i/cards/tfw/v1/' + card_id
+ return {
+ '_type': 'url_transparent',
+ 'ie_key': 'TwitterCard',
+ 'uploader_id': username,
+ 'uploader': name,
+ 'url': card_url,
+ 'webpage_url': url,
+ 'description': description,
+ 'title': username + ' - ' + title,
+ }
From c3dea3f878133f3cbdad9e548609d3077572af66 Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Tue, 21 Jul 2015 16:45:36 -0500
Subject: [PATCH 004/415] Twittercard: support vmapurl method
---
youtube_dl/extractor/twitter.py | 47 ++++++++++++++++++++++++++-------
1 file changed, 37 insertions(+), 10 deletions(-)
diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index a65252cc6..1dd43ff3c 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -12,17 +12,30 @@ from ..utils import (
class TwitterCardIE(InfoExtractor):
_VALID_URL = r'https?://(?:www\.)?twitter\.com/i/cards/tfw/v1/(?P<id>\d+)'
- _TEST = {
- 'url': 'https://twitter.com/i/cards/tfw/v1/560070183650213889',
- 'md5': 'a74f50b310c83170319ba16de6955192',
- 'info_dict': {
- 'id': '560070183650213889',
- 'ext': 'mp4',
- 'title': 'TwitterCard',
- 'thumbnail': 're:^https?://.*\.jpg$',
- 'duration': 30.033,
+ _TESTS = [
+ {
+ 'url': 'https://twitter.com/i/cards/tfw/v1/560070183650213889',
+ 'md5': 'a74f50b310c83170319ba16de6955192',
+ 'info_dict': {
+ 'id': '560070183650213889',
+ 'ext': 'mp4',
+ 'title': 'TwitterCard',
+ 'thumbnail': 're:^https?://.*\.jpg$',
+ 'duration': 30.033,
+ }
},
- }
+ {
+ 'url': 'https://twitter.com/i/cards/tfw/v1/623160978427936768',
+ 'md5': '7ee2a553b63d1bccba97fbed97d9e1c8',
+ 'info_dict': {
+ 'id': '623160978427936768',
+ 'ext': 'mp4',
+ 'title': 'TwitterCard',
+ 'thumbnail': 're:^https?://.*\.jpg',
+ 'duration': 80.155,
+ },
+ }
+ ]
def _real_extract(self, url):
video_id = self._match_id(url)
@@ -44,6 +57,20 @@ class TwitterCardIE(InfoExtractor):
unescapeHTML(self._search_regex(
r'data-player-config="([^"]+)"', webpage, 'data player config')),
video_id)
+ if 'playlist' not in config:
+ if 'vmapUrl' in config:
+ webpage = self._download_webpage(config['vmapUrl'], video_id + ' (xml)')
+ video_url = self._search_regex(
+ r'<MediaFile>\s*<!\[CDATA\[(https?://.+?)\]\]>', webpage, 'data player config (xml)')
+ f = {
+ 'url': video_url,
+ }
+ ext = re.search(r'\.([a-z0-9]{2,4})(\?.+)?$', video_url)
+ if ext:
+ f['ext'] = ext.group(1)
+ formats.append(f)
+ break # same video regardless of UA
+ continue
video_url = config['playlist'][0]['source']
From 9e7e0dffd5e3e3c959e8d99a5e236b9099886fe9 Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Tue, 21 Jul 2015 16:56:35 -0500
Subject: [PATCH 005/415] Actually add the extractor
---
youtube_dl/extractor/__init__.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 50da08830..5c03bf8e8 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -651,7 +651,7 @@ from .twitch import (
TwitchBookmarksIE,
TwitchStreamIE,
)
-from .twitter import TwitterCardIE
+from .twitter import TwitterCardIE, TwitterIE
from .ubu import UbuIE
from .udemy import (
UdemyIE,
From 689fb748ee1ba8e61f99d21a3bcb1bc83b708649 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 11 Sep 2015 04:44:17 +0100
Subject: [PATCH 006/415] [utlis] add extract_attributes for extracting html
tags attributes
---
youtube_dl/utils.py | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 206dd56bc..bcebf9cc5 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -248,6 +248,14 @@ def get_element_by_attribute(attribute, value, html):
return unescapeHTML(res)
+def extract_attributes(attributes_str, attributes_regex=r'(?s)\s*([^\s=]+)\s*=\s*["\']([^"\']+)["\']'):
+ attributes = re.findall(attributes_regex, attributes_str)
+ attributes_dict = {}
+ if attributes:
+ attributes_dict = {attribute_name: attribute_value for (attribute_name, attribute_value) in attributes}
+ return attributes_dict
+
+
def clean_html(html):
"""Clean an HTML snippet into a readable string"""
From ed1269000f24a6ddc683a295ff402ef3ded5c4fb Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 11 Sep 2015 04:46:21 +0100
Subject: [PATCH 007/415] [brightcove] add support for brightcove in page
embed(fixes #6824)
---
youtube_dl/extractor/__init__.py | 5 +-
youtube_dl/extractor/brightcove.py | 92 ++++++++++++++++++++++++++++++
youtube_dl/extractor/generic.py | 21 ++++++-
3 files changed, 116 insertions(+), 2 deletions(-)
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 57f55b479..fcd9edec3 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -59,7 +59,10 @@ from .bloomberg import BloombergIE
from .bpb import BpbIE
from .br import BRIE
from .breakcom import BreakIE
-from .brightcove import BrightcoveIE
+from .brightcove import (
+ BrightcoveIE,
+ BrightcoveInPageEmbedIE,
+)
from .buzzfeed import BuzzFeedIE
from .byutv import BYUtvIE
from .c56 import C56IE
diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 4721c2293..a07c0888f 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -22,6 +22,10 @@ from ..utils import (
fix_xml_ampersands,
unescapeHTML,
unsmuggle_url,
+ js_to_json,
+ int_or_none,
+ parse_iso8601,
+ extract_attributes,
)
@@ -346,3 +350,91 @@ class BrightcoveIE(InfoExtractor):
if 'url' not in info and not info.get('formats'):
raise ExtractorError('Unable to extract video url for %s' % info['id'])
return info
+
+
+class BrightcoveInPageEmbedIE(InfoExtractor):
+ _VALID_URL = r'https?://players\.brightcove\.net/(?P<account_id>\d+)/([a-z0-9-]+)_([a-z]+)/index.html?.*videoId=(?P<video_id>\d+)'
+ TEST = {
+ 'url': 'http://players.brightcove.net/929656772001/e41d32dc-ec74-459e-a845-6c69f7b724ea_default/index.html?videoId=4463358922001',
+ 'info_dict': {
+ 'id': '4463358922001',
+ 'ext': 'flv',
+ 'title': 'Meet the man behind Popcorn Time',
+ 'description': 'md5:a950cc4285c43e44d763d036710cd9cd',
+ 'duration': 165768,
+ }
+ }
+
+ @staticmethod
+ def _extract_url(webpage):
+ video_attributes = re.search(r'(?s)<video([^>]*)>.*?</(?:video|audio)>', webpage)
+ if video_attributes:
+ video_attributes = extract_attributes(video_attributes.group(), r'(?s)\s*data-(account|video-id|playlist-id|policy-key|player|embed)\s*=\s*["\']([^"\']+)["\']')
+ account_id = video_attributes.get('account')
+ player_id = video_attributes.get('player')
+ embed = video_attributes.get('embed')
+ video_id = video_attributes.get('video-id')
+ if account_id and player_id and embed and video_id:
+ return 'http://players.brightcove.net/%s/%s_%s/index.html?videoId=%s' % (account_id, player_id, embed, video_id)
+ return None
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ account_id, player_id, embed, video_id = mobj.groups()
+
+ webpage = self._download_webpage('http://players.brightcove.net/%s/%s_%s/index.min.js' % (account_id, player_id, embed), video_id)
+
+ catalog = self._parse_json(
+ js_to_json(
+ self._search_regex(
+ r'catalog\(({[^}]+})\);',
+ webpage,
+ 'catalog'
+ )
+ ),
+ video_id
+ )
+ policy_key = catalog['policyKey']
+
+ req = compat_urllib_request.Request(
+ 'https://edge.api.brightcove.com/playback/v1/accounts/%s/videos/%s' % (account_id, video_id),
+ headers={'Accept': 'application/json;pk=%s' % policy_key})
+ json_data = self._download_json(req, video_id)
+
+ title = json_data['name']
+ description = json_data.get('description')
+ thumbnail = json_data.get('name')
+ timestamp = parse_iso8601(json_data.get('published_at'))
+ duration = int_or_none(json_data.get('duration'))
+
+ formats = []
+ for source in json_data.get('sources'):
+ source_type = source.get('type')
+ if source_type == 'application/x-mpegURL':
+ formats.extend(self._extract_m3u8_formats(source.get('src'), video_id))
+ else:
+ src = source.get('src')
+ if src:
+ formats.append({
+ 'url': src,
+ 'abr': source.get('avg_bitrate'),
+ 'width': int_or_none(source.get('width')),
+ 'height': int_or_none(source.get('height')),
+ 'filesize': source.get('size'),
+ 'container': source.get('container'),
+ 'vcodec': source.get('container'),
+ })
+ else:
+ formats.extend(self._extract_f4m_formats(source.get('streaming_src'), video_id))
+
+ self._sort_formats(formats)
+
+ return {
+ 'id': video_id,
+ 'title': title,
+ 'description': description,
+ 'thumbnail': thumbnail,
+ 'timestamp': timestamp,
+ 'duration': duration,
+ 'formats': formats,
+ }
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index ec748ed9f..7a3a7f66b 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -29,7 +29,10 @@ from ..utils import (
url_basename,
xpath_text,
)
-from .brightcove import BrightcoveIE
+from .brightcove import (
+ BrightcoveIE,
+ BrightcoveInPageEmbedIE,
+)
from .nbc import NBCSportsVPlayerIE
from .ooyala import OoyalaIE
from .rutv import RUTVIE
@@ -1012,6 +1015,17 @@ class GenericIE(InfoExtractor):
'ext': 'mp4',
'title': 'cinemasnob',
},
+ },
+ # BrightcoveInPageEmbed embed
+ {
+ 'url': 'http://www.geekandsundry.com/tabletop-bonus-wils-final-thoughts-on-dread/',
+ 'info_dict': {
+ 'id': '4238694884001',
+ 'ext': 'flv',
+ 'title': 'Tabletop: Dread, Last Thoughts',
+ 'description': 'Tabletop: Dread, Last Thoughts',
+ 'duration': 51690,
+ },
}
]
@@ -1288,6 +1302,11 @@ class GenericIE(InfoExtractor):
'entries': entries,
}
+ # Look for Brightcove In Page Embed:
+ brightcove_in_page_embed_url = BrightcoveInPageEmbedIE._extract_url(webpage)
+ if brightcove_in_page_embed_url:
+ return self.url_result(brightcove_in_page_embed_url, 'BrightcoveInPageEmbed')
+
# Look for embedded rtl.nl player
matches = re.findall(
r'<iframe[^>]+?src="((?:https?:)?//(?:www\.)?rtl\.nl/system/videoplayer/[^"]+(?:video_)?embed[^"]+)"',
From 53407e3f383ed80c67db9e06b8c3480257aa3184 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 23 Sep 2015 14:02:13 +0100
Subject: [PATCH 008/415] [brightcove] fix streaming_src extraction
---
youtube_dl/extractor/brightcove.py | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index a07c0888f..e4a7befee 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -413,7 +413,7 @@ class BrightcoveInPageEmbedIE(InfoExtractor):
if source_type == 'application/x-mpegURL':
formats.extend(self._extract_m3u8_formats(source.get('src'), video_id))
else:
- src = source.get('src')
+ src = source.get('src') or source.get('streaming_src')
if src:
formats.append({
'url': src,
@@ -424,8 +424,6 @@ class BrightcoveInPageEmbedIE(InfoExtractor):
'container': source.get('container'),
'vcodec': source.get('container'),
})
- else:
- formats.extend(self._extract_f4m_formats(source.get('streaming_src'), video_id))
self._sort_formats(formats)
From c01e1a96aa964ef6d5f0bf7675dbe34096b1d2c8 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Wed, 30 Sep 2015 11:20:43 +0100
Subject: [PATCH 009/415] [brightcove] fix test and fields extraction
---
youtube_dl/extractor/brightcove.py | 18 ++++++++++++------
1 file changed, 12 insertions(+), 6 deletions(-)
diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index e4a7befee..b41cee91b 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -354,14 +354,18 @@ class BrightcoveIE(InfoExtractor):
class BrightcoveInPageEmbedIE(InfoExtractor):
_VALID_URL = r'https?://players\.brightcove\.net/(?P<account_id>\d+)/([a-z0-9-]+)_([a-z]+)/index.html?.*videoId=(?P<video_id>\d+)'
- TEST = {
+ _TEST = {
'url': 'http://players.brightcove.net/929656772001/e41d32dc-ec74-459e-a845-6c69f7b724ea_default/index.html?videoId=4463358922001',
+ 'md5': 'c8100925723840d4b0d243f7025703be',
'info_dict': {
'id': '4463358922001',
- 'ext': 'flv',
+ 'ext': 'mp4',
'title': 'Meet the man behind Popcorn Time',
- 'description': 'md5:a950cc4285c43e44d763d036710cd9cd',
+ 'description': 'md5:eac376a4fe366edc70279bfb681aea16',
+ 'timestamp': 1441391203,
+ 'upload_date': '20150904',
'duration': 165768,
+ 'uploader_id': '929656772001',
}
}
@@ -403,7 +407,7 @@ class BrightcoveInPageEmbedIE(InfoExtractor):
title = json_data['name']
description = json_data.get('description')
- thumbnail = json_data.get('name')
+ thumbnail = json_data.get('thumbnail')
timestamp = parse_iso8601(json_data.get('published_at'))
duration = int_or_none(json_data.get('duration'))
@@ -417,12 +421,13 @@ class BrightcoveInPageEmbedIE(InfoExtractor):
if src:
formats.append({
'url': src,
- 'abr': source.get('avg_bitrate'),
+ 'tbr': source.get('avg_bitrate'),
'width': int_or_none(source.get('width')),
'height': int_or_none(source.get('height')),
'filesize': source.get('size'),
'container': source.get('container'),
- 'vcodec': source.get('container'),
+ 'vcodec': source.get('codec'),
+ 'ext': source.get('container').lower(),
})
self._sort_formats(formats)
@@ -435,4 +440,5 @@ class BrightcoveInPageEmbedIE(InfoExtractor):
'timestamp': timestamp,
'duration': duration,
'formats': formats,
+ 'uploader_id': account_id,
}
From 30787f7259c4e6a08f691cc691f14fa0c8fe4b87 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 3 Oct 2015 19:28:48 +0100
Subject: [PATCH 010/415] [cspan] correct the clip info extraction
---
youtube_dl/extractor/cspan.py | 58 ++++++++++++++++-------------------
1 file changed, 27 insertions(+), 31 deletions(-)
diff --git a/youtube_dl/extractor/cspan.py b/youtube_dl/extractor/cspan.py
index fbefd37d0..994e080d5 100644
--- a/youtube_dl/extractor/cspan.py
+++ b/youtube_dl/extractor/cspan.py
@@ -18,22 +18,21 @@ class CSpanIE(InfoExtractor):
IE_DESC = 'C-SPAN'
_TESTS = [{
'url': 'http://www.c-span.org/video/?313572-1/HolderonV',
- 'md5': '8e44ce11f0f725527daccc453f553eb0',
+ 'md5': '067803f994e049b455a58b16e5aab442',
'info_dict': {
'id': '315139',
'ext': 'mp4',
'title': 'Attorney General Eric Holder on Voting Rights Act Decision',
- 'description': 'Attorney General Eric Holder spoke to reporters following the Supreme Court decision in Shelby County v. Holder in which the court ruled that the preclearance provisions of the Voting Rights Act could not be enforced until Congress established new guidelines for review.',
+ 'description': 'Attorney General Eric Holder speaks to reporters following the Supreme Court decision in [Shelby County v. Holder], in which the court ruled that the preclearance provisions of the Voting Rights Act could not be enforced.',
},
'skip': 'Regularly fails on travis, for unknown reasons',
}, {
'url': 'http://www.c-span.org/video/?c4486943/cspan-international-health-care-models',
- # For whatever reason, the served video alternates between
- # two different ones
+ 'md5': '4eafd1e91a75d2b1e6a3cbd0995816a2',
'info_dict': {
- 'id': '340723',
+ 'id': 'c4486943',
'ext': 'mp4',
- 'title': 'International Health Care Models',
+ 'title': 'CSPAN - International Health Care Models',
'description': 'md5:7a985a2d595dba00af3d9c9f0783c967',
}
}, {
@@ -44,7 +43,7 @@ class CSpanIE(InfoExtractor):
'ext': 'mp4',
'title': 'General Motors Ignition Switch Recall',
'duration': 14848,
- 'description': 'md5:70c7c3b8fa63fa60d42772440596034c'
+ 'description': 'md5:118081aedd24bf1d3b68b3803344e7f3'
},
}, {
# Video from senate.gov
@@ -57,36 +56,33 @@ class CSpanIE(InfoExtractor):
}]
def _real_extract(self, url):
- mobj = re.match(self._VALID_URL, url)
- page_id = mobj.group('id')
- webpage = self._download_webpage(url, page_id)
- video_id = self._search_regex(r'progid=\'?([0-9]+)\'?>', webpage, 'video id')
+ video_id = self._match_id(url)
+ webpage = self._download_webpage(url, video_id)
+ matches = re.search(r'data-(prog|clip)id=\'([0-9]+)\'', webpage)
+ if matches:
+ video_type, video_id = matches.groups()
+ if video_type == 'prog':
+ video_type = 'program'
+ else:
+ senate_isvp_url = SenateISVPIE._search_iframe_url(webpage)
+ if senate_isvp_url:
+ title = self._og_search_title(webpage)
+ surl = smuggle_url(senate_isvp_url, {'force_title': title})
+ return self.url_result(surl, 'SenateISVP', video_id, title)
- description = self._html_search_regex(
- [
- # The full description
- r'<div class=\'expandable\'>(.*?)<a href=\'#\'',
- # If the description is small enough the other div is not
- # present, otherwise this is a stripped version
- r'<p class=\'initial\'>(.*?)</p>'
- ],
- webpage, 'description', flags=re.DOTALL, default=None)
-
- info_url = 'http://c-spanvideo.org/videoLibrary/assets/player/ajax-player.php?os=android&html5=program&id=' + video_id
- data = self._download_json(info_url, video_id)
+ data = self._download_json(
+ 'http://c-spanvideo.org/videoLibrary/assets/player/ajax-player.php?os=android&html5=%s&id=%s' % (video_type, video_id),
+ video_id)
doc = self._download_xml(
- 'http://www.c-span.org/common/services/flashXml.php?programid=' + video_id,
+ 'http://www.c-span.org/common/services/flashXml.php?%sid=%s' % (video_type, video_id),
video_id)
+ description = self._html_search_meta('description', webpage)
+
title = find_xpath_attr(doc, './/string', 'name', 'title').text
thumbnail = find_xpath_attr(doc, './/string', 'name', 'poster').text
- senate_isvp_url = SenateISVPIE._search_iframe_url(webpage)
- if senate_isvp_url:
- surl = smuggle_url(senate_isvp_url, {'force_title': title})
- return self.url_result(surl, 'SenateISVP', video_id, title)
-
files = data['video']['files']
try:
capfile = data['video']['capfile']['#text']
@@ -112,12 +108,12 @@ class CSpanIE(InfoExtractor):
if len(entries) == 1:
entry = dict(entries[0])
- entry['id'] = video_id
+ entry['id'] = 'c' + video_id if video_type == 'clip' else video_id
return entry
else:
return {
'_type': 'playlist',
'entries': entries,
'title': title,
- 'id': video_id,
+ 'id': 'c' + video_id if video_type == 'clip' else video_id,
}
From 41a7b00f183844e93ae2ba46fb4021f257f3ce79 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Lalinsk=C3=BD?= <lukas@oxygene.sk>
Date: Sat, 17 Oct 2015 18:18:40 +0200
Subject: [PATCH 011/415] [vimeo] Extract config URL from (new?) React-based
Vimeo's page
---
youtube_dl/extractor/vimeo.py | 9 ++++++++-
1 file changed, 8 insertions(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index fa1b22049..88e462a4d 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -286,7 +286,14 @@ class VimeoIE(VimeoBaseInfoExtractor):
try:
try:
config_url = self._html_search_regex(
- r' data-config-url="(.+?)"', webpage, 'config URL')
+ r' data-config-url="(.+?)"', webpage,
+ 'config URL', default=None)
+ if not config_url:
+ # New react-based page
+ vimeo_clip_page_config = self._search_regex(
+ r'vimeo\.clip_page_config\s*=\s*({.+?});', webpage,
+ 'vimeo clip page config')
+ config_url = self._parse_json(vimeo_clip_page_config, video_id)['player']['config_url']
config_json = self._download_webpage(config_url, video_id)
config = json.loads(config_json)
except RegexNotFoundError:
From dd8417526b13c541e6db8f4200e717b8922a1620 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 17 Oct 2015 22:48:14 +0600
Subject: [PATCH 012/415] [vimeo] Clarify new react+flux website fallback
---
youtube_dl/extractor/vimeo.py | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 88e462a4d..0f84656c0 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -289,11 +289,14 @@ class VimeoIE(VimeoBaseInfoExtractor):
r' data-config-url="(.+?)"', webpage,
'config URL', default=None)
if not config_url:
- # New react-based page
+ # Sometimes new react-based page is served instead of old one that require
+ # different config URL extraction approach (see
+ # https://github.com/rg3/youtube-dl/pull/7209)
vimeo_clip_page_config = self._search_regex(
r'vimeo\.clip_page_config\s*=\s*({.+?});', webpage,
'vimeo clip page config')
- config_url = self._parse_json(vimeo_clip_page_config, video_id)['player']['config_url']
+ config_url = self._parse_json(
+ vimeo_clip_page_config, video_id)['player']['config_url']
config_json = self._download_webpage(config_url, video_id)
config = json.loads(config_json)
except RegexNotFoundError:
From 59fe4824f80b7e266ea9918ae1b2e49a456b869f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Lalinsk=C3=BD?= <lukas@oxygene.sk>
Date: Sat, 17 Oct 2015 18:52:25 +0200
Subject: [PATCH 013/415] [vidme] Better error message for suspended vidme
videos
---
youtube_dl/extractor/vidme.py | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/youtube_dl/extractor/vidme.py b/youtube_dl/extractor/vidme.py
index 078d283b2..81dcaa231 100644
--- a/youtube_dl/extractor/vidme.py
+++ b/youtube_dl/extractor/vidme.py
@@ -114,6 +114,12 @@ class VidmeIE(InfoExtractor):
video = response['video']
+ if video.get('state') == 'user-disabled':
+ raise ExtractorError(
+ 'Vidme said: This video has been suspended either due to a copyright claim, '
+ 'or for violating the terms of use.',
+ expected=True)
+
formats = [{
'format_id': f.get('type'),
'url': f['uri'],
From 9eb31b265f65ec6b04a508702af1a6feddafb8fe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 17 Oct 2015 23:01:24 +0600
Subject: [PATCH 014/415] [vidme] Add user-disabled test
---
youtube_dl/extractor/vidme.py | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/youtube_dl/extractor/vidme.py b/youtube_dl/extractor/vidme.py
index 81dcaa231..382517a4a 100644
--- a/youtube_dl/extractor/vidme.py
+++ b/youtube_dl/extractor/vidme.py
@@ -93,6 +93,10 @@ class VidmeIE(InfoExtractor):
'params': {
'skip_download': True,
},
+ }, {
+ # nsfw, user-disabled
+ 'url': 'https://vid.me/dzGJ',
+ 'only_matching': True,
}]
def _real_extract(self, url):
From 583882fdce19f8c565402f42523b275f96c91575 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Lalinsk=C3=BD?= <lukas@oxygene.sk>
Date: Sat, 17 Oct 2015 19:26:30 +0200
Subject: [PATCH 015/415] [dailymotion] Report errors from player v5
---
youtube_dl/extractor/dailymotion.py | 17 ++++++++++++++---
1 file changed, 14 insertions(+), 3 deletions(-)
diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index 80a05cfee..ea1edceb1 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -96,6 +96,11 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
'uploader': 'HotWaves1012',
'age_limit': 18,
}
+ },
+ # geo-restricted, player v5
+ {
+ 'url': 'http://www.dailymotion.com/video/xhza0o',
+ 'only_matching': True,
}
]
@@ -124,6 +129,9 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
if player_v5:
player = self._parse_json(player_v5, video_id)
metadata = player['metadata']
+
+ self._check_error(metadata)
+
formats = []
for quality, media_list in metadata['qualities'].items():
for media in media_list:
@@ -201,9 +209,7 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
'video info', flags=re.MULTILINE),
video_id)
- if info.get('error') is not None:
- msg = 'Couldn\'t get video, Dailymotion says: %s' % info['error']['title']
- raise ExtractorError(msg, expected=True)
+ self._check_error(info)
formats = []
for (key, format_id) in self._FORMATS:
@@ -246,6 +252,11 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
'duration': info['duration']
}
+ def _check_error(self, info):
+ if info.get('error') is not None:
+ msg = 'Couldn\'t get video, Dailymotion says: %s' % info['error']['title']
+ raise ExtractorError(msg, expected=True)
+
def _get_subtitles(self, video_id, webpage):
try:
sub_list = self._download_webpage(
From 648e6a1ffe45ceae2995c3f9ec6a9413aad55640 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 18 Oct 2015 00:11:34 +0600
Subject: [PATCH 016/415] [youtube] Generalize playlist entries extraction
(Closes #6699, closes #6992)
---
youtube_dl/extractor/youtube.py | 121 ++++++++++++++------------------
1 file changed, 52 insertions(+), 69 deletions(-)
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index b252e36e1..08e821362 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -178,6 +178,52 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
return
+class YoutubePlaylistBaseInfoExtractor(InfoExtractor):
+ # Extract the video ids from the playlist pages
+ def _entries(self, page, playlist_id):
+ more_widget_html = content_html = page
+ for page_num in itertools.count(1):
+ for video_id, video_title in self.extract_videos_from_page(content_html):
+ yield self.url_result(
+ video_id, 'Youtube', video_id=video_id,
+ video_title=video_title)
+
+ mobj = re.search(r'data-uix-load-more-href="/?(?P<more>[^"]+)"', more_widget_html)
+ if not mobj:
+ break
+
+ more = self._download_json(
+ 'https://youtube.com/%s' % mobj.group('more'), playlist_id,
+ 'Downloading page #%s' % page_num,
+ transform_source=uppercase_escape)
+ content_html = more['content_html']
+ if not content_html.strip():
+ # Some webpages show a "Load more" button but they don't
+ # have more videos
+ break
+ more_widget_html = more['load_more_widget_html']
+
+ def extract_videos_from_page(self, page):
+ ids_in_page = []
+ titles_in_page = []
+ for mobj in re.finditer(self._VIDEO_RE, page):
+ # The link with index 0 is not the first video of the playlist (not sure if still actual)
+ if 'index' in mobj.groupdict() and mobj.group('id') == '0':
+ continue
+ video_id = mobj.group('id')
+ video_title = unescapeHTML(mobj.group('title'))
+ if video_title:
+ video_title = video_title.strip()
+ try:
+ idx = ids_in_page.index(video_id)
+ if video_title and not titles_in_page[idx]:
+ titles_in_page[idx] = video_title
+ except ValueError:
+ ids_in_page.append(video_id)
+ titles_in_page.append(video_title)
+ return zip(ids_in_page, titles_in_page)
+
+
class YoutubeIE(YoutubeBaseInfoExtractor):
IE_DESC = 'YouTube.com'
_VALID_URL = r"""(?x)^
@@ -1419,7 +1465,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
}
-class YoutubePlaylistIE(YoutubeBaseInfoExtractor):
+class YoutubePlaylistIE(YoutubeBaseInfoExtractor, YoutubePlaylistBaseInfoExtractor):
IE_DESC = 'YouTube.com playlists'
_VALID_URL = r"""(?x)(?:
(?:https?://)?
@@ -1440,7 +1486,7 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor):
((?:PL|LL|EC|UU|FL|RD|UL)[0-9A-Za-z-_]{10,})
)"""
_TEMPLATE_URL = 'https://www.youtube.com/playlist?list=%s'
- _VIDEO_RE = r'href="\s*/watch\?v=(?P<id>[0-9A-Za-z_-]{11})&[^"]*?index=(?P<index>\d+)'
+ _VIDEO_RE = r'href="\s*/watch\?v=(?P<id>[0-9A-Za-z_-]{11})&[^"]*?index=(?P<index>\d+)(?:[^>]+>(?P<title>[^<]+))?'
IE_NAME = 'youtube:playlist'
_TESTS = [{
'url': 'https://www.youtube.com/playlist?list=PLwiyx1dc3P2JR9N8gQaQN_BCvlSlap7re',
@@ -1557,37 +1603,11 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor):
else:
self.report_warning('Youtube gives an alert message: ' + match)
- # Extract the video ids from the playlist pages
- def _entries():
- more_widget_html = content_html = page
- for page_num in itertools.count(1):
- matches = re.finditer(self._VIDEO_RE, content_html)
- # We remove the duplicates and the link with index 0
- # (it's not the first video of the playlist)
- new_ids = orderedSet(m.group('id') for m in matches if m.group('index') != '0')
- for vid_id in new_ids:
- yield self.url_result(vid_id, 'Youtube', video_id=vid_id)
-
- mobj = re.search(r'data-uix-load-more-href="/?(?P<more>[^"]+)"', more_widget_html)
- if not mobj:
- break
-
- more = self._download_json(
- 'https://youtube.com/%s' % mobj.group('more'), playlist_id,
- 'Downloading page #%s' % page_num,
- transform_source=uppercase_escape)
- content_html = more['content_html']
- if not content_html.strip():
- # Some webpages show a "Load more" button but they don't
- # have more videos
- break
- more_widget_html = more['load_more_widget_html']
-
playlist_title = self._html_search_regex(
r'(?s)<h1 class="pl-header-title[^"]*">\s*(.*?)\s*</h1>',
page, 'title')
- return self.playlist_result(_entries(), playlist_id, playlist_title)
+ return self.playlist_result(self._entries(page, playlist_id), playlist_id, playlist_title)
def _real_extract(self, url):
# Extract playlist id
@@ -1613,10 +1633,11 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor):
return self._extract_playlist(playlist_id)
-class YoutubeChannelIE(InfoExtractor):
+class YoutubeChannelIE(YoutubePlaylistBaseInfoExtractor):
IE_DESC = 'YouTube.com channels'
_VALID_URL = r'https?://(?:youtu\.be|(?:\w+\.)?youtube(?:-nocookie)?\.com)/channel/(?P<id>[0-9A-Za-z_-]+)'
_TEMPLATE_URL = 'https://www.youtube.com/channel/%s/videos'
+ _VIDEO_RE = r'(?:title="(?P<title>[^"]+)"[^>]+)?href="/watch\?v=(?P<id>[0-9A-Za-z_-]+)&?'
IE_NAME = 'youtube:channel'
_TESTS = [{
'note': 'paginated channel',
@@ -1627,22 +1648,6 @@ class YoutubeChannelIE(InfoExtractor):
}
}]
- @staticmethod
- def extract_videos_from_page(page):
- ids_in_page = []
- titles_in_page = []
- for mobj in re.finditer(r'(?:title="(?P<title>[^"]+)"[^>]+)?href="/watch\?v=(?P<id>[0-9A-Za-z_-]+)&?', page):
- video_id = mobj.group('id')
- video_title = unescapeHTML(mobj.group('title'))
- try:
- idx = ids_in_page.index(video_id)
- if video_title and not titles_in_page[idx]:
- titles_in_page[idx] = video_title
- except ValueError:
- ids_in_page.append(video_id)
- titles_in_page.append(video_title)
- return zip(ids_in_page, titles_in_page)
-
def _real_extract(self, url):
channel_id = self._match_id(url)
@@ -1685,29 +1690,7 @@ class YoutubeChannelIE(InfoExtractor):
for video_id, video_title in self.extract_videos_from_page(channel_page)]
return self.playlist_result(entries, channel_id)
- def _entries():
- more_widget_html = content_html = channel_page
- for pagenum in itertools.count(1):
-
- for video_id, video_title in self.extract_videos_from_page(content_html):
- yield self.url_result(
- video_id, 'Youtube', video_id=video_id,
- video_title=video_title)
-
- mobj = re.search(
- r'data-uix-load-more-href="/?(?P<more>[^"]+)"',
- more_widget_html)
- if not mobj:
- break
-
- more = self._download_json(
- 'https://youtube.com/%s' % mobj.group('more'), channel_id,
- 'Downloading page #%s' % (pagenum + 1),
- transform_source=uppercase_escape)
- content_html = more['content_html']
- more_widget_html = more['load_more_widget_html']
-
- return self.playlist_result(_entries(), channel_id)
+ return self.playlist_result(self._entries(channel_page, channel_id), channel_id)
class YoutubeUserIE(YoutubeChannelIE):
From 8e5b1219489be399de55566090e145c89007fa48 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 18 Oct 2015 00:27:06 +0600
Subject: [PATCH 017/415] [test_youtube_lists] Add test flat playlist entries'
titles
---
test/test_youtube_lists.py | 9 +++++++++
1 file changed, 9 insertions(+)
diff --git a/test/test_youtube_lists.py b/test/test_youtube_lists.py
index c889b6f15..26aadb34f 100644
--- a/test/test_youtube_lists.py
+++ b/test/test_youtube_lists.py
@@ -57,5 +57,14 @@ class TestYoutubeLists(unittest.TestCase):
entries = result['entries']
self.assertEqual(len(entries), 100)
+ def test_youtube_flat_playlist_titles(self):
+ dl = FakeYDL()
+ dl.params['extract_flat'] = True
+ ie = YoutubePlaylistIE(dl)
+ result = ie.extract('https://www.youtube.com/playlist?list=PLwiyx1dc3P2JR9N8gQaQN_BCvlSlap7re')
+ self.assertIsPlaylist(result)
+ for entry in result['entries']:
+ self.assertTrue(entry.get('title'))
+
if __name__ == '__main__':
unittest.main()
From 7593fbaa126f8bf14eecff7f103cb497e3d31de5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 18 Oct 2015 01:00:37 +0600
Subject: [PATCH 018/415] [dailymotion] Error spelling
---
youtube_dl/extractor/dailymotion.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index ea1edceb1..9cd9ff17d 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -254,8 +254,8 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
def _check_error(self, info):
if info.get('error') is not None:
- msg = 'Couldn\'t get video, Dailymotion says: %s' % info['error']['title']
- raise ExtractorError(msg, expected=True)
+ raise ExtractorError(
+ '%s said: %s' % (self.IE_NAME, info['error']['title']), expected=True)
def _get_subtitles(self, video_id, webpage):
try:
From 5a11b793fe70beb6b0c7a74a489db9e52c4a742b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 18 Oct 2015 01:36:03 +0600
Subject: [PATCH 019/415] [lynda] Extract all prioritized streams
---
youtube_dl/extractor/lynda.py | 15 ++++++++-------
1 file changed, 8 insertions(+), 7 deletions(-)
diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py
index 378117270..5c973e75c 100644
--- a/youtube_dl/extractor/lynda.py
+++ b/youtube_dl/extractor/lynda.py
@@ -140,13 +140,14 @@ class LyndaIE(LyndaBaseIE):
prioritized_streams = video_json.get('PrioritizedStreams')
if prioritized_streams:
- formats.extend([
- {
- 'url': video_url,
- 'width': int_or_none(format_id),
- 'format_id': format_id,
- } for format_id, video_url in prioritized_streams['0'].items()
- ])
+ for prioritized_stream_id, prioritized_stream in prioritized_streams.items():
+ formats.extend([
+ {
+ 'url': video_url,
+ 'width': int_or_none(format_id),
+ 'format_id': '%s-%s' % (prioritized_stream_id, format_id),
+ } for format_id, video_url in prioritized_stream.items()
+ ])
self._check_formats(formats, video_id)
self._sort_formats(formats)
From 355c7ad361aa3c8a57ff83e3f702a496dce59e65 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 17 Oct 2015 21:30:38 +0100
Subject: [PATCH 020/415] [cspan] handle error massages and extract qualities
---
youtube_dl/extractor/cspan.py | 67 +++++++++++++++++++++--------------
1 file changed, 41 insertions(+), 26 deletions(-)
diff --git a/youtube_dl/extractor/cspan.py b/youtube_dl/extractor/cspan.py
index 994e080d5..c74b35fd9 100644
--- a/youtube_dl/extractor/cspan.py
+++ b/youtube_dl/extractor/cspan.py
@@ -9,16 +9,21 @@ from ..utils import (
find_xpath_attr,
smuggle_url,
determine_ext,
+ ExtractorError,
)
from .senateisvp import SenateISVPIE
+def get_text_attr(d, attr):
+ return d.get(attr, {}).get('#text')
+
+
class CSpanIE(InfoExtractor):
_VALID_URL = r'http://(?:www\.)?c-span\.org/video/\?(?P<id>[0-9a-f]+)'
IE_DESC = 'C-SPAN'
_TESTS = [{
'url': 'http://www.c-span.org/video/?313572-1/HolderonV',
- 'md5': '067803f994e049b455a58b16e5aab442',
+ 'md5': '94b29a4f131ff03d23471dd6f60b6a1d',
'info_dict': {
'id': '315139',
'ext': 'mp4',
@@ -28,7 +33,7 @@ class CSpanIE(InfoExtractor):
'skip': 'Regularly fails on travis, for unknown reasons',
}, {
'url': 'http://www.c-span.org/video/?c4486943/cspan-international-health-care-models',
- 'md5': '4eafd1e91a75d2b1e6a3cbd0995816a2',
+ 'md5': '8e5fbfabe6ad0f89f3012a7943c1287b',
'info_dict': {
'id': 'c4486943',
'ext': 'mp4',
@@ -37,7 +42,7 @@ class CSpanIE(InfoExtractor):
}
}, {
'url': 'http://www.c-span.org/video/?318608-1/gm-ignition-switch-recall',
- 'md5': '446562a736c6bf97118e389433ed88d4',
+ 'md5': '2ae5051559169baadba13fc35345ae74',
'info_dict': {
'id': '342759',
'ext': 'mp4',
@@ -71,8 +76,10 @@ class CSpanIE(InfoExtractor):
return self.url_result(surl, 'SenateISVP', video_id, title)
data = self._download_json(
- 'http://c-spanvideo.org/videoLibrary/assets/player/ajax-player.php?os=android&html5=%s&id=%s' % (video_type, video_id),
- video_id)
+ 'http://www.c-span.org/assets/player/ajax-player.php?os=android&html5=%s&id=%s' % (video_type, video_id),
+ video_id)['video']
+ if data['@status'] != 'Success':
+ raise ExtractorError('%s said: %s' % (self.IE_NAME, get_text_attr(data, 'error')), expected=True)
doc = self._download_xml(
'http://www.c-span.org/common/services/flashXml.php?%sid=%s' % (video_type, video_id),
@@ -83,28 +90,36 @@ class CSpanIE(InfoExtractor):
title = find_xpath_attr(doc, './/string', 'name', 'title').text
thumbnail = find_xpath_attr(doc, './/string', 'name', 'poster').text
- files = data['video']['files']
- try:
- capfile = data['video']['capfile']['#text']
- except KeyError:
- capfile = None
+ files = data['files']
+ capfile = get_text_attr(data, 'capfile')
- entries = [{
- 'id': '%s_%d' % (video_id, partnum + 1),
- 'title': (
- title if len(files) == 1 else
- '%s part %d' % (title, partnum + 1)),
- 'url': unescapeHTML(f['path']['#text']),
- 'description': description,
- 'thumbnail': thumbnail,
- 'duration': int_or_none(f.get('length', {}).get('#text')),
- 'subtitles': {
- 'en': [{
- 'url': capfile,
- 'ext': determine_ext(capfile, 'dfxp')
- }],
- } if capfile else None,
- } for partnum, f in enumerate(files)]
+ entries = []
+ for partnum, f in enumerate(files):
+ formats = []
+ for quality in f['qualities']:
+ formats.append({
+ 'format_id': '%s-%sp' % (get_text_attr(quality, 'bitrate'), get_text_attr(quality, 'height')),
+ 'url': unescapeHTML(get_text_attr(quality, 'file')),
+ 'height': int_or_none(get_text_attr(quality, 'height')),
+ 'tbr': int_or_none(get_text_attr(quality, 'bitrate')),
+ })
+ self._sort_formats(formats)
+ entries.append({
+ 'id': '%s_%d' % (video_id, partnum + 1),
+ 'title': (
+ title if len(files) == 1 else
+ '%s part %d' % (title, partnum + 1)),
+ 'formats': formats,
+ 'description': description,
+ 'thumbnail': thumbnail,
+ 'duration': int_or_none(get_text_attr(f, 'length')),
+ 'subtitles': {
+ 'en': [{
+ 'url': capfile,
+ 'ext': determine_ext(capfile, 'dfxp')
+ }],
+ } if capfile else None,
+ })
if len(entries) == 1:
entry = dict(entries[0])
From 80f48920c8a909ba55d13932524e55ed970f1c6a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 18 Oct 2015 06:57:57 +0600
Subject: [PATCH 021/415] [crunchyroll] Bypass maturity wall (Closes #7202)
---
youtube_dl/extractor/crunchyroll.py | 59 ++++++++++++++++++-----------
1 file changed, 36 insertions(+), 23 deletions(-)
diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index 95952bc29..aa258bbc2 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -32,6 +32,26 @@ from ..aes import (
class CrunchyrollBaseIE(InfoExtractor):
+ _NETRC_MACHINE = 'crunchyroll'
+
+ def _login(self):
+ (username, password) = self._get_login_info()
+ if username is None:
+ return
+ self.report_login()
+ login_url = 'https://www.crunchyroll.com/?a=formhandler'
+ data = urlencode_postdata({
+ 'formname': 'RpcApiUser_Login',
+ 'name': username,
+ 'password': password,
+ })
+ login_request = compat_urllib_request.Request(login_url, data)
+ login_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
+ self._download_webpage(login_request, None, False, 'Wrong login info')
+
+ def _real_initialize(self):
+ self._login()
+
def _download_webpage(self, url_or_request, video_id, note=None, errnote=None, fatal=True, tries=1, timeout=5, encoding=None):
request = (url_or_request if isinstance(url_or_request, compat_urllib_request.Request)
else compat_urllib_request.Request(url_or_request))
@@ -46,10 +66,22 @@ class CrunchyrollBaseIE(InfoExtractor):
return super(CrunchyrollBaseIE, self)._download_webpage(
request, video_id, note, errnote, fatal, tries, timeout, encoding)
+ @staticmethod
+ def _add_skip_wall(url):
+ parsed_url = compat_urlparse.urlparse(url)
+ qs = compat_urlparse.parse_qs(parsed_url.query)
+ # Always force skip_wall to bypass maturity wall, namely 18+ confirmation message:
+ # > This content may be inappropriate for some people.
+ # > Are you sure you want to continue?
+ # since it's not disabled by default in crunchyroll account's settings.
+ # See https://github.com/rg3/youtube-dl/issues/7202.
+ qs['skip_wall'] = ['1']
+ return compat_urlparse.urlunparse(
+ parsed_url._replace(query=compat_urllib_parse.urlencode(qs, True)))
+
class CrunchyrollIE(CrunchyrollBaseIE):
_VALID_URL = r'https?://(?:(?P<prefix>www|m)\.)?(?P<url>crunchyroll\.(?:com|fr)/(?:media(?:-|/\?id=)|[^/]*/[^/?&]*?)(?P<video_id>[0-9]+))(?:[/?&]|$)'
- _NETRC_MACHINE = 'crunchyroll'
_TESTS = [{
'url': 'http://www.crunchyroll.com/wanna-be-the-strongest-in-the-world/episode-1-an-idol-wrestler-is-born-645513',
'info_dict': {
@@ -81,7 +113,6 @@ class CrunchyrollIE(CrunchyrollBaseIE):
# rtmp
'skip_download': True,
},
-
}, {
'url': 'http://www.crunchyroll.fr/girl-friend-beta/episode-11-goodbye-la-mode-661697',
'only_matching': True,
@@ -94,24 +125,6 @@ class CrunchyrollIE(CrunchyrollBaseIE):
'1080': ('80', '108'),
}
- def _login(self):
- (username, password) = self._get_login_info()
- if username is None:
- return
- self.report_login()
- login_url = 'https://www.crunchyroll.com/?a=formhandler'
- data = urlencode_postdata({
- 'formname': 'RpcApiUser_Login',
- 'name': username,
- 'password': password,
- })
- login_request = compat_urllib_request.Request(login_url, data)
- login_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
- self._download_webpage(login_request, None, False, 'Wrong login info')
-
- def _real_initialize(self):
- self._login()
-
def _decrypt_subtitles(self, data, iv, id):
data = bytes_to_intlist(base64.b64decode(data.encode('utf-8')))
iv = bytes_to_intlist(base64.b64decode(iv.encode('utf-8')))
@@ -254,7 +267,7 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
else:
webpage_url = 'http://www.' + mobj.group('url')
- webpage = self._download_webpage(webpage_url, video_id, 'Downloading webpage')
+ webpage = self._download_webpage(self._add_skip_wall(webpage_url), video_id, 'Downloading webpage')
note_m = self._html_search_regex(
r'<div class="showmedia-trailer-notice">(.+?)</div>',
webpage, 'trailer-notice', default='')
@@ -352,7 +365,7 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
class CrunchyrollShowPlaylistIE(CrunchyrollBaseIE):
IE_NAME = "crunchyroll:playlist"
- _VALID_URL = r'https?://(?:(?P<prefix>www|m)\.)?(?P<url>crunchyroll\.com/(?!(?:news|anime-news|library|forum|launchcalendar|lineup|store|comics|freetrial|login))(?P<id>[\w\-]+))/?$'
+ _VALID_URL = r'https?://(?:(?P<prefix>www|m)\.)?(?P<url>crunchyroll\.com/(?!(?:news|anime-news|library|forum|launchcalendar|lineup|store|comics|freetrial|login))(?P<id>[\w\-]+))/?(?:\?|$)'
_TESTS = [{
'url': 'http://www.crunchyroll.com/a-bridge-to-the-starry-skies-hoshizora-e-kakaru-hashi',
@@ -366,7 +379,7 @@ class CrunchyrollShowPlaylistIE(CrunchyrollBaseIE):
def _real_extract(self, url):
show_id = self._match_id(url)
- webpage = self._download_webpage(url, show_id)
+ webpage = self._download_webpage(self._add_skip_wall(url), show_id)
title = self._html_search_regex(
r'(?s)<h1[^>]*>\s*<span itemprop="name">(.*?)</span>',
webpage, 'title')
From 49941c4e4f6e33785a3be1e0d103bd81657d8a0d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 18 Oct 2015 07:06:47 +0600
Subject: [PATCH 022/415] [crunchyroll] Add maturity wall reference tests
(#7202)
---
youtube_dl/extractor/crunchyroll.py | 17 +++++++++++++++++
1 file changed, 17 insertions(+)
diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index aa258bbc2..cecd0c784 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -116,6 +116,10 @@ class CrunchyrollIE(CrunchyrollBaseIE):
}, {
'url': 'http://www.crunchyroll.fr/girl-friend-beta/episode-11-goodbye-la-mode-661697',
'only_matching': True,
+ }, {
+ # geo-restricted (US), 18+ maturity wall, non-premium available
+ 'url': 'http://www.crunchyroll.com/cosplay-complex-ova/episode-1-the-birth-of-the-cosplay-club-565617',
+ 'only_matching': True,
}]
_FORMAT_IDS = {
@@ -374,6 +378,19 @@ class CrunchyrollShowPlaylistIE(CrunchyrollBaseIE):
'title': 'A Bridge to the Starry Skies - Hoshizora e Kakaru Hashi'
},
'playlist_count': 13,
+ }, {
+ # geo-restricted (US), 18+ maturity wall, non-premium available
+ 'url': 'http://www.crunchyroll.com/cosplay-complex-ova',
+ 'info_dict': {
+ 'id': 'cosplay-complex-ova',
+ 'title': 'Cosplay Complex OVA'
+ },
+ 'playlist_count': 3,
+ 'skip': 'Georestricted',
+ }, {
+ # geo-restricted (US), 18+ maturity wall, non-premium will be available since 2015.11.14
+ 'url': 'http://www.crunchyroll.com/ladies-versus-butlers?skip_wall=1',
+ 'only_matching': True,
}]
def _real_extract(self, url):
From 448ef1f31c8bcc1550cf907fd46e31026ec981b2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 18 Oct 2015 09:11:02 +0600
Subject: [PATCH 023/415] [extractor/common] Allow angle brackets in attributes
in _og_regexes (#7215)
---
test/test_InfoExtractor.py | 4 ++++
youtube_dl/extractor/common.py | 2 +-
2 files changed, 5 insertions(+), 1 deletion(-)
diff --git a/test/test_InfoExtractor.py b/test/test_InfoExtractor.py
index 2a00d09a5..938466a80 100644
--- a/test/test_InfoExtractor.py
+++ b/test/test_InfoExtractor.py
@@ -37,12 +37,16 @@ class TestInfoExtractor(unittest.TestCase):
<meta property='og:image' content='http://domain.com/pic.jpg?key1=val1&key2=val2'/>
<meta content='application/x-shockwave-flash' property='og:video:type'>
<meta content='Foo' property=og:foobar>
+ <meta name="og:test1" content='foo > < bar'/>
+ <meta name="og:test2" content="foo >//< bar"/>
'''
self.assertEqual(ie._og_search_title(html), 'Foo')
self.assertEqual(ie._og_search_description(html), 'Some video\'s description ')
self.assertEqual(ie._og_search_thumbnail(html), 'http://domain.com/pic.jpg?key1=val1&key2=val2')
self.assertEqual(ie._og_search_video_url(html, default=None), None)
self.assertEqual(ie._og_search_property('foobar', html), 'Foo')
+ self.assertEqual(ie._og_search_property('test1', html), 'foo > < bar')
+ self.assertEqual(ie._og_search_property('test2', html), 'foo >//< bar')
def test_html_search_meta(self):
ie = self.ie
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index a0c4af92f..4365077f1 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -645,7 +645,7 @@ class InfoExtractor(object):
# Helper functions for extracting OpenGraph info
@staticmethod
def _og_regexes(prop):
- content_re = r'content=(?:"([^>]+?)"|\'([^>]+?)\'|\s*([^\s"\'=<>`]+?))'
+ content_re = r'content=(?:"([^"]+?)"|\'([^\']+?)\'|\s*([^\s"\'=<>`]+?))'
property_re = (r'(?:name|property)=(?:\'og:%(prop)s\'|"og:%(prop)s"|\s*og:%(prop)s\b)'
% {'prop': re.escape(prop)})
template = r'<meta[^>]+?%s[^>]+?%s'
From 94a773feb94a20be66526348a57ebe20495eba3f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Lalinsk=C3=BD?= <lukas@oxygene.sk>
Date: Sat, 17 Oct 2015 22:25:08 +0200
Subject: [PATCH 024/415] [vine] Use JS data to get title/alt_title
---
youtube_dl/extractor/vine.py | 19 +++++++++++++++++--
1 file changed, 17 insertions(+), 2 deletions(-)
diff --git a/youtube_dl/extractor/vine.py b/youtube_dl/extractor/vine.py
index c733a48fa..d80b580a0 100644
--- a/youtube_dl/extractor/vine.py
+++ b/youtube_dl/extractor/vine.py
@@ -51,6 +51,21 @@ class VineIE(InfoExtractor):
}, {
'url': 'https://vine.co/oembed/MYxVapFvz2z.json',
'only_matching': True,
+ }, {
+ 'url': 'https://vine.co/v/e192BnZnZ9V',
+ 'info_dict': {
+ 'id': 'e192BnZnZ9V',
+ 'ext': 'mp4',
+ 'title': u'\u0e22\u0e34\u0e49\u0e21~ \u0e40\u0e02\u0e34\u0e19~ \u0e2d\u0e32\u0e22~ \u0e19\u0e48\u0e32\u0e23\u0e49\u0e32\u0e01\u0e2d\u0e49\u0e30 >//< @n_whitewo @orlameena #lovesicktheseries #lovesickseason2',
+ 'alt_title': 'Vine by Pimry_zaa',
+ 'description': u'\u0e22\u0e34\u0e49\u0e21~ \u0e40\u0e02\u0e34\u0e19~ \u0e2d\u0e32\u0e22~ \u0e19\u0e48\u0e32\u0e23\u0e49\u0e32\u0e01\u0e2d\u0e49\u0e30 >//< @n_whitewo @orlameena #lovesicktheseries #lovesickseason2',
+ 'upload_date': '20150705',
+ 'uploader': 'Pimry_zaa',
+ 'uploader_id': '1135760698325307392',
+ },
+ 'params': {
+ 'skip_download': True,
+ },
}]
def _real_extract(self, url):
@@ -74,8 +89,8 @@ class VineIE(InfoExtractor):
return {
'id': video_id,
- 'title': self._og_search_title(webpage),
- 'alt_title': self._og_search_description(webpage, default=None),
+ 'title': data['description'],
+ 'alt_title': 'Vine by %s' % data['username'],
'description': data['description'],
'thumbnail': data['thumbnailUrl'],
'upload_date': unified_strdate(data['created']),
From 10c38c7ca248d06c2c0f069c5a810e27e207c61e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Lalinsk=C3=BD?= <lukas@oxygene.sk>
Date: Sat, 17 Oct 2015 22:29:49 +0200
Subject: [PATCH 025/415] [vine] Fix download tests
---
youtube_dl/extractor/vine.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/youtube_dl/extractor/vine.py b/youtube_dl/extractor/vine.py
index d80b580a0..d1dbec893 100644
--- a/youtube_dl/extractor/vine.py
+++ b/youtube_dl/extractor/vine.py
@@ -29,10 +29,10 @@ class VineIE(InfoExtractor):
'id': 'MYxVapFvz2z',
'ext': 'mp4',
'title': 'Fuck Da Police #Mikebrown #justice #ferguson #prayforferguson #protesting #NMOS14',
- 'alt_title': 'Vine by Luna',
+ 'alt_title': 'Vine by Mars Ruiz',
'description': 'Fuck Da Police #Mikebrown #justice #ferguson #prayforferguson #protesting #NMOS14',
'upload_date': '20140815',
- 'uploader': 'Luna',
+ 'uploader': 'Mars Ruiz',
'uploader_id': '1102363502380728320',
},
}, {
From 91816e8f16408a3a2753fb254a9e963ad9429ced Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 18 Oct 2015 09:32:08 +0600
Subject: [PATCH 026/415] [vine] Remove duplicate metadata, make more robust
and modernize (Closes #7215)
---
youtube_dl/extractor/vine.py | 39 ++++++++++++++++++------------------
1 file changed, 20 insertions(+), 19 deletions(-)
diff --git a/youtube_dl/extractor/vine.py b/youtube_dl/extractor/vine.py
index d1dbec893..6e72cc253 100644
--- a/youtube_dl/extractor/vine.py
+++ b/youtube_dl/extractor/vine.py
@@ -1,10 +1,14 @@
+# coding: utf-8
from __future__ import unicode_literals
import re
import itertools
from .common import InfoExtractor
-from ..utils import unified_strdate
+from ..utils import (
+ int_or_none,
+ unified_strdate,
+)
class VineIE(InfoExtractor):
@@ -17,7 +21,6 @@ class VineIE(InfoExtractor):
'ext': 'mp4',
'title': 'Chicken.',
'alt_title': 'Vine by Jack Dorsey',
- 'description': 'Chicken.',
'upload_date': '20130519',
'uploader': 'Jack Dorsey',
'uploader_id': '76',
@@ -30,7 +33,6 @@ class VineIE(InfoExtractor):
'ext': 'mp4',
'title': 'Fuck Da Police #Mikebrown #justice #ferguson #prayforferguson #protesting #NMOS14',
'alt_title': 'Vine by Mars Ruiz',
- 'description': 'Fuck Da Police #Mikebrown #justice #ferguson #prayforferguson #protesting #NMOS14',
'upload_date': '20140815',
'uploader': 'Mars Ruiz',
'uploader_id': '1102363502380728320',
@@ -43,7 +45,6 @@ class VineIE(InfoExtractor):
'ext': 'mp4',
'title': '#mw3 #ac130 #killcam #angelofdeath',
'alt_title': 'Vine by Z3k3',
- 'description': '#mw3 #ac130 #killcam #angelofdeath',
'upload_date': '20130430',
'uploader': 'Z3k3',
'uploader_id': '936470460173008896',
@@ -56,9 +57,8 @@ class VineIE(InfoExtractor):
'info_dict': {
'id': 'e192BnZnZ9V',
'ext': 'mp4',
- 'title': u'\u0e22\u0e34\u0e49\u0e21~ \u0e40\u0e02\u0e34\u0e19~ \u0e2d\u0e32\u0e22~ \u0e19\u0e48\u0e32\u0e23\u0e49\u0e32\u0e01\u0e2d\u0e49\u0e30 >//< @n_whitewo @orlameena #lovesicktheseries #lovesickseason2',
+ 'title': 'ยิ้ม~ เขิน~ อาย~ น่าร้ากอ้ะ >//< @n_whitewo @orlameena #lovesicktheseries #lovesickseason2',
'alt_title': 'Vine by Pimry_zaa',
- 'description': u'\u0e22\u0e34\u0e49\u0e21~ \u0e40\u0e02\u0e34\u0e19~ \u0e2d\u0e32\u0e22~ \u0e19\u0e48\u0e32\u0e23\u0e49\u0e32\u0e01\u0e2d\u0e49\u0e30 >//< @n_whitewo @orlameena #lovesicktheseries #lovesickseason2',
'upload_date': '20150705',
'uploader': 'Pimry_zaa',
'uploader_id': '1135760698325307392',
@@ -80,25 +80,26 @@ class VineIE(InfoExtractor):
formats = [{
'format_id': '%(format)s-%(rate)s' % f,
- 'vcodec': f['format'],
- 'quality': f['rate'],
+ 'vcodec': f.get('format'),
+ 'quality': f.get('rate'),
'url': f['videoUrl'],
- } for f in data['videoUrls']]
+ } for f in data['videoUrls'] if f.get('videoUrl')]
self._sort_formats(formats)
+ username = data.get('username')
+
return {
'id': video_id,
- 'title': data['description'],
- 'alt_title': 'Vine by %s' % data['username'],
- 'description': data['description'],
- 'thumbnail': data['thumbnailUrl'],
- 'upload_date': unified_strdate(data['created']),
- 'uploader': data['username'],
- 'uploader_id': data['userIdStr'],
- 'like_count': data['likes']['count'],
- 'comment_count': data['comments']['count'],
- 'repost_count': data['reposts']['count'],
+ 'title': data.get('description') or self._og_search_title(webpage),
+ 'alt_title': 'Vine by %s' % username if username else self._og_search_description(webpage, default=None),
+ 'thumbnail': data.get('thumbnailUrl'),
+ 'upload_date': unified_strdate(data.get('created')),
+ 'uploader': username,
+ 'uploader_id': data.get('userIdStr'),
+ 'like_count': int_or_none(data.get('likes', {}).get('count')),
+ 'comment_count': int_or_none(data.get('comments', {}).get('count')),
+ 'repost_count': int_or_none(data.get('reposts', {}).get('count')),
'formats': formats,
}
From 02835c6bf4403a907c058d43220a83b3b427e181 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 18 Oct 2015 09:34:54 +0600
Subject: [PATCH 027/415] [extractor/common] Document repost_count
---
youtube_dl/extractor/common.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 4365077f1..6169fbbeb 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -172,6 +172,7 @@ class InfoExtractor(object):
view_count: How many users have watched the video on the platform.
like_count: Number of positive ratings of the video
dislike_count: Number of negative ratings of the video
+ repost_count: Number of reposts of the video
average_rating: Average rating give by users, the scale used depends on the webpage
comment_count: Number of comments on the video
comments: A list of comments, each with one or more of the following
From 2e022397c45fbcfd2ef6da43d14b0770221aabd5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 18 Oct 2015 09:36:19 +0600
Subject: [PATCH 028/415] [vine] Add counters to tests
---
youtube_dl/extractor/vine.py | 12 ++++++++++++
1 file changed, 12 insertions(+)
diff --git a/youtube_dl/extractor/vine.py b/youtube_dl/extractor/vine.py
index 6e72cc253..be72f3147 100644
--- a/youtube_dl/extractor/vine.py
+++ b/youtube_dl/extractor/vine.py
@@ -24,6 +24,9 @@ class VineIE(InfoExtractor):
'upload_date': '20130519',
'uploader': 'Jack Dorsey',
'uploader_id': '76',
+ 'like_count': int,
+ 'comment_count': int,
+ 'repost_count': int,
},
}, {
'url': 'https://vine.co/v/MYxVapFvz2z',
@@ -36,6 +39,9 @@ class VineIE(InfoExtractor):
'upload_date': '20140815',
'uploader': 'Mars Ruiz',
'uploader_id': '1102363502380728320',
+ 'like_count': int,
+ 'comment_count': int,
+ 'repost_count': int,
},
}, {
'url': 'https://vine.co/v/bxVjBbZlPUH',
@@ -48,6 +54,9 @@ class VineIE(InfoExtractor):
'upload_date': '20130430',
'uploader': 'Z3k3',
'uploader_id': '936470460173008896',
+ 'like_count': int,
+ 'comment_count': int,
+ 'repost_count': int,
},
}, {
'url': 'https://vine.co/oembed/MYxVapFvz2z.json',
@@ -62,6 +71,9 @@ class VineIE(InfoExtractor):
'upload_date': '20150705',
'uploader': 'Pimry_zaa',
'uploader_id': '1135760698325307392',
+ 'like_count': int,
+ 'comment_count': int,
+ 'repost_count': int,
},
'params': {
'skip_download': True,
From 1e399778ee870ee583135e65458268cd7c0fb923 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 22 Jul 2015 20:03:05 +0800
Subject: [PATCH 029/415] [letv] Fix extraction
Using data URIs for passing the decrypted M3U8 manifest, which is
supported by ffmpeg only.
---
youtube_dl/extractor/letv.py | 70 ++++++++++++++++++++++++++----------
youtube_dl/utils.py | 5 +++
2 files changed, 57 insertions(+), 18 deletions(-)
diff --git a/youtube_dl/extractor/letv.py b/youtube_dl/extractor/letv.py
index a28abb0f0..9ebbc8089 100644
--- a/youtube_dl/extractor/letv.py
+++ b/youtube_dl/extractor/letv.py
@@ -9,13 +9,14 @@ from .common import InfoExtractor
from ..compat import (
compat_urllib_parse,
compat_urllib_request,
- compat_urlparse,
+ compat_ord,
)
from ..utils import (
determine_ext,
ExtractorError,
parse_iso8601,
int_or_none,
+ encode_data_uri,
)
@@ -25,15 +26,16 @@ class LetvIE(InfoExtractor):
_TESTS = [{
'url': 'http://www.letv.com/ptv/vplay/22005890.html',
- 'md5': 'cab23bd68d5a8db9be31c9a222c1e8df',
+ 'md5': 'edadcfe5406976f42f9f266057ee5e40',
'info_dict': {
'id': '22005890',
'ext': 'mp4',
'title': '第87届奥斯卡颁奖礼完美落幕 《鸟人》成最大赢家',
- 'timestamp': 1424747397,
- 'upload_date': '20150224',
'description': 'md5:a9cb175fd753e2962176b7beca21a47c',
- }
+ },
+ 'params': {
+ 'hls_prefer_native': True,
+ },
}, {
'url': 'http://www.letv.com/ptv/vplay/1415246.html',
'info_dict': {
@@ -42,16 +44,22 @@ class LetvIE(InfoExtractor):
'title': '美人天下01',
'description': 'md5:f88573d9d7225ada1359eaf0dbf8bcda',
},
+ 'params': {
+ 'hls_prefer_native': True,
+ },
}, {
'note': 'This video is available only in Mainland China, thus a proxy is needed',
'url': 'http://www.letv.com/ptv/vplay/1118082.html',
- 'md5': 'f80936fbe20fb2f58648e81386ff7927',
+ 'md5': '2424c74948a62e5f31988438979c5ad1',
'info_dict': {
'id': '1118082',
'ext': 'mp4',
'title': '与龙共舞 完整版',
'description': 'md5:7506a5eeb1722bb9d4068f85024e3986',
},
+ 'params': {
+ 'hls_prefer_native': True,
+ },
'skip': 'Only available in China',
}]
@@ -74,6 +82,27 @@ class LetvIE(InfoExtractor):
_loc3_ = self.ror(_loc3_, _loc2_ % 17)
return _loc3_
+ # see M3U8Encryption class in KLetvPlayer.swf
+ @staticmethod
+ def decrypt_m3u8(encrypted_data):
+ if encrypted_data[:5].decode('utf-8').lower() != 'vc_01':
+ return encrypted_data
+ encrypted_data = encrypted_data[5:]
+
+ _loc4_ = bytearray()
+ while encrypted_data:
+ b = compat_ord(encrypted_data[0])
+ _loc4_.extend([b // 16, b & 0x0f])
+ encrypted_data = encrypted_data[1:]
+ idx = len(_loc4_) - 11
+ _loc4_ = _loc4_[idx:] + _loc4_[:idx]
+ _loc7_ = bytearray()
+ while _loc4_:
+ _loc7_.append(_loc4_[0] * 16 + _loc4_[1])
+ _loc4_ = _loc4_[2:]
+
+ return bytes(_loc7_)
+
def _real_extract(self, url):
media_id = self._match_id(url)
page = self._download_webpage(url, media_id)
@@ -115,23 +144,28 @@ class LetvIE(InfoExtractor):
for format_id in formats:
if format_id in dispatch:
media_url = playurl['domain'][0] + dispatch[format_id][0]
-
- # Mimic what flvxz.com do
- url_parts = list(compat_urlparse.urlparse(media_url))
- qs = dict(compat_urlparse.parse_qs(url_parts[4]))
- qs.update({
- 'platid': '14',
- 'splatid': '1401',
- 'tss': 'no',
- 'retry': 1
+ media_url += '&' + compat_urllib_parse.urlencode({
+ 'm3v': 1,
+ 'format': 1,
+ 'expect': 3,
+ 'rateid': format_id,
})
- url_parts[4] = compat_urllib_parse.urlencode(qs)
- media_url = compat_urlparse.urlunparse(url_parts)
+
+ nodes_data = self._download_json(
+ media_url, media_id,
+ 'Download JSON metadata for format %s' % format_id)
+
+ req = self._request_webpage(
+ nodes_data['nodelist'][0]['location'], media_id,
+ note='Downloading m3u8 information for format %s' % format_id)
+
+ m3u8_data = self.decrypt_m3u8(req.read())
url_info_dict = {
- 'url': media_url,
+ 'url': encode_data_uri(m3u8_data, 'application/x-mpegURL'),
'ext': determine_ext(dispatch[format_id][1]),
'format_id': format_id,
+ 'protocol': 'm3u8',
}
if format_id[-1:] == 'p':
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 7dbe25661..db5b3698e 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -3,6 +3,7 @@
from __future__ import unicode_literals
+import base64
import calendar
import codecs
import contextlib
@@ -1795,6 +1796,10 @@ def urlhandle_detect_ext(url_handle):
return mimetype2ext(getheader('Content-Type'))
+def encode_data_uri(data, mime_type):
+ return 'data:%s;base64,%s' % (mime_type, base64.b64encode(data).decode('ascii'))
+
+
def age_restricted(content_limit, age_limit):
""" Returns True iff the content should be blocked """
From 985e4fdc07f00a3fdc8e7b7b4119471ee97f3890 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 17 Oct 2015 22:49:05 +0800
Subject: [PATCH 030/415] [downloader/hls] Add headers only for http(s) URLs
ffmpeg 2.8.1 raises an error with -headers and non-http input files.
---
youtube_dl/downloader/hls.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index a62d2047b..9a83a73dd 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -30,7 +30,7 @@ class HlsFD(FileDownloader):
args = [ffpp.executable, '-y']
- if info_dict['http_headers']:
+ if info_dict['http_headers'] and re.match(r'^https?://', url):
# Trailing \r\n after each HTTP header is important to prevent warning from ffmpeg/avconv:
# [http @ 00000000003d2fa0] No trailing CRLF found in HTTP header.
args += [
From 0a67a3632bb9cf76f64658986defc1947090ef50 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 17 Oct 2015 23:15:01 +0800
Subject: [PATCH 031/415] [compat] Add compat_urllib_request_DataHandler
---
youtube_dl/compat.py | 44 ++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 44 insertions(+)
diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 192e1c515..d103ab9ad 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -1,7 +1,10 @@
from __future__ import unicode_literals
+import binascii
import collections
+import email
import getpass
+import io
import optparse
import os
import re
@@ -38,6 +41,11 @@ try:
except ImportError: # Python 2
import urlparse as compat_urlparse
+try:
+ import urllib.response as compat_urllib_response
+except ImportError: # Python 2
+ import urllib as compat_urllib_response
+
try:
import http.cookiejar as compat_cookiejar
except ImportError: # Python 2
@@ -155,6 +163,40 @@ except ImportError: # Python 2
string = string.replace('+', ' ')
return compat_urllib_parse_unquote(string, encoding, errors)
+try:
+ from urllib.request import DataHandler as compat_urllib_request_DataHandler
+except ImportError: # Python < 3.4
+ # Ported from CPython 98774:1733b3bd46db, Lib/urllib/request.py
+ class compat_urllib_request_DataHandler(compat_urllib_request.BaseHandler):
+ def data_open(self, req):
+ # data URLs as specified in RFC 2397.
+ #
+ # ignores POSTed data
+ #
+ # syntax:
+ # dataurl := "data:" [ mediatype ] [ ";base64" ] "," data
+ # mediatype := [ type "/" subtype ] *( ";" parameter )
+ # data := *urlchar
+ # parameter := attribute "=" value
+ url = req.get_full_url()
+
+ scheme, data = url.split(":", 1)
+ mediatype, data = data.split(",", 1)
+
+ # even base64 encoded data URLs might be quoted so unquote in any case:
+ data = compat_urllib_parse_unquote_to_bytes(data)
+ if mediatype.endswith(";base64"):
+ data = binascii.a2b_base64(data)
+ mediatype = mediatype[:-7]
+
+ if not mediatype:
+ mediatype = "text/plain;charset=US-ASCII"
+
+ headers = email.message_from_string(
+ "Content-type: %s\nContent-length: %d\n" % (mediatype, len(data)))
+
+ return compat_urllib_response.addinfourl(io.BytesIO(data), headers, url)
+
try:
compat_basestring = basestring # Python 2
except NameError:
@@ -489,6 +531,8 @@ __all__ = [
'compat_urllib_parse_unquote_to_bytes',
'compat_urllib_parse_urlparse',
'compat_urllib_request',
+ 'compat_urllib_request_DataHandler',
+ 'compat_urllib_response',
'compat_urlparse',
'compat_urlretrieve',
'compat_xml_parse_error',
From 8b172c2e10fb38c62c213673304c7e8dcd17b768 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 17 Oct 2015 23:16:40 +0800
Subject: [PATCH 032/415] [YoutubeDL] Use DataHandler
---
youtube_dl/YoutubeDL.py | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index adf70d658..12977bf80 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -37,6 +37,7 @@ from .compat import (
compat_tokenize_tokenize,
compat_urllib_error,
compat_urllib_request,
+ compat_urllib_request_DataHandler,
)
from .utils import (
ContentTooShortError,
@@ -1967,8 +1968,9 @@ class YoutubeDL(object):
debuglevel = 1 if self.params.get('debug_printtraffic') else 0
https_handler = make_HTTPS_handler(self.params, debuglevel=debuglevel)
ydlh = YoutubeDLHandler(self.params, debuglevel=debuglevel)
+ data_handler = compat_urllib_request_DataHandler()
opener = compat_urllib_request.build_opener(
- proxy_handler, https_handler, cookie_processor, ydlh)
+ proxy_handler, https_handler, cookie_processor, ydlh, data_handler)
# Delete the default user-agent header, which would otherwise apply in
# cases where our custom HTTP handler doesn't come into play
From 48aae2d2cf49843d0efa227fa393a0c783fc3c1e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 18 Oct 2015 17:07:48 +0800
Subject: [PATCH 033/415] [twitter] Update tests
---
youtube_dl/extractor/twitter.py | 19 ++++++++++---------
1 file changed, 10 insertions(+), 9 deletions(-)
diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index 1dd43ff3c..b2fff73b9 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -1,3 +1,4 @@
+# coding: utf-8
from __future__ import unicode_literals
import re
@@ -15,7 +16,7 @@ class TwitterCardIE(InfoExtractor):
_TESTS = [
{
'url': 'https://twitter.com/i/cards/tfw/v1/560070183650213889',
- 'md5': 'a74f50b310c83170319ba16de6955192',
+ 'md5': '7d2f6b4d2eb841a7ccc893d479bfceb4',
'info_dict': {
'id': '560070183650213889',
'ext': 'mp4',
@@ -103,17 +104,17 @@ class TwitterIE(TwitterCardIE):
_VALID_URL = r'https?://(?:www|m|mobile)?\.?twitter\.com/(?P<id>[^/]+/status/\d+)'
_TESTS = [{
- 'url': 'https://m.twitter.com/thereaIbanksy/status/614301758345490432',
- 'md5': '8bbccb487bd7a31349b775915fcd412f',
+ 'url': 'https://twitter.com/freethenipple/status/643211948184596480',
+ 'md5': '31cd83a116fc41f99ae3d909d4caf6a0',
'info_dict': {
- 'id': '614301758345490432',
+ 'id': '643211948184596480',
'ext': 'mp4',
- 'title': 'thereaIbanksy - This time lapse is so pretty \U0001f60d\U0001f60d',
+ 'title': 'freethenipple - FTN supporters on Hollywood Blvd today!',
'thumbnail': 're:^https?://.*\.jpg',
- 'duration': 29.5,
- 'description': 'banksy on Twitter: "This time lapse is so pretty \U0001f60d\U0001f60d http://t.co/QB8DDbqiR1"',
- 'uploader': 'banksy',
- 'uploader_id': 'thereaIbanksy',
+ 'duration': 12.922,
+ 'description': 'FREE THE NIPPLE on Twitter: "FTN supporters on Hollywood Blvd today! http://t.co/c7jHH749xJ"',
+ 'uploader': 'FREE THE NIPPLE',
+ 'uploader_id': 'freethenipple',
},
}]
From 01d22d47039dedace1c5414c83e9fecfca41b5a5 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 18 Oct 2015 17:11:55 +0800
Subject: [PATCH 034/415] [twitter] Use _download_xml
---
youtube_dl/extractor/twitter.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index b2fff73b9..37a9fd5fd 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -8,6 +8,7 @@ from ..compat import compat_urllib_request
from ..utils import (
float_or_none,
unescapeHTML,
+ xpath_text,
)
@@ -60,9 +61,8 @@ class TwitterCardIE(InfoExtractor):
video_id)
if 'playlist' not in config:
if 'vmapUrl' in config:
- webpage = self._download_webpage(config['vmapUrl'], video_id + ' (xml)')
- video_url = self._search_regex(
- r'<MediaFile>\s*<!\[CDATA\[(https?://.+?)\]\]>', webpage, 'data player config (xml)')
+ vmap_data = self._download_xml(config['vmapUrl'], video_id)
+ video_url = xpath_text(vmap_data, './/MediaFile').strip()
f = {
'url': video_url,
}
From 014e880372e896cdd63f9075864d2a3bba60e706 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 18 Oct 2015 17:13:58 +0800
Subject: [PATCH 035/415] [twitter] Add IE_NAMEs
---
youtube_dl/extractor/twitter.py | 2 ++
1 file changed, 2 insertions(+)
diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index 37a9fd5fd..5f697782e 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -13,6 +13,7 @@ from ..utils import (
class TwitterCardIE(InfoExtractor):
+ IE_NAME = 'twitter:card'
_VALID_URL = r'https?://(?:www\.)?twitter\.com/i/cards/tfw/v1/(?P<id>\d+)'
_TESTS = [
{
@@ -101,6 +102,7 @@ class TwitterCardIE(InfoExtractor):
class TwitterIE(TwitterCardIE):
+ IE_NAME = 'twitter'
_VALID_URL = r'https?://(?:www|m|mobile)?\.?twitter\.com/(?P<id>[^/]+/status/\d+)'
_TESTS = [{
From f322bfb0638aeeb527459ebcf00f8a3dde26280c Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 18 Oct 2015 17:15:47 +0800
Subject: [PATCH 036/415] [twitter:card] Remove unneeded 'ext'
---
youtube_dl/extractor/twitter.py | 8 ++------
1 file changed, 2 insertions(+), 6 deletions(-)
diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index 5f697782e..48bef5d80 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -64,13 +64,9 @@ class TwitterCardIE(InfoExtractor):
if 'vmapUrl' in config:
vmap_data = self._download_xml(config['vmapUrl'], video_id)
video_url = xpath_text(vmap_data, './/MediaFile').strip()
- f = {
+ formats.append({
'url': video_url,
- }
- ext = re.search(r'\.([a-z0-9]{2,4})(\?.+)?$', video_url)
- if ext:
- f['ext'] = ext.group(1)
- formats.append(f)
+ })
break # same video regardless of UA
continue
From e04edad621efe56347e155b6dc59a0c3d589b3bd Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 18 Oct 2015 17:16:57 +0800
Subject: [PATCH 037/415] [twitter] Inherit from InfoExtractor directly
---
youtube_dl/extractor/twitter.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index 48bef5d80..c9b783745 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -97,11 +97,11 @@ class TwitterCardIE(InfoExtractor):
}
-class TwitterIE(TwitterCardIE):
+class TwitterIE(InfoExtractor):
IE_NAME = 'twitter'
_VALID_URL = r'https?://(?:www|m|mobile)?\.?twitter\.com/(?P<id>[^/]+/status/\d+)'
- _TESTS = [{
+ _TEST = {
'url': 'https://twitter.com/freethenipple/status/643211948184596480',
'md5': '31cd83a116fc41f99ae3d909d4caf6a0',
'info_dict': {
@@ -114,7 +114,7 @@ class TwitterIE(TwitterCardIE):
'uploader': 'FREE THE NIPPLE',
'uploader_id': 'freethenipple',
},
- }]
+ }
def _real_extract(self, url):
id = self._match_id(url)
From f6dfd6603a9e9bb88ebcdcd52490974a34d1bd11 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 18 Oct 2015 17:18:01 +0800
Subject: [PATCH 038/415] [twitter] Use _html_search_regex
---
youtube_dl/extractor/twitter.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index c9b783745..6ff15369c 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -122,7 +122,7 @@ class TwitterIE(InfoExtractor):
name = username
url = re.sub(r'https?://(m|mobile)\.', 'https://', url)
webpage = self._download_webpage(url, 'tweet: ' + url)
- description = unescapeHTML(self._search_regex('<title>\s*(.+?)\s*</title>', webpage, 'title'))
+ description = self._html_search_regex('<title>\s*(.+?)\s*</title>', webpage, 'title')
title = description.replace('\n', ' ')
splitdesc = re.match(r'^(.+?)\s*on Twitter:\s* "(.+?)"$', title)
if splitdesc:
From 575036b40504bc921b18f05bde64e0e7dceacec6 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 18 Oct 2015 18:04:13 +0800
Subject: [PATCH 039/415] [twitter] Simplify and improve
---
youtube_dl/extractor/twitter.py | 41 +++++++++++++++++++--------------
1 file changed, 24 insertions(+), 17 deletions(-)
diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index 6ff15369c..6b3b39aee 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -9,6 +9,7 @@ from ..utils import (
float_or_none,
unescapeHTML,
xpath_text,
+ remove_end,
)
@@ -99,7 +100,8 @@ class TwitterCardIE(InfoExtractor):
class TwitterIE(InfoExtractor):
IE_NAME = 'twitter'
- _VALID_URL = r'https?://(?:www|m|mobile)?\.?twitter\.com/(?P<id>[^/]+/status/\d+)'
+ _VALID_URL = r'https?://(?:www\.|m\.|mobile\.)?twitter\.com/(?P<user_id>[^/]+)/status/(?P<id>\d+)'
+ _TEMPLATE_URL = 'https://twitter.com/%s/status/%s'
_TEST = {
'url': 'https://twitter.com/freethenipple/status/643211948184596480',
@@ -107,7 +109,7 @@ class TwitterIE(InfoExtractor):
'info_dict': {
'id': '643211948184596480',
'ext': 'mp4',
- 'title': 'freethenipple - FTN supporters on Hollywood Blvd today!',
+ 'title': 'FREE THE NIPPLE - FTN supporters on Hollywood Blvd today!',
'thumbnail': 're:^https?://.*\.jpg',
'duration': 12.922,
'description': 'FREE THE NIPPLE on Twitter: "FTN supporters on Hollywood Blvd today! http://t.co/c7jHH749xJ"',
@@ -117,26 +119,31 @@ class TwitterIE(InfoExtractor):
}
def _real_extract(self, url):
- id = self._match_id(url)
- username, twid = re.match(r'([^/]+)/status/(\d+)', id).groups()
- name = username
- url = re.sub(r'https?://(m|mobile)\.', 'https://', url)
- webpage = self._download_webpage(url, 'tweet: ' + url)
- description = self._html_search_regex('<title>\s*(.+?)\s*</title>', webpage, 'title')
- title = description.replace('\n', ' ')
- splitdesc = re.match(r'^(.+?)\s*on Twitter:\s* "(.+?)"$', title)
- if splitdesc:
- name, title = splitdesc.groups()
- title = re.sub(r'\s*https?://[^ ]+', '', title) # strip 'https -_t.co_BJYgOjSeGA' junk from filenames
- card_id = self._search_regex(r'["\']/i/cards/tfw/v1/(\d+)', webpage, '/i/card/...')
+ mobj = re.match(self._VALID_URL, url)
+ user_id = mobj.group('user_id')
+ twid = mobj.group('id')
+
+ webpage = self._download_webpage(self._TEMPLATE_URL % (user_id, twid), twid)
+
+ username = remove_end(self._og_search_title(webpage), ' on Twitter')
+
+ title = self._og_search_description(webpage).strip('').replace('\n', ' ')
+
+ # strip 'https -_t.co_BJYgOjSeGA' junk from filenames
+ mobj = re.match(r'“(.*)\s+(http://[^ ]+)”', title)
+ title, short_url = mobj.groups()
+
+ card_id = self._search_regex(
+ r'["\']/i/cards/tfw/v1/(\d+)', webpage, 'twitter card url')
card_url = 'https://twitter.com/i/cards/tfw/v1/' + card_id
+
return {
'_type': 'url_transparent',
'ie_key': 'TwitterCard',
- 'uploader_id': username,
- 'uploader': name,
+ 'uploader_id': user_id,
+ 'uploader': username,
'url': card_url,
'webpage_url': url,
- 'description': description,
+ 'description': '%s on Twitter: "%s %s"' % (username, title, short_url),
'title': username + ' - ' + title,
}
From 77a54b6a658059a11de415d793588fdbfec14194 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 18 Oct 2015 18:08:24 +0800
Subject: [PATCH 040/415] [twitter:card] Use _html_search_regex
---
youtube_dl/extractor/twitter.py | 6 ++----
1 file changed, 2 insertions(+), 4 deletions(-)
diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index 6b3b39aee..1cdca544c 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -7,7 +7,6 @@ from .common import InfoExtractor
from ..compat import compat_urllib_request
from ..utils import (
float_or_none,
- unescapeHTML,
xpath_text,
remove_end,
)
@@ -57,9 +56,8 @@ class TwitterCardIE(InfoExtractor):
request.add_header('User-Agent', user_agent)
webpage = self._download_webpage(request, video_id)
- config = self._parse_json(
- unescapeHTML(self._search_regex(
- r'data-player-config="([^"]+)"', webpage, 'data player config')),
+ config = self._parse_json(self._html_search_regex(
+ r'data-player-config="([^"]+)"', webpage, 'data player config'),
video_id)
if 'playlist' not in config:
if 'vmapUrl' in config:
From c88aec845a680ef9404b637b3dbcf706dcf00b68 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 18 Oct 2015 18:23:56 +0800
Subject: [PATCH 041/415] [twitter] Fix short URL extraction
---
youtube_dl/extractor/twitter.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index 1cdca544c..1472f22a7 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -128,7 +128,7 @@ class TwitterIE(InfoExtractor):
title = self._og_search_description(webpage).strip('').replace('\n', ' ')
# strip 'https -_t.co_BJYgOjSeGA' junk from filenames
- mobj = re.match(r'“(.*)\s+(http://[^ ]+)”', title)
+ mobj = re.match(r'“(.*)\s+(https?://[^ ]+)”', title)
title, short_url = mobj.groups()
card_id = self._search_regex(
From 4a7b79038425f614af49116edab7897f0db13e5a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 18 Oct 2015 19:07:37 +0800
Subject: [PATCH 042/415] [twitter:card] Support YouTube embeds
---
youtube_dl/extractor/twitter.py | 19 +++++++++++++++++++
1 file changed, 19 insertions(+)
diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index 1472f22a7..9d3e46b94 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -37,6 +37,19 @@ class TwitterCardIE(InfoExtractor):
'thumbnail': 're:^https?://.*\.jpg',
'duration': 80.155,
},
+ },
+ {
+ 'url': 'https://twitter.com/i/cards/tfw/v1/654001591733886977',
+ 'md5': 'b6f35e8b08a0bec6c8af77a2f4b3a814',
+ 'info_dict': {
+ 'id': 'dq4Oj5quskI',
+ 'ext': 'mp4',
+ 'title': 'Ubuntu 11.10 Overview',
+ 'description': 'Take a quick peek at what\'s new and improved in Ubuntu 11.10.\n\nOnce installed take a look at 10 Things to Do After Installing: http://www.omgubuntu.co.uk/2011/10/10-things-to-do-after-installing-ubuntu-11-10/',
+ 'upload_date': '20111013',
+ 'uploader': 'OMG! Ubuntu!',
+ 'uploader_id': 'omgubuntu',
+ },
}
]
@@ -56,6 +69,12 @@ class TwitterCardIE(InfoExtractor):
request.add_header('User-Agent', user_agent)
webpage = self._download_webpage(request, video_id)
+ youtube_url = self._html_search_regex(
+ r'<iframe[^>]+src="((?:https?:)?//www.youtube.com/embed/[^"]+)"',
+ webpage, 'youtube iframe', default=None)
+ if youtube_url:
+ return self.url_result(youtube_url, 'Youtube')
+
config = self._parse_json(self._html_search_regex(
r'data-player-config="([^"]+)"', webpage, 'data player config'),
video_id)
From 05a3879f1c142cc2bf0287cde4690d8ccadcdc8f Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 18 Oct 2015 19:19:46 +0800
Subject: [PATCH 043/415] [letv] Update M3U8's MIME type
The new MIME type appears in the following places:
https://www.iana.org/assignments/media-types/media-types.xhtml#application
https://hg.python.org/cpython/file/tip/Lib/mimetypes.py
---
youtube_dl/extractor/letv.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/letv.py b/youtube_dl/extractor/letv.py
index 9ebbc8089..effd9eb92 100644
--- a/youtube_dl/extractor/letv.py
+++ b/youtube_dl/extractor/letv.py
@@ -162,7 +162,7 @@ class LetvIE(InfoExtractor):
m3u8_data = self.decrypt_m3u8(req.read())
url_info_dict = {
- 'url': encode_data_uri(m3u8_data, 'application/x-mpegURL'),
+ 'url': encode_data_uri(m3u8_data, 'application/vnd.apple.mpegurl'),
'ext': determine_ext(dispatch[format_id][1]),
'format_id': format_id,
'protocol': 'm3u8',
From dd67702a3ea007369109ee8e4b67043064e1f759 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
<jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 18 Oct 2015 14:13:06 +0200
Subject: [PATCH 044/415] [imdb] Fix extraction (fixes #7220)
---
youtube_dl/extractor/imdb.py | 29 +++++++++++++++++++----------
1 file changed, 19 insertions(+), 10 deletions(-)
diff --git a/youtube_dl/extractor/imdb.py b/youtube_dl/extractor/imdb.py
index 4bb574cf3..02e1e428e 100644
--- a/youtube_dl/extractor/imdb.py
+++ b/youtube_dl/extractor/imdb.py
@@ -4,8 +4,8 @@ import re
import json
from .common import InfoExtractor
-from ..compat import (
- compat_urlparse,
+from ..utils import (
+ qualities,
)
@@ -30,24 +30,33 @@ class ImdbIE(InfoExtractor):
descr = self._html_search_regex(
r'(?s)<span itemprop="description">(.*?)</span>',
webpage, 'description', fatal=False)
- available_formats = re.findall(
- r'case \'(?P<f_id>.*?)\' :$\s+url = \'(?P<path>.*?)\'', webpage,
- flags=re.MULTILINE)
+ player_url = 'http://www.imdb.com/video/imdb/vi%s/imdb/single' % video_id
+ player_page = self._download_webpage(
+ player_url, video_id, 'Downloading player page')
+ # the player page contains the info for the default format, we have to
+ # fetch other pages for the rest of the formats
+ extra_formats = re.findall(r'href="(?P<url>%s.*?)".*?>(?P<name>.*?)<' % re.escape(player_url), player_page)
+ format_pages = [
+ self._download_webpage(
+ f_url, video_id, 'Downloading info for %s format' % f_name)
+ for f_url, f_name in extra_formats]
+ format_pages.append(player_page)
+
+ quality = qualities(['SD', '480p', '720p'])
formats = []
- for f_id, f_path in available_formats:
- f_path = f_path.strip()
- format_page = self._download_webpage(
- compat_urlparse.urljoin(url, f_path),
- 'Downloading info for %s format' % f_id)
+ for format_page in format_pages:
json_data = self._search_regex(
r'<script[^>]+class="imdb-player-data"[^>]*?>(.*?)</script>',
format_page, 'json data', flags=re.DOTALL)
info = json.loads(json_data)
format_info = info['videoPlayerObject']['video']
+ f_id = format_info['ffname']
formats.append({
'format_id': f_id,
'url': format_info['videoInfoList'][0]['videoUrl'],
+ 'quality': quality(f_id),
})
+ self._sort_formats(formats)
return {
'id': video_id,
From b0f001a6cbd220c8b10c0ce359f17072d6347a8f Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Mon, 21 Sep 2015 15:52:36 +0100
Subject: [PATCH 045/415] [canalc2] fix info extraction
---
youtube_dl/extractor/canalc2.py | 30 ++++++++++++++++++------------
1 file changed, 18 insertions(+), 12 deletions(-)
diff --git a/youtube_dl/extractor/canalc2.py b/youtube_dl/extractor/canalc2.py
index c4fefefe4..66a9ff093 100644
--- a/youtube_dl/extractor/canalc2.py
+++ b/youtube_dl/extractor/canalc2.py
@@ -8,34 +8,40 @@ from .common import InfoExtractor
class Canalc2IE(InfoExtractor):
IE_NAME = 'canalc2.tv'
- _VALID_URL = r'http://.*?\.canalc2\.tv/video\.asp\?.*?idVideo=(?P<id>\d+)'
+ _VALID_URL = r'https?://(www\.)?canalc2\.tv/video/(?P<id>\d+)'
_TEST = {
- 'url': 'http://www.canalc2.tv/video.asp?idVideo=12163&voir=oui',
+ 'url': 'http://www.canalc2.tv/video/12163',
'md5': '060158428b650f896c542dfbb3d6487f',
'info_dict': {
'id': '12163',
'ext': 'mp4',
'title': 'Terrasses du Numérique'
+ },
+ 'params': {
+ 'skip_download': True, # Requires rtmpdump
}
}
def _real_extract(self, url):
- video_id = re.match(self._VALID_URL, url).group('id')
- # We need to set the voir field for getting the file name
- url = 'http://www.canalc2.tv/video.asp?idVideo=%s&voir=oui' % video_id
+ video_id = self._match_id(url)
webpage = self._download_webpage(url, video_id)
- file_name = self._search_regex(
- r"so\.addVariable\('file','(.*?)'\);",
- webpage, 'file name')
- video_url = 'http://vod-flash.u-strasbg.fr:8080/' + file_name
+ video_url = self._search_regex(
+ r'jwplayer\("Player"\).setup\({[^}]*file: "([^"]+)"',
+ webpage, 'video_url')
+ formats = [{'url': video_url}]
+ if video_url.startswith('rtmp://'):
+ rtmp = re.search(r'^(?P<url>rtmp://[^/]+/(?P<app>.+))/(?P<play_path>mp4:.+)$', video_url)
+ formats[0].update({
+ 'app': rtmp.group('app'),
+ 'play_path': rtmp.group('play_path'),
+ })
title = self._html_search_regex(
- r'class="evenement8">(.*?)</a>', webpage, 'title')
+ r'(?s)class="[^"]*col_description[^"]*">.*?<h3>(.*?)</h3>', webpage, 'title')
return {
'id': video_id,
- 'ext': 'mp4',
- 'url': video_url,
+ 'formats': formats,
'title': title,
}
From 6682049dee5e73b98e99e1359b959240d0920d6b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 18 Oct 2015 19:19:43 +0600
Subject: [PATCH 046/415] [canalc2] Improve rtmp extraction
---
youtube_dl/extractor/canalc2.py | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/canalc2.py b/youtube_dl/extractor/canalc2.py
index 66a9ff093..648af2e18 100644
--- a/youtube_dl/extractor/canalc2.py
+++ b/youtube_dl/extractor/canalc2.py
@@ -31,10 +31,12 @@ class Canalc2IE(InfoExtractor):
webpage, 'video_url')
formats = [{'url': video_url}]
if video_url.startswith('rtmp://'):
- rtmp = re.search(r'^(?P<url>rtmp://[^/]+/(?P<app>.+))/(?P<play_path>mp4:.+)$', video_url)
+ rtmp = re.search(r'^(?P<url>rtmp://[^/]+/(?P<app>.+/))(?P<play_path>mp4:.+)$', video_url)
formats[0].update({
+ 'url': rtmp.group('url'),
'app': rtmp.group('app'),
'play_path': rtmp.group('play_path'),
+ 'page_url': url,
})
title = self._html_search_regex(
From ef6c868f23f2fe0d493831e0d4cba71c735bd160 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 18 Oct 2015 19:23:31 +0600
Subject: [PATCH 047/415] [canalc2] Improve some regexes
---
youtube_dl/extractor/canalc2.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/youtube_dl/extractor/canalc2.py b/youtube_dl/extractor/canalc2.py
index 648af2e18..d9137e2ef 100644
--- a/youtube_dl/extractor/canalc2.py
+++ b/youtube_dl/extractor/canalc2.py
@@ -8,7 +8,7 @@ from .common import InfoExtractor
class Canalc2IE(InfoExtractor):
IE_NAME = 'canalc2.tv'
- _VALID_URL = r'https?://(www\.)?canalc2\.tv/video/(?P<id>\d+)'
+ _VALID_URL = r'https?://(?:www\.)?canalc2\.tv/video/(?P<id>\d+)'
_TEST = {
'url': 'http://www.canalc2.tv/video/12163',
@@ -27,8 +27,8 @@ class Canalc2IE(InfoExtractor):
video_id = self._match_id(url)
webpage = self._download_webpage(url, video_id)
video_url = self._search_regex(
- r'jwplayer\("Player"\).setup\({[^}]*file: "([^"]+)"',
- webpage, 'video_url')
+ r'jwplayer\((["\'])Player\1\)\.setup\({[^}]*file\s*:\s*(["\'])(?P<file>.+?)\2',
+ webpage, 'video_url', group='file')
formats = [{'url': video_url}]
if video_url.startswith('rtmp://'):
rtmp = re.search(r'^(?P<url>rtmp://[^/]+/(?P<app>.+/))(?P<play_path>mp4:.+)$', video_url)
From 14bddf35fbe8253e283042630e24b134996b2575 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 18 Oct 2015 19:23:52 +0600
Subject: [PATCH 048/415] [canalc2] Add ext
---
youtube_dl/extractor/canalc2.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/youtube_dl/extractor/canalc2.py b/youtube_dl/extractor/canalc2.py
index d9137e2ef..ba82bb2b7 100644
--- a/youtube_dl/extractor/canalc2.py
+++ b/youtube_dl/extractor/canalc2.py
@@ -34,6 +34,7 @@ class Canalc2IE(InfoExtractor):
rtmp = re.search(r'^(?P<url>rtmp://[^/]+/(?P<app>.+/))(?P<play_path>mp4:.+)$', video_url)
formats[0].update({
'url': rtmp.group('url'),
+ 'ext': 'flv',
'app': rtmp.group('app'),
'play_path': rtmp.group('play_path'),
'page_url': url,
From b1bf063503893192637f95e929d1a9147de59a7e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 18 Oct 2015 19:27:05 +0600
Subject: [PATCH 049/415] [canalc2] Extract duration
---
youtube_dl/extractor/canalc2.py | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/canalc2.py b/youtube_dl/extractor/canalc2.py
index ba82bb2b7..e326b8fbd 100644
--- a/youtube_dl/extractor/canalc2.py
+++ b/youtube_dl/extractor/canalc2.py
@@ -4,6 +4,7 @@ from __future__ import unicode_literals
import re
from .common import InfoExtractor
+from ..utils import parse_duration
class Canalc2IE(InfoExtractor):
@@ -42,9 +43,13 @@ class Canalc2IE(InfoExtractor):
title = self._html_search_regex(
r'(?s)class="[^"]*col_description[^"]*">.*?<h3>(.*?)</h3>', webpage, 'title')
+ duration = parse_duration(self._search_regex(
+ r'id=["\']video_duree["\'][^>]*>([^<]+)',
+ webpage, 'duration', fatal=False))
return {
'id': video_id,
- 'formats': formats,
'title': title,
+ 'duration': duration,
+ 'formats': formats,
}
From 608945d44a7e47fa5115295839c993af545936eb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 18 Oct 2015 19:27:22 +0600
Subject: [PATCH 050/415] [canalc2] Fix test
---
youtube_dl/extractor/canalc2.py | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/youtube_dl/extractor/canalc2.py b/youtube_dl/extractor/canalc2.py
index e326b8fbd..f6a1ff381 100644
--- a/youtube_dl/extractor/canalc2.py
+++ b/youtube_dl/extractor/canalc2.py
@@ -16,8 +16,9 @@ class Canalc2IE(InfoExtractor):
'md5': '060158428b650f896c542dfbb3d6487f',
'info_dict': {
'id': '12163',
- 'ext': 'mp4',
- 'title': 'Terrasses du Numérique'
+ 'ext': 'flv',
+ 'title': 'Terrasses du Numérique',
+ 'duration': 122,
},
'params': {
'skip_download': True, # Requires rtmpdump
From dedd35c6bc33eb88f19b16eeb37498cee076c47a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 18 Oct 2015 19:59:18 +0600
Subject: [PATCH 051/415] [viewster] Fix failing m3u8
---
youtube_dl/extractor/viewster.py | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/youtube_dl/extractor/viewster.py b/youtube_dl/extractor/viewster.py
index 632e57fb4..7cf930d69 100644
--- a/youtube_dl/extractor/viewster.py
+++ b/youtube_dl/extractor/viewster.py
@@ -131,10 +131,11 @@ class ViewsterIE(InfoExtractor):
formats.extend(self._extract_f4m_formats(
video_url, video_id, f4m_id='hds'))
elif ext == 'm3u8':
- formats.extend(self._extract_m3u8_formats(
+ m3u8_formats = self._extract_m3u8_formats(
video_url, video_id, 'mp4', m3u8_id='hls',
- fatal=False # m3u8 sometimes fail
- ))
+ fatal=False) # m3u8 sometimes fail
+ if m3u8_formats:
+ formats.extend(m3u8_formats)
else:
format_id = media.get('Bitrate')
f = {
From e36963e0eb57294f156a98c38df891dec41ebaa4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 18 Oct 2015 20:24:33 +0600
Subject: [PATCH 052/415] [eagleplatform] Identify hls formats
---
youtube_dl/extractor/eagleplatform.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/eagleplatform.py b/youtube_dl/extractor/eagleplatform.py
index e529b9b96..7bbf617d4 100644
--- a/youtube_dl/extractor/eagleplatform.py
+++ b/youtube_dl/extractor/eagleplatform.py
@@ -87,7 +87,7 @@ class EaglePlatformIE(InfoExtractor):
m3u8_url = self._get_video_url(secure_m3u8, video_id, 'Downloading m3u8 JSON')
formats = self._extract_m3u8_formats(
m3u8_url, video_id,
- 'mp4', entry_protocol='m3u8_native')
+ 'mp4', entry_protocol='m3u8_native', m3u8_id='hls')
mp4_url = self._get_video_url(
# Secure mp4 URL is constructed according to Player.prototype.mp4 from
From a6e0afa2bbc93d145b31911b8ce40c502994e2a1 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 18 Oct 2015 19:23:40 +0200
Subject: [PATCH 053/415] release 2015.10.18
---
docs/supportedsites.md | 3 ++-
youtube_dl/version.py | 2 +-
2 files changed, 3 insertions(+), 2 deletions(-)
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 47f7da86d..cfa665d88 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -588,7 +588,8 @@
- **twitch:stream**
- **twitch:video**
- **twitch:vod**
- - **TwitterCard**
+ - **twitter**
+ - **twitter:card**
- **Ubu**
- **udemy**
- **udemy:course**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 31d2a9dc0..660b0050b 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
from __future__ import unicode_literals
-__version__ = '2015.10.16'
+__version__ = '2015.10.18'
From 264b23e1a42378d52f8774a07c1d906cd1cff96c Mon Sep 17 00:00:00 2001
From: kennell <kevin@fileperms.org>
Date: Sun, 18 Oct 2015 19:56:22 +0200
Subject: [PATCH 054/415] adds thumbnail support for ZDF Mediathek extractor
---
youtube_dl/extractor/zdf.py | 19 +++++++++++++++++++
1 file changed, 19 insertions(+)
diff --git a/youtube_dl/extractor/zdf.py b/youtube_dl/extractor/zdf.py
index 98f15177b..f376025e1 100644
--- a/youtube_dl/extractor/zdf.py
+++ b/youtube_dl/extractor/zdf.py
@@ -70,6 +70,23 @@ def extract_from_xml_url(ie, video_id, xml_url):
'_available': is_available,
}
+ def xml_to_thumbnails(fnode):
+ thumbnails = list()
+ for node in fnode:
+ width_x_height = node.attrib['key']
+ thumbnail = {
+ 'url': node.text,
+ 'width': int(width_x_height.split('x')[0]),
+ 'height': int(width_x_height.split('x')[1])
+ }
+ thumbnails.append(thumbnail)
+ return thumbnails
+
+
+ thumbnail_nodes = doc.findall('.//teaserimages/teaserimage')
+ thumbnails = xml_to_thumbnails(thumbnail_nodes)
+ thumbnail = thumbnails[-1]['url']
+
format_nodes = doc.findall('.//formitaeten/formitaet')
formats = list(filter(
lambda f: f['_available'],
@@ -81,6 +98,8 @@ def extract_from_xml_url(ie, video_id, xml_url):
'title': title,
'description': description,
'duration': duration,
+ 'thumbnail': thumbnail,
+ 'thumbnails': thumbnails,
'uploader': uploader,
'uploader_id': uploader_id,
'upload_date': upload_date,
From d762f86e940ad656e8f7e7b93636292e4cf36de5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 19 Oct 2015 00:11:16 +0600
Subject: [PATCH 055/415] [ok] Extend _VALID_URL
---
youtube_dl/extractor/odnoklassniki.py | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/odnoklassniki.py b/youtube_dl/extractor/odnoklassniki.py
index ccc88cfb1..184c7a323 100644
--- a/youtube_dl/extractor/odnoklassniki.py
+++ b/youtube_dl/extractor/odnoklassniki.py
@@ -13,7 +13,7 @@ from ..utils import (
class OdnoklassnikiIE(InfoExtractor):
- _VALID_URL = r'https?://(?:www\.)?(?:odnoklassniki|ok)\.ru/(?:video|web-api/video/moviePlayer)/(?P<id>[\d-]+)'
+ _VALID_URL = r'https?://(?:www\.)?(?:odnoklassniki|ok)\.ru/(?:video(?:embed)?|web-api/video/moviePlayer)/(?P<id>[\d-]+)'
_TESTS = [{
# metadata in JSON
'url': 'http://ok.ru/video/20079905452',
@@ -66,6 +66,9 @@ class OdnoklassnikiIE(InfoExtractor):
}, {
'url': 'http://www.ok.ru/video/20648036891',
'only_matching': True,
+ }, {
+ 'url': 'http://www.ok.ru/videoembed/20648036891',
+ 'only_matching': True,
}]
def _real_extract(self, url):
From 8cc83d301dd0e8029aff804e362860d36e3d7e7a Mon Sep 17 00:00:00 2001
From: kennell <kevin@fileperms.org>
Date: Sun, 18 Oct 2015 20:47:42 +0200
Subject: [PATCH 056/415] use int_or_none, check if attrib exists, remove
thumbnail
---
youtube_dl/extractor/zdf.py | 13 +++++--------
1 file changed, 5 insertions(+), 8 deletions(-)
diff --git a/youtube_dl/extractor/zdf.py b/youtube_dl/extractor/zdf.py
index f376025e1..d41c4e712 100644
--- a/youtube_dl/extractor/zdf.py
+++ b/youtube_dl/extractor/zdf.py
@@ -73,19 +73,17 @@ def extract_from_xml_url(ie, video_id, xml_url):
def xml_to_thumbnails(fnode):
thumbnails = list()
for node in fnode:
- width_x_height = node.attrib['key']
- thumbnail = {
- 'url': node.text,
- 'width': int(width_x_height.split('x')[0]),
- 'height': int(width_x_height.split('x')[1])
- }
+ thumbnail = {'url': node.text}
+ if 'key' in node.attrib:
+ width_x_height = node.attrib['key']
+ thumbnail['width'] = int_or_none(width_x_height.split('x')[0])
+ thumbnail['height'] = int_or_none(width_x_height.split('x')[1])
thumbnails.append(thumbnail)
return thumbnails
thumbnail_nodes = doc.findall('.//teaserimages/teaserimage')
thumbnails = xml_to_thumbnails(thumbnail_nodes)
- thumbnail = thumbnails[-1]['url']
format_nodes = doc.findall('.//formitaeten/formitaet')
formats = list(filter(
@@ -98,7 +96,6 @@ def extract_from_xml_url(ie, video_id, xml_url):
'title': title,
'description': description,
'duration': duration,
- 'thumbnail': thumbnail,
'thumbnails': thumbnails,
'uploader': uploader,
'uploader_id': uploader_id,
From b243340f0ce311443a15a2dfd4356a9504e18c04 Mon Sep 17 00:00:00 2001
From: kennell <kevin@fileperms.org>
Date: Sun, 18 Oct 2015 21:07:52 +0200
Subject: [PATCH 057/415] check if key attrib matches resolution pattern
---
youtube_dl/extractor/zdf.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/youtube_dl/extractor/zdf.py b/youtube_dl/extractor/zdf.py
index d41c4e712..ed385450c 100644
--- a/youtube_dl/extractor/zdf.py
+++ b/youtube_dl/extractor/zdf.py
@@ -75,9 +75,9 @@ def extract_from_xml_url(ie, video_id, xml_url):
for node in fnode:
thumbnail = {'url': node.text}
if 'key' in node.attrib:
- width_x_height = node.attrib['key']
- thumbnail['width'] = int_or_none(width_x_height.split('x')[0])
- thumbnail['height'] = int_or_none(width_x_height.split('x')[1])
+ if re.match("^[0-9]+x[0-9]+$", node.attrib['key']):
+ thumbnail['width'] = int_or_none(node.attrib['key'].split('x')[0])
+ thumbnail['height'] = int_or_none(node.attrib['key'].split('x')[1])
thumbnails.append(thumbnail)
return thumbnails
From 2038ad6ee71c842420b83cb6c5ce3c6898e8e380 Mon Sep 17 00:00:00 2001
From: "Sergey M." <dstftw@gmail.com>
Date: Mon, 19 Oct 2015 01:12:41 +0600
Subject: [PATCH 058/415] [README.md] Add uploader extraction sample in example
extractor
---
README.md | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/README.md b/README.md
index cf4aebf3d..a6ec9619c 100644
--- a/README.md
+++ b/README.md
@@ -710,12 +710,13 @@ If you want to add support for a new site, you can follow this quick list (assum
webpage = self._download_webpage(url, video_id)
# TODO more code goes here, for example ...
- title = self._html_search_regex(r'<h1>(.*?)</h1>', webpage, 'title')
+ title = self._html_search_regex(r'<h1>(.+?)</h1>', webpage, 'title')
return {
'id': video_id,
'title': title,
'description': self._og_search_description(webpage),
+ 'uploader': self._search_regex(r'<div[^>]+id="uploader"[^>]*>([^<]+)<', webpage, 'uploader', fatal=False),
# TODO more properties (see youtube_dl/extractor/common.py)
}
```
From b7cedb16043c60d4032b206a83539acbd39f994f Mon Sep 17 00:00:00 2001
From: kennell <kevin@fileperms.org>
Date: Sun, 18 Oct 2015 21:25:26 +0200
Subject: [PATCH 059/415] simplify thumbnail dict building
---
youtube_dl/extractor/zdf.py | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/youtube_dl/extractor/zdf.py b/youtube_dl/extractor/zdf.py
index ed385450c..c2b196504 100644
--- a/youtube_dl/extractor/zdf.py
+++ b/youtube_dl/extractor/zdf.py
@@ -75,9 +75,10 @@ def extract_from_xml_url(ie, video_id, xml_url):
for node in fnode:
thumbnail = {'url': node.text}
if 'key' in node.attrib:
- if re.match("^[0-9]+x[0-9]+$", node.attrib['key']):
- thumbnail['width'] = int_or_none(node.attrib['key'].split('x')[0])
- thumbnail['height'] = int_or_none(node.attrib['key'].split('x')[1])
+ m = re.match('^([0-9]+)x([0-9]+)$', node.attrib['key'])
+ if m:
+ thumbnail['width'] = int(m.group(1))
+ thumbnail['height'] = int(m.group(2))
thumbnails.append(thumbnail)
return thumbnails
From 7b091c370c0f187545df8b1b1cc990fcf95df108 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 19 Oct 2015 01:48:05 +0600
Subject: [PATCH 060/415] [zdf] Modernize and PEP 8
---
youtube_dl/extractor/zdf.py | 43 +++++++++++++++++++------------------
1 file changed, 22 insertions(+), 21 deletions(-)
diff --git a/youtube_dl/extractor/zdf.py b/youtube_dl/extractor/zdf.py
index c2b196504..a795f56b3 100644
--- a/youtube_dl/extractor/zdf.py
+++ b/youtube_dl/extractor/zdf.py
@@ -9,6 +9,7 @@ from ..utils import (
int_or_none,
unified_strdate,
OnDemandPagedList,
+ xpath_text,
)
@@ -19,13 +20,11 @@ def extract_from_xml_url(ie, video_id, xml_url):
errnote='Failed to download video info')
title = doc.find('.//information/title').text
- description = doc.find('.//information/detail').text
- duration = int(doc.find('.//details/lengthSec').text)
- uploader_node = doc.find('.//details/originChannelTitle')
- uploader = None if uploader_node is None else uploader_node.text
- uploader_id_node = doc.find('.//details/originChannelId')
- uploader_id = None if uploader_id_node is None else uploader_id_node.text
- upload_date = unified_strdate(doc.find('.//details/airtime').text)
+ description = xpath_text(doc, './/information/detail', 'description')
+ duration = int_or_none(xpath_text(doc, './/details/lengthSec', 'duration'))
+ uploader = xpath_text(doc, './/details/originChannelTitle', 'uploader')
+ uploader_id = xpath_text(doc, './/details/originChannelId', 'uploader id')
+ upload_date = unified_strdate(xpath_text(doc, './/details/airtime', 'upload date'))
def xml_to_format(fnode):
video_url = fnode.find('url').text
@@ -40,15 +39,14 @@ def extract_from_xml_url(ie, video_id, xml_url):
ext = format_m.group('container')
proto = format_m.group('proto').lower()
- quality = fnode.find('./quality').text
- abr = int(fnode.find('./audioBitrate').text) // 1000
- vbr_node = fnode.find('./videoBitrate')
- vbr = None if vbr_node is None else int(vbr_node.text) // 1000
+ quality = xpath_text(fnode, './quality', 'quality')
+ abr = int_or_none(xpath_text(fnode, './audioBitrate', 'abr'), 1000)
+ vbr = int_or_none(xpath_text(fnode, './videoBitrate', 'vbr'), 1000)
- width_node = fnode.find('./width')
- width = None if width_node is None else int_or_none(width_node.text)
- height_node = fnode.find('./height')
- height = None if height_node is None else int_or_none(height_node.text)
+ width = int_or_none(xpath_text(fnode, './width', 'width'))
+ height = int_or_none(xpath_text(fnode, './height', 'height'))
+
+ filesize = int_or_none(xpath_text(fnode, './filesize', 'filesize'))
format_note = ''
if not format_note:
@@ -64,16 +62,21 @@ def extract_from_xml_url(ie, video_id, xml_url):
'vbr': vbr,
'width': width,
'height': height,
- 'filesize': int_or_none(fnode.find('./filesize').text),
+ 'filesize': filesize,
'format_note': format_note,
'protocol': proto,
'_available': is_available,
}
def xml_to_thumbnails(fnode):
- thumbnails = list()
+ thumbnails = []
for node in fnode:
- thumbnail = {'url': node.text}
+ thumbnail_url = node.text
+ if not thumbnail_url:
+ continue
+ thumbnail = {
+ 'url': thumbnail_url,
+ }
if 'key' in node.attrib:
m = re.match('^([0-9]+)x([0-9]+)$', node.attrib['key'])
if m:
@@ -82,9 +85,7 @@ def extract_from_xml_url(ie, video_id, xml_url):
thumbnails.append(thumbnail)
return thumbnails
-
- thumbnail_nodes = doc.findall('.//teaserimages/teaserimage')
- thumbnails = xml_to_thumbnails(thumbnail_nodes)
+ thumbnails = xml_to_thumbnails(doc.findall('.//teaserimages/teaserimage'))
format_nodes = doc.findall('.//formitaeten/formitaet')
formats = list(filter(
From 0be30bafa42dbfa99644a9eb7fefa5cebb70f121 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Lalinsk=C3=BD?= <lukas@oxygene.sk>
Date: Mon, 19 Oct 2015 20:53:27 +0200
Subject: [PATCH 061/415] [vidme] Stream URL fallback, better error message for
suspended videos
---
youtube_dl/extractor/vidme.py | 37 +++++++++++++++++++++++++++++++++--
1 file changed, 35 insertions(+), 2 deletions(-)
diff --git a/youtube_dl/extractor/vidme.py b/youtube_dl/extractor/vidme.py
index 382517a4a..393970a12 100644
--- a/youtube_dl/extractor/vidme.py
+++ b/youtube_dl/extractor/vidme.py
@@ -97,6 +97,31 @@ class VidmeIE(InfoExtractor):
# nsfw, user-disabled
'url': 'https://vid.me/dzGJ',
'only_matching': True,
+ }, {
+ # suspended
+ 'url': 'https://vid.me/Ox3G',
+ 'only_matching': True,
+ }, {
+ # no formats in the API response
+ 'url': 'https://vid.me/e5g',
+ 'info_dict': {
+ 'id': 'e5g',
+ 'ext': 'mp4',
+ 'title': 'e5g',
+ 'thumbnail': 're:^https?://.*\.jpg',
+ 'timestamp': 1401480195,
+ 'upload_date': '20140530',
+ 'uploader': None,
+ 'uploader_id': None,
+ 'age_limit': 0,
+ 'duration': 483,
+ 'view_count': int,
+ 'like_count': int,
+ 'comment_count': int,
+ },
+ 'params': {
+ 'skip_download': True,
+ },
}]
def _real_extract(self, url):
@@ -118,7 +143,7 @@ class VidmeIE(InfoExtractor):
video = response['video']
- if video.get('state') == 'user-disabled':
+ if video.get('state') in ('user-disabled', 'suspended'):
raise ExtractorError(
'Vidme said: This video has been suspended either due to a copyright claim, '
'or for violating the terms of use.',
@@ -131,6 +156,14 @@ class VidmeIE(InfoExtractor):
'height': int_or_none(f.get('height')),
'preference': 0 if f.get('type', '').endswith('clip') else 1,
} for f in video.get('formats', []) if f.get('uri')]
+
+ if not formats and video.get('complete_url'):
+ formats.append({
+ 'url': video.get('complete_url'),
+ 'width': int_or_none(video.get('width')),
+ 'height': int_or_none(video.get('height')),
+ })
+
self._sort_formats(formats)
title = video['title']
@@ -147,7 +180,7 @@ class VidmeIE(InfoExtractor):
return {
'id': video_id,
- 'title': title,
+ 'title': title or video_id,
'description': description,
'thumbnail': thumbnail,
'uploader': uploader,
From 4bf56141950f3c24000381403417d20095f04460 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 20 Oct 2015 07:43:39 +0100
Subject: [PATCH 062/415] [cspan] move get_text_attr to CSpanIE
---
youtube_dl/extractor/cspan.py | 21 ++++++++++-----------
1 file changed, 10 insertions(+), 11 deletions(-)
diff --git a/youtube_dl/extractor/cspan.py b/youtube_dl/extractor/cspan.py
index c74b35fd9..388460a32 100644
--- a/youtube_dl/extractor/cspan.py
+++ b/youtube_dl/extractor/cspan.py
@@ -14,10 +14,6 @@ from ..utils import (
from .senateisvp import SenateISVPIE
-def get_text_attr(d, attr):
- return d.get(attr, {}).get('#text')
-
-
class CSpanIE(InfoExtractor):
_VALID_URL = r'http://(?:www\.)?c-span\.org/video/\?(?P<id>[0-9a-f]+)'
IE_DESC = 'C-SPAN'
@@ -60,6 +56,9 @@ class CSpanIE(InfoExtractor):
}
}]
+ def get_text_attr(self, d, attr):
+ return d.get(attr, {}).get('#text')
+
def _real_extract(self, url):
video_id = self._match_id(url)
webpage = self._download_webpage(url, video_id)
@@ -79,7 +78,7 @@ class CSpanIE(InfoExtractor):
'http://www.c-span.org/assets/player/ajax-player.php?os=android&html5=%s&id=%s' % (video_type, video_id),
video_id)['video']
if data['@status'] != 'Success':
- raise ExtractorError('%s said: %s' % (self.IE_NAME, get_text_attr(data, 'error')), expected=True)
+ raise ExtractorError('%s said: %s' % (self.IE_NAME, self.get_text_attr(data, 'error')), expected=True)
doc = self._download_xml(
'http://www.c-span.org/common/services/flashXml.php?%sid=%s' % (video_type, video_id),
@@ -91,17 +90,17 @@ class CSpanIE(InfoExtractor):
thumbnail = find_xpath_attr(doc, './/string', 'name', 'poster').text
files = data['files']
- capfile = get_text_attr(data, 'capfile')
+ capfile = self.get_text_attr(data, 'capfile')
entries = []
for partnum, f in enumerate(files):
formats = []
for quality in f['qualities']:
formats.append({
- 'format_id': '%s-%sp' % (get_text_attr(quality, 'bitrate'), get_text_attr(quality, 'height')),
- 'url': unescapeHTML(get_text_attr(quality, 'file')),
- 'height': int_or_none(get_text_attr(quality, 'height')),
- 'tbr': int_or_none(get_text_attr(quality, 'bitrate')),
+ 'format_id': '%s-%sp' % (self.get_text_attr(quality, 'bitrate'), self.get_text_attr(quality, 'height')),
+ 'url': unescapeHTML(self.get_text_attr(quality, 'file')),
+ 'height': int_or_none(self.get_text_attr(quality, 'height')),
+ 'tbr': int_or_none(self.get_text_attr(quality, 'bitrate')),
})
self._sort_formats(formats)
entries.append({
@@ -112,7 +111,7 @@ class CSpanIE(InfoExtractor):
'formats': formats,
'description': description,
'thumbnail': thumbnail,
- 'duration': int_or_none(get_text_attr(f, 'length')),
+ 'duration': int_or_none(self.get_text_attr(f, 'length')),
'subtitles': {
'en': [{
'url': capfile,
From b6aa99aff8278142fed94e37e500f1cfb62defd1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Lalinsk=C3=BD?= <lukas@oxygene.sk>
Date: Tue, 20 Oct 2015 10:30:31 +0200
Subject: [PATCH 063/415] [vimeo] Fix error parsing
---
youtube_dl/extractor/vimeo.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 0f84656c0..bdec79341 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -273,13 +273,13 @@ class VimeoIE(VimeoBaseInfoExtractor):
self.report_extraction(video_id)
vimeo_config = self._search_regex(
- r'vimeo\.config\s*=\s*({.+?});', webpage,
+ r'vimeo\.config\s*=\s*(?:({.+?})|_extend\([^,]+,\s+({.+?})\));', webpage,
'vimeo config', default=None)
if vimeo_config:
seed_status = self._parse_json(vimeo_config, video_id).get('seed_status', {})
if seed_status.get('state') == 'failed':
raise ExtractorError(
- '%s returned error: %s' % (self.IE_NAME, seed_status['title']),
+ '%s said: %s' % (self.IE_NAME, seed_status['title']),
expected=True)
# Extract the config JSON
From 4a8963770e37568c484841338cbb6761cf3cb5c3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 20 Oct 2015 20:17:54 +0600
Subject: [PATCH 064/415] [vidme] Use original vid.me title template for
untitled videos
---
youtube_dl/extractor/vidme.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/youtube_dl/extractor/vidme.py b/youtube_dl/extractor/vidme.py
index 393970a12..296e00423 100644
--- a/youtube_dl/extractor/vidme.py
+++ b/youtube_dl/extractor/vidme.py
@@ -107,7 +107,7 @@ class VidmeIE(InfoExtractor):
'info_dict': {
'id': 'e5g',
'ext': 'mp4',
- 'title': 'e5g',
+ 'title': 'Video upload (e5g)',
'thumbnail': 're:^https?://.*\.jpg',
'timestamp': 1401480195,
'upload_date': '20140530',
@@ -180,7 +180,7 @@ class VidmeIE(InfoExtractor):
return {
'id': video_id,
- 'title': title or video_id,
+ 'title': title or 'Video upload (%s)' % video_id,
'description': description,
'thumbnail': thumbnail,
'uploader': uploader,
From d65889bbc0a6b4a1eafe6a8c0e0e26170dc75586 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 20 Oct 2015 20:18:23 +0600
Subject: [PATCH 065/415] [vidme] Update test
---
youtube_dl/extractor/vidme.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/vidme.py b/youtube_dl/extractor/vidme.py
index 296e00423..eb5cde761 100644
--- a/youtube_dl/extractor/vidme.py
+++ b/youtube_dl/extractor/vidme.py
@@ -14,7 +14,7 @@ class VidmeIE(InfoExtractor):
_VALID_URL = r'https?://vid\.me/(?:e/)?(?P<id>[\da-zA-Z]+)'
_TESTS = [{
'url': 'https://vid.me/QNB',
- 'md5': 'c62f1156138dc3323902188c5b5a8bd6',
+ 'md5': 'f42d05e7149aeaec5c037b17e5d3dc82',
'info_dict': {
'id': 'QNB',
'ext': 'mp4',
From 8bea039b8329074af9a95fe51e7622c8074f6218 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Lalinsk=C3=BD?= <lukas@oxygene.sk>
Date: Tue, 20 Oct 2015 16:38:44 +0200
Subject: [PATCH 066/415] [vimeo] New test, fixed one older test
---
youtube_dl/extractor/vimeo.py | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index bdec79341..2437ae1eb 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -133,7 +133,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
'uploader_id': 'user18948128',
'uploader': 'Jaime Marquínez Ferrándiz',
'duration': 10,
- 'description': 'This is "youtube-dl password protected test video" by Jaime Marquínez Ferrándiz on Vimeo, the home for high quality videos and the people who love them.',
+ 'description': 'This is "youtube-dl password protected test video" by Jaime Marquínez Ferrándiz on Vimeo, the home for high quality videos and the people\u2026',
},
'params': {
'videopassword': 'youtube-dl',
@@ -181,6 +181,11 @@ class VimeoIE(VimeoBaseInfoExtractor):
'uploader_id': 'user28849593',
},
},
+ {
+ 'url': 'https://vimeo.com/109815029',
+ 'note': 'Video not completely processed, "failed" seed status',
+ 'only_matching': True,
+ },
]
@staticmethod
From d01949dc89feb2441f251e42e8a6bfa4711b9715 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 20 Oct 2015 23:09:51 +0600
Subject: [PATCH 067/415] [utils:js_to_json] Fix bad escape in double quoted
strings
---
test/test_utils.py | 3 +++
youtube_dl/utils.py | 4 ++--
2 files changed, 5 insertions(+), 2 deletions(-)
diff --git a/test/test_utils.py b/test/test_utils.py
index a5f164c49..918a7a9ef 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -495,6 +495,9 @@ class TestUtil(unittest.TestCase):
"playlist":[{"controls":{"all":null}}]
}''')
+ inp = '''"The CW\\'s \\'Crazy Ex-Girlfriend\\'"'''
+ self.assertEqual(js_to_json(inp), '''"The CW's 'Crazy Ex-Girlfriend'"''')
+
inp = '"SAND Number: SAND 2013-7800P\\nPresenter: Tom Russo\\nHabanero Software Training - Xyce Software\\nXyce, Sandia\\u0027s"'
json_code = js_to_json(inp)
self.assertEqual(json.loads(json_code), json.loads(inp))
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index db5b3698e..a61e47646 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1701,8 +1701,8 @@ def js_to_json(code):
if v in ('true', 'false', 'null'):
return v
if v.startswith('"'):
- return v
- if v.startswith("'"):
+ v = re.sub(r"\\'", "'", v[1:-1])
+ elif v.startswith("'"):
v = v[1:-1]
v = re.sub(r"\\\\|\\'|\"", lambda m: {
'\\\\': '\\\\',
From 4211c83aa4dec0cf9874a6a485665360570e2a89 Mon Sep 17 00:00:00 2001
From: mjdubell <localhost@localhost>
Date: Mon, 19 Oct 2015 03:36:07 +0200
Subject: [PATCH 068/415] [stitcher] Add extractor
Stitcher review updates
Removed re import
Stitcher review updates
---
youtube_dl/extractor/__init__.py | 1 +
youtube_dl/extractor/stitcher.py | 37 ++++++++++++++++++++++++++++++++
2 files changed, 38 insertions(+)
create mode 100644 youtube_dl/extractor/stitcher.py
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index bd6eb6ae0..eac5e7d5e 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -586,6 +586,7 @@ from .spankwire import SpankwireIE
from .spiegel import SpiegelIE, SpiegelArticleIE
from .spiegeltv import SpiegeltvIE
from .spike import SpikeIE
+from .stitcher import StitcherIE
from .sport5 import Sport5IE
from .sportbox import (
SportBoxIE,
diff --git a/youtube_dl/extractor/stitcher.py b/youtube_dl/extractor/stitcher.py
new file mode 100644
index 000000000..a547debbd
--- /dev/null
+++ b/youtube_dl/extractor/stitcher.py
@@ -0,0 +1,37 @@
+# coding: utf-8
+from __future__ import unicode_literals
+from .common import InfoExtractor
+from ..utils import int_or_none
+
+
+class StitcherIE(InfoExtractor):
+ _VALID_URL = r'https?://(?:www\.)?stitcher\.com/podcast/[\/a-z\-]+(?P<id>\d+)'
+ _TEST = {
+ 'url': 'http://www.stitcher.com/podcast/the-talking-machines/e/40789481?autoplay=true',
+ 'md5': '391dd4e021e6edeb7b8e68fbf2e9e940',
+ 'info_dict': {
+ 'id': '40789481',
+ 'ext': 'mp3',
+ 'title': 'Machine Learning Mastery and Cancer Clusters from Talking Machines',
+ }
+ }
+
+ def _real_extract(self, url):
+ audio_id = self._match_id(url)
+
+ webpage = self._download_webpage(url, audio_id)
+
+ title = self._og_search_title(webpage)
+ url = self._search_regex(r'episodeURL: "(.+?)"', webpage, 'url')
+ episode_image = self._search_regex(r'episodeImage: "(.+?)"', webpage, 'episode_image', fatal=False)
+ duration = int_or_none(self._search_regex(r'duration: (\d+?),', webpage, 'duration', fatal=False))
+
+ return {
+ 'id': audio_id,
+ 'url': url,
+ 'title': title,
+ 'duration': duration,
+ 'thumbnail': episode_image,
+ 'ext': 'mp3',
+ 'vcodec': 'none',
+ }
From 7308b8cb3df5a2df0a86e8050c83b951004a0aca Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 20 Oct 2015 23:12:13 +0600
Subject: [PATCH 069/415] [stitcher] Improve (Closes #7162, closes #7228)
---
youtube_dl/extractor/stitcher.py | 78 +++++++++++++++++++++++++-------
1 file changed, 61 insertions(+), 17 deletions(-)
diff --git a/youtube_dl/extractor/stitcher.py b/youtube_dl/extractor/stitcher.py
index a547debbd..971a1c466 100644
--- a/youtube_dl/extractor/stitcher.py
+++ b/youtube_dl/extractor/stitcher.py
@@ -1,37 +1,81 @@
-# coding: utf-8
from __future__ import unicode_literals
+
+import re
+
from .common import InfoExtractor
-from ..utils import int_or_none
+from ..utils import (
+ determine_ext,
+ int_or_none,
+ js_to_json,
+ unescapeHTML,
+)
class StitcherIE(InfoExtractor):
- _VALID_URL = r'https?://(?:www\.)?stitcher\.com/podcast/[\/a-z\-]+(?P<id>\d+)'
- _TEST = {
+ _VALID_URL = r'https?://(?:www\.)?stitcher\.com/podcast/(?:[^/]+/)+e/(?:(?P<display_id>[^/#?&]+?)-)?(?P<id>\d+)(?:[/#?&]|$)'
+ _TESTS = [{
'url': 'http://www.stitcher.com/podcast/the-talking-machines/e/40789481?autoplay=true',
'md5': '391dd4e021e6edeb7b8e68fbf2e9e940',
'info_dict': {
'id': '40789481',
'ext': 'mp3',
- 'title': 'Machine Learning Mastery and Cancer Clusters from Talking Machines',
- }
- }
+ 'title': 'Machine Learning Mastery and Cancer Clusters',
+ 'description': 'md5:55163197a44e915a14a1ac3a1de0f2d3',
+ 'duration': 1604,
+ 'thumbnail': 're:^https?://.*\.jpg',
+ },
+ }, {
+ 'url': 'http://www.stitcher.com/podcast/panoply/vulture-tv/e/the-rare-hourlong-comedy-plus-40846275?autoplay=true',
+ 'info_dict': {
+ 'id': '40846275',
+ 'display_id': 'the-rare-hourlong-comedy-plus',
+ 'ext': 'mp3',
+ 'title': "The CW's 'Crazy Ex-Girlfriend'",
+ 'description': 'md5:04f1e2f98eb3f5cbb094cea0f9e19b17',
+ 'duration': 2235,
+ 'thumbnail': 're:^https?://.*\.jpg',
+ },
+ 'params': {
+ 'skip_download': True,
+ },
+ }, {
+ # escaped title
+ 'url': 'http://www.stitcher.com/podcast/marketplace-on-stitcher/e/40910226?autoplay=true',
+ 'only_matching': True,
+ }, {
+ 'url': 'http://www.stitcher.com/podcast/panoply/getting-in/e/episode-2a-how-many-extracurriculars-should-i-have-40876278?autoplay=true',
+ 'only_matching': True,
+ }]
def _real_extract(self, url):
- audio_id = self._match_id(url)
+ mobj = re.match(self._VALID_URL, url)
+ audio_id = mobj.group('id')
+ display_id = mobj.group('display_id') or audio_id
- webpage = self._download_webpage(url, audio_id)
+ webpage = self._download_webpage(url, display_id)
- title = self._og_search_title(webpage)
- url = self._search_regex(r'episodeURL: "(.+?)"', webpage, 'url')
- episode_image = self._search_regex(r'episodeImage: "(.+?)"', webpage, 'episode_image', fatal=False)
- duration = int_or_none(self._search_regex(r'duration: (\d+?),', webpage, 'duration', fatal=False))
+ episode = self._parse_json(
+ js_to_json(self._search_regex(
+ r'(?s)var\s+stitcher\s*=\s*({.+?});\n', webpage, 'episode config')),
+ display_id)['config']['episode']
+
+ title = unescapeHTML(episode['title'])
+ formats = [{
+ 'url': episode[episode_key],
+ 'ext': determine_ext(episode[episode_key]) or 'mp3',
+ 'vcodec': 'none',
+ } for episode_key in ('origEpisodeURL', 'episodeURL') if episode.get(episode_key)]
+ description = self._search_regex(
+ r'Episode Info:\s*</span>([^<]+)<', webpage, 'description', fatal=False)
+ duration = int_or_none(episode.get('duration'))
+ thumbnail = episode.get('episodeImage')
return {
'id': audio_id,
- 'url': url,
+ 'display_id': display_id,
'title': title,
+ 'description': description,
'duration': duration,
- 'thumbnail': episode_image,
- 'ext': 'mp3',
- 'vcodec': 'none',
+ 'thumbnail': thumbnail,
+ 'formats': formats,
}
From cc449417c4b3835c31f89e47a5e08e0f0c42ac5a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 21 Oct 2015 20:35:22 +0600
Subject: [PATCH 070/415] [vine] Use _search_regex for JSON data (Closes #7254,
closes #7255)
---
youtube_dl/extractor/vine.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/youtube_dl/extractor/vine.py b/youtube_dl/extractor/vine.py
index be72f3147..cb2a4b0b5 100644
--- a/youtube_dl/extractor/vine.py
+++ b/youtube_dl/extractor/vine.py
@@ -85,8 +85,8 @@ class VineIE(InfoExtractor):
webpage = self._download_webpage('https://vine.co/v/' + video_id, video_id)
data = self._parse_json(
- self._html_search_regex(
- r'window\.POST_DATA = { %s: ({.+?}) };\s*</script>' % video_id,
+ self._search_regex(
+ r'window\.POST_DATA\s*=\s*{\s*%s\s*:\s*({.+?})\s*};\s*</script>' % video_id,
webpage, 'vine data'),
video_id)
From 44d6dd08b299ccf17eb04901cf09a8d333769783 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 21 Oct 2015 21:35:57 +0600
Subject: [PATCH 071/415] [facebook] Fix extraction (Closes #7252)
---
youtube_dl/extractor/facebook.py | 23 ++++++++++++-----------
1 file changed, 12 insertions(+), 11 deletions(-)
diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index 178a7ca4c..f53c51615 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -14,7 +14,6 @@ from ..compat import (
)
from ..utils import (
ExtractorError,
- int_or_none,
limit_length,
urlencode_postdata,
get_element_by_id,
@@ -142,16 +141,20 @@ class FacebookIE(InfoExtractor):
data = dict(json.loads(m.group(1)))
params_raw = compat_urllib_parse_unquote(data['params'])
params = json.loads(params_raw)
- video_data = params['video_data'][0]
formats = []
- for quality in ['sd', 'hd']:
- src = video_data.get('%s_src' % quality)
- if src is not None:
- formats.append({
- 'format_id': quality,
- 'url': src,
- })
+ for format_id, f in params['video_data'].items():
+ if not f or not isinstance(f, list):
+ continue
+ for quality in ('sd', 'hd'):
+ for src_type in ('src', 'src_no_ratelimit'):
+ src = f[0].get('%s_%s' % (quality, src_type))
+ if src:
+ formats.append({
+ 'format_id': '%s_%s_%s' % (format_id, quality, src_type),
+ 'url': src,
+ 'preference': -10 if format_id == 'progressive' else 0,
+ })
if not formats:
raise ExtractorError('Cannot find video formats')
@@ -171,7 +174,5 @@ class FacebookIE(InfoExtractor):
'id': video_id,
'title': video_title,
'formats': formats,
- 'duration': int_or_none(video_data.get('video_duration')),
- 'thumbnail': video_data.get('thumbnail_src'),
'uploader': uploader,
}
From 8c3533ba976af15ca9fac8acd68547b195dc4e8a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
<jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 21 Oct 2015 23:57:23 +0200
Subject: [PATCH 072/415] [adultswim] Don't default to the native m3u8
downloader (closes #7243)
Some of the streams are encrypted, which is not supported .
---
youtube_dl/extractor/adultswim.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/adultswim.py b/youtube_dl/extractor/adultswim.py
index 130afe791..3ae618e71 100644
--- a/youtube_dl/extractor/adultswim.py
+++ b/youtube_dl/extractor/adultswim.py
@@ -183,7 +183,7 @@ class AdultSwimIE(InfoExtractor):
media_url = file_el.text
if determine_ext(media_url) == 'm3u8':
formats.extend(self._extract_m3u8_formats(
- media_url, segment_title, 'mp4', 'm3u8_native', preference=0, m3u8_id='hls'))
+ media_url, segment_title, 'mp4', preference=0, m3u8_id='hls'))
else:
formats.append({
'format_id': '%s_%s' % (bitrate, ftype),
From 89d5fbf354cf0b49098582a19f76cef67358d375 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 22 Oct 2015 17:47:11 +0800
Subject: [PATCH 073/415] [iqiyi] Update key
---
youtube_dl/extractor/iqiyi.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/youtube_dl/extractor/iqiyi.py b/youtube_dl/extractor/iqiyi.py
index 0e53cb154..2df1da3f0 100644
--- a/youtube_dl/extractor/iqiyi.py
+++ b/youtube_dl/extractor/iqiyi.py
@@ -205,9 +205,9 @@ class IqiyiIE(InfoExtractor):
def get_enc_key(self, swf_url, video_id):
# TODO: automatic key extraction
- # last update at 2015-10-10 for Zombie::bite
- # '7239670519b6ac209a0bee4ef0446a6b24894b8ac2751506e42116212a0d0272e505'[2:66][1::2]
- enc_key = '97596c0abee04ab49ba25564161ad225'
+ # last update at 2015-10-22 for Zombie::bite
+ # '7223c67061dbea1259d0ceb44f44b6d62288f4f80c972170de5201d2321060270e05'[2:66][0::2]
+ enc_key = '2c76de15dcb44bd28ff0927d50d31620'
return enc_key
def _real_extract(self, url):
From 7033bc1a5117068c493931cb736d53e68d50f9a1 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 22 Oct 2015 21:12:29 +0800
Subject: [PATCH 074/415] [bbc] Fix test_BBC_9
---
youtube_dl/extractor/bbc.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 1b3a33e4e..ea67e3f2d 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -625,6 +625,7 @@ class BBCIE(BBCCoUkIE):
'id': 'p02xycnp',
'ext': 'mp4',
'title': 'Transfers: Cristiano Ronaldo to Man Utd, Arsenal to spend?',
+ 'description': 'BBC Sport\'s David Ornstein has the latest transfer gossip, including rumours of a Manchester United return for Cristiano Ronaldo.',
'duration': 140,
},
'params': {
From a65402ef42c42477f78469f0a6c4af1583d97a31 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 22 Oct 2015 21:13:03 +0800
Subject: [PATCH 075/415] [bbc.co.uk:article] Add new extractor (#7257)
---
youtube_dl/extractor/__init__.py | 1 +
youtube_dl/extractor/bbc.py | 34 ++++++++++++++++++++++++++++++--
2 files changed, 33 insertions(+), 2 deletions(-)
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index eac5e7d5e..6318ac4a2 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -45,6 +45,7 @@ from .bambuser import BambuserIE, BambuserChannelIE
from .bandcamp import BandcampIE, BandcampAlbumIE
from .bbc import (
BBCCoUkIE,
+ BBCCoUkArticleIE,
BBCIE,
)
from .beeg import BeegIE
diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index ea67e3f2d..2cdce1eb9 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -20,7 +20,7 @@ from ..compat import compat_HTTPError
class BBCCoUkIE(InfoExtractor):
IE_NAME = 'bbc.co.uk'
IE_DESC = 'BBC iPlayer'
- _VALID_URL = r'https?://(?:www\.)?bbc\.co\.uk/(?:(?:(?:programmes|iplayer(?:/[^/]+)?/(?:episode|playlist))/)|music/clips[/#])(?P<id>[\da-z]{8})'
+ _VALID_URL = r'https?://(?:www\.)?bbc\.co\.uk/(?:(?:programmes/(?!articles/)|iplayer(?:/[^/]+)?/(?:episode/|playlist/))|music/clips[/#])(?P<id>[\da-z]{8})'
_MEDIASELECTOR_URLS = [
# Provides HQ HLS streams with even better quality that pc mediaset but fails
@@ -652,7 +652,7 @@ class BBCIE(BBCCoUkIE):
@classmethod
def suitable(cls, url):
- return False if BBCCoUkIE.suitable(url) else super(BBCIE, cls).suitable(url)
+ return False if BBCCoUkIE.suitable(url) or BBCCoUkArticleIE.suitable(url) else super(BBCIE, cls).suitable(url)
def _extract_from_media_meta(self, media_meta, video_id):
# Direct links to media in media metadata (e.g.
@@ -903,3 +903,33 @@ class BBCIE(BBCCoUkIE):
})
return self.playlist_result(entries, playlist_id, playlist_title, playlist_description)
+
+
+class BBCCoUkArticleIE(InfoExtractor):
+ _VALID_URL = 'http://www.bbc.co.uk/programmes/articles/(?P<id>[a-zA-Z0-9]+)'
+ IE_NAME = 'bbc.co.uk:article'
+ IE_DESC = 'BBC articles'
+
+ _TEST = {
+ 'url': 'http://www.bbc.co.uk/programmes/articles/3jNQLTMrPlYGTBn0WV6M2MS/not-your-typical-role-model-ada-lovelace-the-19th-century-programmer',
+ 'info_dict': {
+ 'id': '3jNQLTMrPlYGTBn0WV6M2MS',
+ 'title': 'Calculating Ada: The Countess of Computing - Not your typical role model: Ada Lovelace the 19th century programmer - BBC Four',
+ 'description': 'Hannah Fry reveals some of her surprising discoveries about Ada Lovelace during filming.',
+ },
+ 'playlist_count': 4,
+ 'add_ie': ['BBCCoUk'],
+ }
+
+ def _real_extract(self, url):
+ playlist_id = self._match_id(url)
+
+ webpage = self._download_webpage(url, playlist_id)
+
+ title = self._og_search_title(webpage)
+ description = self._og_search_description(webpage).strip()
+
+ entries = [self.url_result(programme_url) for programme_url in re.findall(
+ r'<div[^>]+typeof="Clip"[^>]+resource="([^"]+)"', webpage)]
+
+ return self.playlist_result(entries, playlist_id, title, description)
From 769078755318896fa1a6c5c8aba6f76d6aeddf78 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 22 Oct 2015 20:34:11 +0600
Subject: [PATCH 076/415] [crunchyroll] Improve subtitle regex (Closes #7262)
---
youtube_dl/extractor/crunchyroll.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index cecd0c784..f8ce10111 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -245,7 +245,7 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
def _get_subtitles(self, video_id, webpage):
subtitles = {}
- for sub_id, sub_name in re.findall(r'\?ssid=([0-9]+)" title="([^"]+)', webpage):
+ for sub_id, sub_name in re.findall(r'\bssid=([0-9]+)"[^>]+?\btitle="([^"]+)', webpage):
sub_page = self._download_webpage(
'http://www.crunchyroll.com/xml/?req=RpcApiSubtitle_GetXml&subtitle_script_id=' + sub_id,
video_id, note='Downloading subtitles for ' + sub_name)
From ab03c0b47c142bfb649eacb3c72ce9cb67184535 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 23 Oct 2015 09:33:05 +0200
Subject: [PATCH 077/415] release 2015.10.23
---
CONTRIBUTING.md | 3 ++-
docs/supportedsites.md | 2 ++
youtube_dl/version.py | 2 +-
3 files changed, 5 insertions(+), 2 deletions(-)
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 32c2fd84c..aebded4ce 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -114,12 +114,13 @@ If you want to add support for a new site, you can follow this quick list (assum
webpage = self._download_webpage(url, video_id)
# TODO more code goes here, for example ...
- title = self._html_search_regex(r'<h1>(.*?)</h1>', webpage, 'title')
+ title = self._html_search_regex(r'<h1>(.+?)</h1>', webpage, 'title')
return {
'id': video_id,
'title': title,
'description': self._og_search_description(webpage),
+ 'uploader': self._search_regex(r'<div[^>]+id="uploader"[^>]*>([^<]+)<', webpage, 'uploader', fatal=False),
# TODO more properties (see youtube_dl/extractor/common.py)
}
```
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index cfa665d88..03561b87d 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -53,6 +53,7 @@
- **Bandcamp:album**
- **bbc**: BBC
- **bbc.co.uk**: BBC iPlayer
+ - **bbc.co.uk:article**: BBC articles
- **BeatportPro**
- **Beeg**
- **BehindKink**
@@ -515,6 +516,7 @@
- **SSA**
- **stanfordoc**: Stanford Open ClassRoom
- **Steam**
+ - **Stitcher**
- **streamcloud.eu**
- **StreamCZ**
- **StreetVoice**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 660b0050b..d5c7f338d 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
from __future__ import unicode_literals
-__version__ = '2015.10.18'
+__version__ = '2015.10.23'
From 65d49afa48086b568364bbcbab29feef71031178 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
<jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 23 Oct 2015 14:12:46 +0200
Subject: [PATCH 078/415] [test/test_download] Use extract_flat = 'in_playlist'
for playlist items
Some playlist extractors return a 'url' result, which wouldn't be resolved.
---
test/test_download.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/test/test_download.py b/test/test_download.py
index 284418834..a3f1c0644 100644
--- a/test/test_download.py
+++ b/test/test_download.py
@@ -102,7 +102,7 @@ def generator(test_case):
params = get_params(test_case.get('params', {}))
if is_playlist and 'playlist' not in test_case:
- params.setdefault('extract_flat', True)
+ params.setdefault('extract_flat', 'in_playlist')
params.setdefault('skip_download', True)
ydl = YoutubeDL(params, auto_init=False)
From 9170ca5b16f3420892ff06bbe5cccf1679eb75e4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
<jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 23 Oct 2015 14:16:08 +0200
Subject: [PATCH 079/415] [youtube:channel] Fix test
---
youtube_dl/extractor/youtube.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 08e821362..bae1b1117 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1644,7 +1644,8 @@ class YoutubeChannelIE(YoutubePlaylistBaseInfoExtractor):
'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
'playlist_mincount': 91,
'info_dict': {
- 'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
+ 'id': 'UUKfVa3S1e4PHvxWcwyMMg8w',
+ 'title': 'Uploads from lex will',
}
}]
From 5c43afd40f8ba101e0cf90b8fcb5713b378a62c9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
<jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 23 Oct 2015 14:23:45 +0200
Subject: [PATCH 080/415] [youtube:channel] Support age restricted channels
(fixes #7277)
---
youtube_dl/extractor/youtube.py | 11 ++++++++++-
1 file changed, 10 insertions(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index bae1b1117..d7eda7aa7 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1647,6 +1647,15 @@ class YoutubeChannelIE(YoutubePlaylistBaseInfoExtractor):
'id': 'UUKfVa3S1e4PHvxWcwyMMg8w',
'title': 'Uploads from lex will',
}
+ }, {
+ 'note': 'Age restricted channel',
+ # from https://www.youtube.com/user/DeusExOfficial
+ 'url': 'https://www.youtube.com/channel/UCs0ifCMCm1icqRbqhUINa0w',
+ 'playlist_mincount': 64,
+ 'info_dict': {
+ 'id': 'UUs0ifCMCm1icqRbqhUINa0w',
+ 'title': 'Uploads from Deus Ex',
+ },
}]
def _real_extract(self, url):
@@ -1667,7 +1676,7 @@ class YoutubeChannelIE(YoutubePlaylistBaseInfoExtractor):
'channelId', channel_page, 'channel id', default=None)
if not channel_playlist_id:
channel_playlist_id = self._search_regex(
- r'data-channel-external-id="([^"]+)"',
+ r'data-(?:channel-external-|yt)id="([^"]+)"',
channel_page, 'channel id', default=None)
if channel_playlist_id and channel_playlist_id.startswith('UC'):
playlist_id = 'UU' + channel_playlist_id[2:]
From edeb3e7cb1ab2d82ff7c712a7cc1e338a9dcd8f8 Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Fri, 23 Oct 2015 15:58:24 +0000
Subject: [PATCH 081/415] [README.md] Fix typo
---
README.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/README.md b/README.md
index a6ec9619c..38db97c59 100644
--- a/README.md
+++ b/README.md
@@ -795,7 +795,7 @@ Bugs and suggestions should be reported at: <https://github.com/rg3/youtube-dl/i
**Please include the full output of youtube-dl when run with `-v`**.
-The output (including the first lines) contain important debugging information. Issues without the full output are often not reproducible and therefore do not get solved in short order, if ever.
+The output (including the first lines) contains important debugging information. Issues without the full output are often not reproducible and therefore do not get solved in short order, if ever.
Please re-read your issue once again to avoid a couple of common mistakes (you can and should use this as a checklist):
From dae69640d086ca1e2683ca81b60f48a0c6c83eac Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 24 Oct 2015 00:10:28 +0200
Subject: [PATCH 082/415] Fix py2exe build (#7276)
---
setup.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/setup.py b/setup.py
index 4686260e0..bfe931f5b 100644
--- a/setup.py
+++ b/setup.py
@@ -28,7 +28,7 @@ py2exe_options = {
"compressed": 1,
"optimize": 2,
"dist_dir": '.',
- "dll_excludes": ['w9xpopen.exe'],
+ "dll_excludes": ['w9xpopen.exe', 'crypt32.dll'],
}
py2exe_console = [{
From ab9c7214ee6c831a68216caf1cd1f9f3c183e4fd Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 24 Oct 2015 00:10:41 +0200
Subject: [PATCH 083/415] release 2015.10.24
---
CONTRIBUTING.md | 2 +-
youtube_dl/version.py | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index aebded4ce..09ce98ca2 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -1,6 +1,6 @@
**Please include the full output of youtube-dl when run with `-v`**.
-The output (including the first lines) contain important debugging information. Issues without the full output are often not reproducible and therefore do not get solved in short order, if ever.
+The output (including the first lines) contains important debugging information. Issues without the full output are often not reproducible and therefore do not get solved in short order, if ever.
Please re-read your issue once again to avoid a couple of common mistakes (you can and should use this as a checklist):
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index d5c7f338d..125e8ccf5 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
from __future__ import unicode_literals
-__version__ = '2015.10.23'
+__version__ = '2015.10.24'
From c93153852f342ef26005b37649d58fa944c53fe3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
<jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 24 Oct 2015 12:10:53 +0200
Subject: [PATCH 084/415] [mitele] Don't encode the URL query (closes #7280)
This seems to produce sporadic errors when trying to access the URL, because on python 3.x when you do '%s' % b'somedata' you get "b'somedata'".
---
youtube_dl/extractor/mitele.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/mitele.py b/youtube_dl/extractor/mitele.py
index 54993e2c9..ccb5c1467 100644
--- a/youtube_dl/extractor/mitele.py
+++ b/youtube_dl/extractor/mitele.py
@@ -56,7 +56,7 @@ class MiTeleIE(InfoExtractor):
'sta': '0',
}
media = self._download_json(
- '%s/?%s' % (gat, compat_urllib_parse.urlencode(encode_dict(token_data)).encode('utf-8')),
+ '%s/?%s' % (gat, compat_urllib_parse.urlencode(encode_dict(token_data))),
display_id, 'Downloading %s JSON' % location['loc'])
file_ = media.get('file')
if not file_:
From 6856139705aea86ab1f950c08e605dd47f839be0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
<jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 24 Oct 2015 12:13:26 +0200
Subject: [PATCH 085/415] [mitele] Fix test checksum
---
youtube_dl/extractor/mitele.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/mitele.py b/youtube_dl/extractor/mitele.py
index ccb5c1467..3142fcde2 100644
--- a/youtube_dl/extractor/mitele.py
+++ b/youtube_dl/extractor/mitele.py
@@ -15,7 +15,7 @@ class MiTeleIE(InfoExtractor):
_TESTS = [{
'url': 'http://www.mitele.es/programas-tv/diario-de/la-redaccion/programa-144/',
- 'md5': 'ace7635b2a0b286aaa37d3ff192d2a8a',
+ 'md5': '757b0b66cbd7e0a97226d7d3156cb3e9',
'info_dict': {
'id': '0NF1jJnxS1Wu3pHrmvFyw2',
'display_id': 'programa-144',
From 0198807ef95338bec69cfdcd67249c007e4d4141 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christoph=20D=C3=B6pmann?=
<cdoepmann@users.noreply.github.com>
Date: Sat, 24 Oct 2015 11:35:41 +0200
Subject: [PATCH 086/415] [spiegeltv] Fix Accept-Encoding issue (server chokes
on gzip)
---
youtube_dl/extractor/spiegeltv.py | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/youtube_dl/extractor/spiegeltv.py b/youtube_dl/extractor/spiegeltv.py
index 27f4033c5..a85305281 100644
--- a/youtube_dl/extractor/spiegeltv.py
+++ b/youtube_dl/extractor/spiegeltv.py
@@ -83,6 +83,10 @@ class SpiegeltvIE(InfoExtractor):
preference=1, # Prefer hls since it allows to workaround georestriction
m3u8_id='hls', fatal=False)
if m3u8_formats is not False:
+ for m3u8_format in m3u8_formats:
+ m3u8_format['http_headers'] = {
+ 'Accept-Encoding': 'deflate', # gzip causes trouble on the server side
+ }
formats.extend(m3u8_formats)
else:
formats.append({
From 50f01302d347738647262f9442bc4f5d06f013c6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 24 Oct 2015 16:24:08 +0600
Subject: [PATCH 087/415] [spiegeltv] Do not extract m3u8 formats since it's
already a format
---
youtube_dl/extractor/spiegeltv.py | 21 ++++++++++-----------
1 file changed, 10 insertions(+), 11 deletions(-)
diff --git a/youtube_dl/extractor/spiegeltv.py b/youtube_dl/extractor/spiegeltv.py
index a85305281..d976bf33c 100644
--- a/youtube_dl/extractor/spiegeltv.py
+++ b/youtube_dl/extractor/spiegeltv.py
@@ -77,17 +77,16 @@ class SpiegeltvIE(InfoExtractor):
'rtmp_live': True,
})
elif determine_ext(endpoint) == 'm3u8':
- m3u8_formats = self._extract_m3u8_formats(
- endpoint.replace('[video]', play_path),
- video_id, 'm4v',
- preference=1, # Prefer hls since it allows to workaround georestriction
- m3u8_id='hls', fatal=False)
- if m3u8_formats is not False:
- for m3u8_format in m3u8_formats:
- m3u8_format['http_headers'] = {
- 'Accept-Encoding': 'deflate', # gzip causes trouble on the server side
- }
- formats.extend(m3u8_formats)
+ formats.append({
+ 'url': endpoint.replace('[video]', play_path),
+ 'ext': 'm4v',
+ 'format_id': 'hls', # Prefer hls since it allows to workaround georestriction
+ 'protocol': 'm3u8',
+ 'preference': 1,
+ 'http_headers': {
+ 'Accept-Encoding': 'deflate', # gzip causes trouble on the server side
+ },
+ })
else:
formats.append({
'url': endpoint,
From 943a1e24b896a64c869bbd302f32fe5bd2afec96 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 24 Oct 2015 16:25:04 +0600
Subject: [PATCH 088/415] [extractor/common] Use more generic URLError in
_is_valid_url
---
youtube_dl/extractor/common.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 6169fbbeb..720033ddf 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -842,7 +842,7 @@ class InfoExtractor(object):
self._request_webpage(url, video_id, 'Checking %s URL' % item)
return True
except ExtractorError as e:
- if isinstance(e.cause, compat_HTTPError):
+ if isinstance(e.cause, compat_urllib_error.URLError):
self.to_screen(
'%s: %s URL is invalid, skipping' % (video_id, item))
return False
From ac21e7196856d7b689f74ed3f9953cbcbe90bee5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 24 Oct 2015 16:25:44 +0600
Subject: [PATCH 089/415] [spiegeltv] Check formats
---
youtube_dl/extractor/spiegeltv.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/youtube_dl/extractor/spiegeltv.py b/youtube_dl/extractor/spiegeltv.py
index d976bf33c..0981e325a 100644
--- a/youtube_dl/extractor/spiegeltv.py
+++ b/youtube_dl/extractor/spiegeltv.py
@@ -91,6 +91,7 @@ class SpiegeltvIE(InfoExtractor):
formats.append({
'url': endpoint,
})
+ self._check_formats(formats, video_id)
thumbnails = []
for image in media_json['images']:
From 865d1fbafc671815904b2ba3da76544d66c593c9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
<jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 24 Oct 2015 12:39:23 +0200
Subject: [PATCH 090/415] [extractor/common] Remove unused import
---
youtube_dl/extractor/common.py | 1 -
1 file changed, 1 deletion(-)
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 720033ddf..04b699972 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -16,7 +16,6 @@ from ..compat import (
compat_cookiejar,
compat_cookies,
compat_getpass,
- compat_HTTPError,
compat_http_client,
compat_urllib_error,
compat_urllib_parse,
From 36d72810374ef2dba0232706a461d6dc4aa292d8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
<jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 24 Oct 2015 12:41:41 +0200
Subject: [PATCH 091/415] [spiegeltv] Fix style issue
Use two spaces before comment.
---
youtube_dl/extractor/spiegeltv.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/spiegeltv.py b/youtube_dl/extractor/spiegeltv.py
index 0981e325a..034bd47ff 100644
--- a/youtube_dl/extractor/spiegeltv.py
+++ b/youtube_dl/extractor/spiegeltv.py
@@ -84,7 +84,7 @@ class SpiegeltvIE(InfoExtractor):
'protocol': 'm3u8',
'preference': 1,
'http_headers': {
- 'Accept-Encoding': 'deflate', # gzip causes trouble on the server side
+ 'Accept-Encoding': 'deflate', # gzip causes trouble on the server side
},
})
else:
From 7687b354c59efea076fae762206c00de273fbe04 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 23 Oct 2015 07:09:41 +0100
Subject: [PATCH 092/415] [abc] add support for audio extraction
---
youtube_dl/extractor/abc.py | 13 ++++++++++++-
1 file changed, 12 insertions(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/abc.py b/youtube_dl/extractor/abc.py
index f9a389f67..ae80dc529 100644
--- a/youtube_dl/extractor/abc.py
+++ b/youtube_dl/extractor/abc.py
@@ -36,6 +36,15 @@ class ABCIE(InfoExtractor):
'title': 'Marriage Equality: Warren Entsch introduces same sex marriage bill',
},
'add_ie': ['Youtube'],
+ }, {
+ 'url': 'http://www.abc.net.au/news/2015-10-23/nab-lifts-interest-rates-following-westpac-and-cba/6880080',
+ 'md5': 'b96eee7c9edf4fc5a358a0252881cc1f',
+ 'info_dict': {
+ 'id': '6880080',
+ 'ext': 'mp3',
+ 'title': 'NAB lifts interest rates, following Westpac and CBA',
+ 'description': 'md5:f13d8edc81e462fce4a0437c7dc04728',
+ },
}]
def _real_extract(self, url):
@@ -43,7 +52,7 @@ class ABCIE(InfoExtractor):
webpage = self._download_webpage(url, video_id)
mobj = re.search(
- r'inline(?P<type>Video|YouTube)Data\.push\((?P<json_data>[^)]+)\);',
+ r'inline(?P<type>Video|Audio|YouTube)Data\.push\((?P<json_data>[^)]+)\);',
webpage)
if mobj is None:
raise ExtractorError('Unable to extract video urls')
@@ -60,11 +69,13 @@ class ABCIE(InfoExtractor):
formats = [{
'url': url_info['url'],
+ 'vcodec': url_info.get('codec') if mobj.group('type') == 'Video' else 'none',
'width': int_or_none(url_info.get('width')),
'height': int_or_none(url_info.get('height')),
'tbr': int_or_none(url_info.get('bitrate')),
'filesize': int_or_none(url_info.get('filesize')),
} for url_info in urls_info]
+
self._sort_formats(formats)
return {
From d97da29da26c920ae31fde94bba5e3b4e1f5a36a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
<jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 24 Oct 2015 12:31:42 +0200
Subject: [PATCH 093/415] [abc] Support more URL formats
---
youtube_dl/extractor/abc.py | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/abc.py b/youtube_dl/extractor/abc.py
index ae80dc529..c0e5d1abf 100644
--- a/youtube_dl/extractor/abc.py
+++ b/youtube_dl/extractor/abc.py
@@ -12,7 +12,7 @@ from ..utils import (
class ABCIE(InfoExtractor):
IE_NAME = 'abc.net.au'
- _VALID_URL = r'http://www\.abc\.net\.au/news/[^/]+/[^/]+/(?P<id>\d+)'
+ _VALID_URL = r'http://www\.abc\.net\.au/news/(?:[^/]+/){1,2}(?P<id>\d+)'
_TESTS = [{
'url': 'http://www.abc.net.au/news/2014-11-05/australia-to-staff-ebola-treatment-centre-in-sierra-leone/5868334',
@@ -45,6 +45,9 @@ class ABCIE(InfoExtractor):
'title': 'NAB lifts interest rates, following Westpac and CBA',
'description': 'md5:f13d8edc81e462fce4a0437c7dc04728',
},
+ }, {
+ 'url': 'http://www.abc.net.au/news/2015-10-19/6866214',
+ 'only_matching': True,
}]
def _real_extract(self, url):
From 50b936936dcf53b448557c35a90e4678239aaf81 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
<jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 24 Oct 2015 14:22:47 +0200
Subject: [PATCH 094/415] [tutv] Fix test
---
youtube_dl/extractor/tutv.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/youtube_dl/extractor/tutv.py b/youtube_dl/extractor/tutv.py
index fad720b68..822372ea1 100644
--- a/youtube_dl/extractor/tutv.py
+++ b/youtube_dl/extractor/tutv.py
@@ -10,10 +10,10 @@ class TutvIE(InfoExtractor):
_VALID_URL = r'https?://(?:www\.)?tu\.tv/videos/(?P<id>[^/?]+)'
_TEST = {
'url': 'http://tu.tv/videos/robots-futbolistas',
- 'md5': '627c7c124ac2a9b5ab6addb94e0e65f7',
+ 'md5': '0cd9e28ad270488911b0d2a72323395d',
'info_dict': {
'id': '2973058',
- 'ext': 'flv',
+ 'ext': 'mp4',
'title': 'Robots futbolistas',
},
}
From 3711304510d3be6a5f9b2b18084aad8687e78001 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Tue, 8 Sep 2015 19:35:41 +0100
Subject: [PATCH 095/415] [extractor/common] get the redirected m3u8_url in
_extract_m3u8_formats
---
youtube_dl/extractor/common.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 04b699972..10c0d5d1f 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -943,13 +943,14 @@ class InfoExtractor(object):
if re.match(r'^https?://', u)
else compat_urlparse.urljoin(m3u8_url, u))
- m3u8_doc = self._download_webpage(
+ m3u8_doc, urlh = self._download_webpage_handle(
m3u8_url, video_id,
note=note or 'Downloading m3u8 information',
errnote=errnote or 'Failed to download m3u8 information',
fatal=fatal)
if m3u8_doc is False:
return m3u8_doc
+ m3u8_url = urlh.geturl()
last_info = None
last_media = None
kv_rex = re.compile(
From 324ac0a243c14340f7e4cd909e2e7c62828a2425 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Thu, 10 Sep 2015 20:49:43 +0100
Subject: [PATCH 096/415] [downloader/f4m] get the redirected f4m_url and
handle url query string properly
---
youtube_dl/downloader/f4m.py | 15 ++++++++++-----
1 file changed, 10 insertions(+), 5 deletions(-)
diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index 174180db5..b8db6bf9b 100644
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -11,6 +11,7 @@ from .fragment import FragmentFD
from ..compat import (
compat_urlparse,
compat_urllib_error,
+ compat_urllib_parse_urlparse,
)
from ..utils import (
encodeFilename,
@@ -285,7 +286,9 @@ class F4mFD(FragmentFD):
man_url = info_dict['url']
requested_bitrate = info_dict.get('tbr')
self.to_screen('[%s] Downloading f4m manifest' % self.FD_NAME)
- manifest = self.ydl.urlopen(man_url).read()
+ urlh = self.ydl.urlopen(man_url)
+ man_url = urlh.geturl()
+ manifest = urlh.read()
doc = etree.fromstring(manifest)
formats = [(int(f.attrib.get('bitrate', -1)), f)
@@ -329,20 +332,22 @@ class F4mFD(FragmentFD):
if not live:
write_metadata_tag(dest_stream, metadata)
+ base_url_parsed = compat_urllib_parse_urlparse(base_url)
+
self._start_frag_download(ctx)
frags_filenames = []
while fragments_list:
seg_i, frag_i = fragments_list.pop(0)
name = 'Seg%d-Frag%d' % (seg_i, frag_i)
- url = base_url + name
+ url_parsed = base_url_parsed._replace(path=base_url_parsed.path + name)
if akamai_pv:
- url += '?' + akamai_pv.strip(';')
+ url_parsed = url_parsed._replace(query=url_parsed.query + akamai_pv.strip(';'))
if info_dict.get('extra_param_to_segment_url'):
- url += info_dict.get('extra_param_to_segment_url')
+ url_parsed = url_parsed._replace(query=url_parsed.query + info_dict.get('extra_param_to_segment_url'))
frag_filename = '%s-%s' % (ctx['tmpfilename'], name)
try:
- success = ctx['dl'].download(frag_filename, {'url': url})
+ success = ctx['dl'].download(frag_filename, {'url': url_parsed.geturl()})
if not success:
return False
(down, frag_sanitized) = sanitize_open(frag_filename, 'rb')
From 8cd9614abf81cb41055142d87158b5eda4353a4d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 24 Oct 2015 21:02:31 +0600
Subject: [PATCH 097/415] [downloader/f4m] More accurate fragment URL
construction
---
youtube_dl/downloader/f4m.py | 9 ++++++---
1 file changed, 6 insertions(+), 3 deletions(-)
diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index b8db6bf9b..7f6143954 100644
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -340,11 +340,14 @@ class F4mFD(FragmentFD):
while fragments_list:
seg_i, frag_i = fragments_list.pop(0)
name = 'Seg%d-Frag%d' % (seg_i, frag_i)
- url_parsed = base_url_parsed._replace(path=base_url_parsed.path + name)
+ query = []
+ if base_url_parsed.query:
+ query.append(base_url_parsed.query)
if akamai_pv:
- url_parsed = url_parsed._replace(query=url_parsed.query + akamai_pv.strip(';'))
+ query.append(akamai_pv.strip(';'))
if info_dict.get('extra_param_to_segment_url'):
- url_parsed = url_parsed._replace(query=url_parsed.query + info_dict.get('extra_param_to_segment_url'))
+ query.append(info_dict['extra_param_to_segment_url'])
+ url_parsed = base_url_parsed._replace(path=base_url_parsed.path + name, query='&'.join(query))
frag_filename = '%s-%s' % (ctx['tmpfilename'], name)
try:
success = ctx['dl'].download(frag_filename, {'url': url_parsed.geturl()})
From ec29539e06e156a2bb589af774a80d156b2c2f76 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 24 Oct 2015 21:03:45 +0600
Subject: [PATCH 098/415] [senateisvp] Pass extra param as query segment
without `?`
---
youtube_dl/extractor/senateisvp.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/youtube_dl/extractor/senateisvp.py b/youtube_dl/extractor/senateisvp.py
index 9c53704ea..474ebb49b 100644
--- a/youtube_dl/extractor/senateisvp.py
+++ b/youtube_dl/extractor/senateisvp.py
@@ -121,9 +121,9 @@ class SenateISVPIE(InfoExtractor):
'url': compat_urlparse.urljoin(domain, filename) + '?v=3.1.0&fp=&r=&g=',
}]
else:
- hdcore_sign = '?hdcore=3.1.0'
+ hdcore_sign = 'hdcore=3.1.0'
url_params = (domain, video_id, stream_num)
- f4m_url = '%s/z/%s_1@%s/manifest.f4m' % url_params + hdcore_sign
+ f4m_url = '%s/z/%s_1@%s/manifest.f4m?' % url_params + hdcore_sign
m3u8_url = '%s/i/%s_1@%s/master.m3u8' % url_params
for entry in self._extract_f4m_formats(f4m_url, video_id, f4m_id='f4m'):
# URLs without the extra param induce an 404 error
From 8e82ecfe8f0dc2b9dfb6a2cda68e7b5f7926b0e8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 24 Oct 2015 21:04:09 +0600
Subject: [PATCH 099/415] [dailymotion] Extract f4m formats
---
youtube_dl/extractor/dailymotion.py | 14 +++++++++++---
1 file changed, 11 insertions(+), 3 deletions(-)
diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index 9cd9ff17d..bc7823931 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -141,9 +141,17 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
type_ = media.get('type')
if type_ == 'application/vnd.lumberjack.manifest':
continue
- if type_ == 'application/x-mpegURL' or determine_ext(media_url) == 'm3u8':
- formats.extend(self._extract_m3u8_formats(
- media_url, video_id, 'mp4', m3u8_id='hls'))
+ ext = determine_ext(media_url)
+ if type_ == 'application/x-mpegURL' or ext == 'm3u8':
+ m3u8_formats = self._extract_m3u8_formats(
+ media_url, video_id, 'mp4', m3u8_id='hls', fatal=False)
+ if m3u8_formats:
+ formats.extend(m3u8_formats)
+ elif type_ == 'application/f4m' or ext == 'f4m':
+ f4m_formats = self._extract_f4m_formats(
+ media_url, video_id, preference=-1, f4m_id='hds', fatal=False)
+ if f4m_formats:
+ formats.extend(f4m_formats)
else:
f = {
'url': media_url,
From 7e0dc61334cfe9c79e92fd79d9996d191990a80a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 25 Oct 2015 20:48:29 +0600
Subject: [PATCH 100/415] [njoy] Add support for URLs without display id
---
youtube_dl/extractor/ndr.py | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
diff --git a/youtube_dl/extractor/ndr.py b/youtube_dl/extractor/ndr.py
index e3cc6fde8..ba06d8a98 100644
--- a/youtube_dl/extractor/ndr.py
+++ b/youtube_dl/extractor/ndr.py
@@ -14,7 +14,8 @@ from ..utils import (
class NDRBaseIE(InfoExtractor):
def _real_extract(self, url):
- display_id = self._match_id(url)
+ mobj = re.match(self._VALID_URL, url)
+ display_id = next(group for group in mobj.groups() if group)
webpage = self._download_webpage(url, display_id)
return self._extract_embed(webpage, display_id)
@@ -101,7 +102,7 @@ class NDRIE(NDRBaseIE):
class NJoyIE(NDRBaseIE):
IE_NAME = 'njoy'
IE_DESC = 'N-JOY'
- _VALID_URL = r'https?://www\.n-joy\.de/(?:[^/]+/)+(?P<id>[^/?#]+),[\da-z]+\.html'
+ _VALID_URL = r'https?://www\.n-joy\.de/(?:[^/]+/)+(?:(?P<display_id>[^/?#]+),)?(?P<id>[\da-z]+)\.html'
_TESTS = [{
# httpVideo, same content id
'url': 'http://www.n-joy.de/entertainment/comedy/comedy_contest/Benaissa-beim-NDR-Comedy-Contest,comedycontest2480.html',
@@ -136,6 +137,9 @@ class NJoyIE(NDRBaseIE):
'params': {
'skip_download': True,
},
+ }, {
+ 'url': 'http://www.n-joy.de/radio/webradio/morningshow209.html',
+ 'only_matching': True,
}]
def _extract_embed(self, webpage, display_id):
From e572a1010b81f5864e610808c86848db4d6ed8e4 Mon Sep 17 00:00:00 2001
From: Erik <erik@vanderleer.nl>
Date: Sat, 17 Oct 2015 19:22:47 +0200
Subject: [PATCH 101/415] [youporn] Fix extraction
[youporn] Added description and thumbnail
[youporn] Added uploader and date
[youporn] Removed Try and Except lines
[youporn] Fixed date, fatal, formats and /s*
[youporn] Undid removing comment about video url components & Undid and fixed removal of encrypted URL detection
[youporn] Fix: Add encrypted link to links array only if not already in it
[youporn] Fix: Add encrypted link to links array only if not already in it
[youporn] Fix: cleanup
---
youtube_dl/extractor/youporn.py | 55 ++++++++++++++-------------------
1 file changed, 23 insertions(+), 32 deletions(-)
diff --git a/youtube_dl/extractor/youporn.py b/youtube_dl/extractor/youporn.py
index 4ba7c36db..546985f3a 100644
--- a/youtube_dl/extractor/youporn.py
+++ b/youtube_dl/extractor/youporn.py
@@ -4,6 +4,7 @@ from __future__ import unicode_literals
import json
import re
import sys
+import datetime
from .common import InfoExtractor
from ..compat import (
@@ -27,10 +28,11 @@ class YouPornIE(InfoExtractor):
'info_dict': {
'id': '505835',
'ext': 'mp4',
- 'upload_date': '20101221',
- 'description': 'Love & Sex Answers: http://bit.ly/DanAndJenn -- Is It Unhealthy To Masturbate Daily?',
- 'uploader': 'Ask Dan And Jennifer',
'title': 'Sex Ed: Is It Safe To Masturbate Daily?',
+ 'description': 'Watch Sex Ed: Is It Safe To Masturbate Daily? at YouPorn.com - YouPorn is the biggest free porn tube site on the net!',
+ 'uploader': 'Ask Dan And Jennifer',
+ 'thumbnail': 'http://cdn5.image.youporn.phncdn.com/201012/17/505835/640x480/8/sex-ed-is-it-safe-to-masturbate-daily-8.jpg',
+ 'date': '20101221',
'age_limit': 18,
}
}
@@ -45,45 +47,34 @@ class YouPornIE(InfoExtractor):
webpage = self._download_webpage(req, video_id)
age_limit = self._rta_search(webpage)
- # Get JSON parameters
- json_params = self._search_regex(
- [r'videoJa?son\s*=\s*({.+})',
- r'var\s+currentVideo\s*=\s*new\s+Video\((.+?)\)[,;]'],
- webpage, 'JSON parameters')
- try:
- params = json.loads(json_params)
- except ValueError:
- raise ExtractorError('Invalid JSON')
-
self.report_extraction(video_id)
- try:
- video_title = params['title']
- upload_date = unified_strdate(params['release_date_f'])
- video_description = params['description']
- video_uploader = params['submitted_by']
- thumbnail = params['thumbnails'][0]['image']
- except KeyError:
- raise ExtractorError('Missing JSON parameter: ' + sys.exc_info()[1])
+ video_title = self._html_search_regex(r'page_params.video_title = \'(.+?)\';', webpage, 'video URL', fatal=False)
+ video_description = self._html_search_meta('description', webpage, 'video DESC', fatal=False)
+ video_thumbnail = self._html_search_regex(r'page_params.imageurl\t=\t"(.+?)";', webpage, 'video THUMB', fatal=False)
+ video_uploader = self._html_search_regex(r"<div class=\'videoInfoBy\'>By:</div>\n<a href=\"[^>]+\">(.+?)</a>", webpage, 'video UPLOADER', fatal=False)
+ video_date = self._html_search_regex(r"<div class='videoInfoTime'>\n<i class='icon-clock'></i> (.+?)\n</div>", webpage, 'video DATE', fatal=False)
+ video_date = datetime.datetime.strptime(video_date, '%B %d, %Y').strftime('%Y%m%d')
# Get all of the links from the page
- DOWNLOAD_LIST_RE = r'(?s)<ul class="downloadList">(?P<download_list>.*?)</ul>'
- download_list_html = self._search_regex(DOWNLOAD_LIST_RE,
- webpage, 'download list').strip()
- LINK_RE = r'<a href="([^"]+)">'
+ DOWNLOAD_LIST_RE = r'(?s)sources: {\n(?P<download_list>.*?)}'
+ download_list_html = self._search_regex(DOWNLOAD_LIST_RE, webpage, 'download list').strip()
+ LINK_RE = r': \'(.+?)\','
links = re.findall(LINK_RE, download_list_html)
# Get all encrypted links
- encrypted_links = re.findall(r'var encryptedQuality[0-9]{3}URL = \'([a-zA-Z0-9+/]+={0,2})\';', webpage)
+ encrypted_links = re.findall(r'page_params.encryptedQuality[0-9]{3,4}URL\s=\s\'([a-zA-Z0-9+/]+={0,2})\';', webpage)
for encrypted_link in encrypted_links:
link = aes_decrypt_text(encrypted_link, video_title, 32).decode('utf-8')
- links.append(link)
+ # it's unclear if encryted links still differ from normal ones, so only include in links array if it's unique
+ if link not in links:
+ links.append(link)
formats = []
for link in links:
# A link looks like this:
- # http://cdn1.download.youporn.phncdn.com/201210/31/8004515/480p_370k_8004515/YouPorn%20-%20Nubile%20Films%20The%20Pillow%20Fight.mp4?nvb=20121113051249&nva=20121114051249&ir=1200&sr=1200&hash=014b882080310e95fb6a0
+ # http://cdn2b.public.youporn.phncdn.com/201012/17/505835/720p_1500k_505835/YouPorn%20-%20Sex%20Ed%20Is%20It%20Safe%20To%20Masturbate%20Daily.mp4?rs=200&ri=2500&s=1445599900&e=1445773500&h=5345d19ce9944ec52eb167abf24af248
# A path looks like this:
- # /201210/31/8004515/480p_370k_8004515/YouPorn%20-%20Nubile%20Films%20The%20Pillow%20Fight.mp4
+ # 201012/17/505835/720p_1500k_505835/YouPorn%20-%20Sex%20Ed%20Is%20It%20Safe%20To%20Masturbate%20Daily.mp4
video_url = unescapeHTML(link)
path = compat_urllib_parse_urlparse(video_url).path
format_parts = path.split('/')[4].split('_')[:2]
@@ -111,11 +102,11 @@ class YouPornIE(InfoExtractor):
return {
'id': video_id,
- 'uploader': video_uploader,
- 'upload_date': upload_date,
'title': video_title,
- 'thumbnail': thumbnail,
'description': video_description,
+ 'thumbnail': video_thumbnail,
+ 'uploader': video_uploader,
+ 'date': video_date,
'age_limit': age_limit,
'formats': formats,
}
From 589c33dadeec18a9d50713a4a200e3e2d9e297bb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 25 Oct 2015 22:56:35 +0600
Subject: [PATCH 102/415] [youporn] Improve and make more robust (Closes #6888,
closes #7214)
---
youtube_dl/extractor/youporn.py | 173 +++++++++++++++++++-------------
1 file changed, 102 insertions(+), 71 deletions(-)
diff --git a/youtube_dl/extractor/youporn.py b/youtube_dl/extractor/youporn.py
index 546985f3a..d10ebb0bf 100644
--- a/youtube_dl/extractor/youporn.py
+++ b/youtube_dl/extractor/youporn.py
@@ -1,112 +1,143 @@
from __future__ import unicode_literals
-
-import json
import re
-import sys
-import datetime
from .common import InfoExtractor
-from ..compat import (
- compat_urllib_parse_urlparse,
- compat_urllib_request,
-)
+from ..compat import compat_urllib_request
from ..utils import (
- ExtractorError,
+ int_or_none,
+ str_to_int,
unescapeHTML,
unified_strdate,
)
-from ..aes import (
- aes_decrypt_text
-)
+from ..aes import aes_decrypt_text
class YouPornIE(InfoExtractor):
- _VALID_URL = r'^(?P<proto>https?://)(?:www\.)?(?P<url>youporn\.com/watch/(?P<videoid>[0-9]+)/(?P<title>[^/]+))'
+ _VALID_URL = r'https?://(?:www\.)?youporn\.com/watch/(?P<id>\d+)/(?P<display_id>[^/?#&]+)'
_TEST = {
'url': 'http://www.youporn.com/watch/505835/sex-ed-is-it-safe-to-masturbate-daily/',
'info_dict': {
'id': '505835',
+ 'display_id': 'sex-ed-is-it-safe-to-masturbate-daily',
'ext': 'mp4',
'title': 'Sex Ed: Is It Safe To Masturbate Daily?',
- 'description': 'Watch Sex Ed: Is It Safe To Masturbate Daily? at YouPorn.com - YouPorn is the biggest free porn tube site on the net!',
+ 'description': 'Love & Sex Answers: http://bit.ly/DanAndJenn -- Is It Unhealthy To Masturbate Daily?',
+ 'thumbnail': 're:^https?://.*\.jpg$',
'uploader': 'Ask Dan And Jennifer',
- 'thumbnail': 'http://cdn5.image.youporn.phncdn.com/201012/17/505835/640x480/8/sex-ed-is-it-safe-to-masturbate-daily-8.jpg',
- 'date': '20101221',
+ 'upload_date': '20101221',
+ 'average_rating': int,
+ 'view_count': int,
+ 'categories': list,
+ 'tags': list,
'age_limit': 18,
}
}
def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url)
- video_id = mobj.group('videoid')
- url = mobj.group('proto') + 'www.' + mobj.group('url')
+ video_id = mobj.group('id')
+ display_id = mobj.group('display_id')
- req = compat_urllib_request.Request(url)
- req.add_header('Cookie', 'age_verified=1')
- webpage = self._download_webpage(req, video_id)
- age_limit = self._rta_search(webpage)
+ request = compat_urllib_request.Request(url)
+ request.add_header('Cookie', 'age_verified=1')
+ webpage = self._download_webpage(request, display_id)
- self.report_extraction(video_id)
- video_title = self._html_search_regex(r'page_params.video_title = \'(.+?)\';', webpage, 'video URL', fatal=False)
- video_description = self._html_search_meta('description', webpage, 'video DESC', fatal=False)
- video_thumbnail = self._html_search_regex(r'page_params.imageurl\t=\t"(.+?)";', webpage, 'video THUMB', fatal=False)
- video_uploader = self._html_search_regex(r"<div class=\'videoInfoBy\'>By:</div>\n<a href=\"[^>]+\">(.+?)</a>", webpage, 'video UPLOADER', fatal=False)
- video_date = self._html_search_regex(r"<div class='videoInfoTime'>\n<i class='icon-clock'></i> (.+?)\n</div>", webpage, 'video DATE', fatal=False)
- video_date = datetime.datetime.strptime(video_date, '%B %d, %Y').strftime('%Y%m%d')
+ title = self._search_regex(
+ [r'(?:video_titles|videoTitle)\s*[:=]\s*(["\'])(?P<title>.+?)\1',
+ r'<h1[^>]+class=["\']heading\d?["\'][^>]*>([^<])<'],
+ webpage, 'title', group='title')
- # Get all of the links from the page
- DOWNLOAD_LIST_RE = r'(?s)sources: {\n(?P<download_list>.*?)}'
- download_list_html = self._search_regex(DOWNLOAD_LIST_RE, webpage, 'download list').strip()
- LINK_RE = r': \'(.+?)\','
- links = re.findall(LINK_RE, download_list_html)
+ links = []
- # Get all encrypted links
- encrypted_links = re.findall(r'page_params.encryptedQuality[0-9]{3,4}URL\s=\s\'([a-zA-Z0-9+/]+={0,2})\';', webpage)
- for encrypted_link in encrypted_links:
- link = aes_decrypt_text(encrypted_link, video_title, 32).decode('utf-8')
- # it's unclear if encryted links still differ from normal ones, so only include in links array if it's unique
- if link not in links:
+ sources = self._search_regex(
+ r'sources\s*:\s*({.+?})', webpage, 'sources', default=None)
+ if sources:
+ for _, link in re.findall(r'[^:]+\s*:\s*(["\'])(http.+?)\1', sources):
links.append(link)
+ # Fallback #1
+ for _, link in re.findall(
+ r'(?:videoUrl|videoSrc|videoIpadUrl|html5PlayerSrc)\s*[:=]\s*(["\'])(http.+?)\1', webpage):
+ links.append(link)
+
+ # Fallback #2, this also contains extra low quality 180p format
+ for _, link in re.findall(r'<a[^>]+href=(["\'])(http.+?)\1[^>]+title=["\']Download [Vv]ideo', webpage):
+ links.append(link)
+
+ # Fallback #3, encrypted links
+ for _, encrypted_link in re.findall(
+ r'encryptedQuality\d{3,4}URL\s*=\s*(["\'])([\da-zA-Z+/=]+)\1', webpage):
+ links.append(aes_decrypt_text(encrypted_link, title, 32).decode('utf-8'))
+
formats = []
- for link in links:
- # A link looks like this:
- # http://cdn2b.public.youporn.phncdn.com/201012/17/505835/720p_1500k_505835/YouPorn%20-%20Sex%20Ed%20Is%20It%20Safe%20To%20Masturbate%20Daily.mp4?rs=200&ri=2500&s=1445599900&e=1445773500&h=5345d19ce9944ec52eb167abf24af248
- # A path looks like this:
- # 201012/17/505835/720p_1500k_505835/YouPorn%20-%20Sex%20Ed%20Is%20It%20Safe%20To%20Masturbate%20Daily.mp4
- video_url = unescapeHTML(link)
- path = compat_urllib_parse_urlparse(video_url).path
- format_parts = path.split('/')[4].split('_')[:2]
-
- dn = compat_urllib_parse_urlparse(video_url).netloc.partition('.')[0]
-
- resolution = format_parts[0]
- height = int(resolution[:-len('p')])
- bitrate = int(format_parts[1][:-len('k')])
- format = '-'.join(format_parts) + '-' + dn
-
- formats.append({
+ for video_url in set(unescapeHTML(link) for link in links):
+ f = {
'url': video_url,
- 'format': format,
- 'format_id': format,
- 'height': height,
- 'tbr': bitrate,
- 'resolution': resolution,
- })
-
+ }
+ # Video URL's path looks like this:
+ # /201012/17/505835/720p_1500k_505835/YouPorn%20-%20Sex%20Ed%20Is%20It%20Safe%20To%20Masturbate%20Daily.mp4
+ # We will benefit from it by extracting some metadata
+ mobj = re.search(r'/(?P<height>\d{3,4})[pP]_(?P<bitrate>\d+)[kK]_\d+/', video_url)
+ if mobj:
+ height = int(mobj.group('height'))
+ bitrate = int(mobj.group('bitrate'))
+ f.update({
+ 'format_id': '%dp-%dk' % (height, bitrate),
+ 'height': height,
+ 'tbr': bitrate,
+ })
+ formats.append(f)
self._sort_formats(formats)
- if not formats:
- raise ExtractorError('ERROR: no known formats available for video')
+ description = self._html_search_regex(
+ r'(?s)<div[^>]+class=["\']video-description["\'][^>]*>(.+?)</div>',
+ webpage, 'description', fatal=False)
+ thumbnail = self._search_regex(
+ r'(?:imageurl\s*=|poster\s*:)\s*(["\'])(?P<thumbnail>.+?)\1',
+ webpage, 'thumbnail', fatal=False, group='thumbnail')
+
+ uploader = self._search_regex(
+ r'<div[^>]+class=["\']videoInfoBy["\'][^>]*>\s*By:\s*</div>\s*<a[^>]+href="[^"]*">([^<]+)</a>',
+ webpage, 'uploader', fatal=False)
+ upload_date = unified_strdate(self._html_search_regex(
+ r'(?s)<div[^>]+class=["\']videoInfoTime["\'][^>]*>(.+?)</div>',
+ webpage, 'upload date', fatal=False))
+
+ age_limit = self._rta_search(webpage)
+
+ average_rating = int_or_none(self._search_regex(
+ r'<div[^>]+class=["\']videoInfoRating["\'][^>]*>\s*<div[^>]+class=["\']videoRatingPercentage["\'][^>]*>(\d+)%</div>',
+ webpage, 'average rating', fatal=False))
+
+ view_count = str_to_int(self._search_regex(
+ r'(?s)<div[^>]+class=["\']videoInfoViews["\'][^>]*>.*?([\d,.]+)\s*</div>',
+ webpage, 'view count', fatal=False))
+
+ def extract_tag_box(title):
+ tag_box = self._search_regex(
+ (r'<div[^>]+class=["\']tagBoxTitle["\'][^>]*>\s*%s\b.*?</div>\s*'
+ '<div[^>]+class=["\']tagBoxContent["\']>(.+?)</div>') % re.escape(title),
+ webpage, '%s tag box' % title, default=None)
+ if not tag_box:
+ return []
+ return re.findall(r'<a[^>]+href=[^>]+>([^<]+)', tag_box)
+
+ categories = extract_tag_box('Category')
+ tags = extract_tag_box('Tags')
return {
'id': video_id,
- 'title': video_title,
- 'description': video_description,
- 'thumbnail': video_thumbnail,
- 'uploader': video_uploader,
- 'date': video_date,
+ 'display_id': display_id,
+ 'title': title,
+ 'description': description,
+ 'thumbnail': thumbnail,
+ 'uploader': uploader,
+ 'upload_date': upload_date,
+ 'average_rating': average_rating,
+ 'view_count': view_count,
+ 'categories': categories,
+ 'tags': tags,
'age_limit': age_limit,
'formats': formats,
}
From feb7711cf58863a19cae770a878d22a8424e3c61 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 25 Oct 2015 23:01:12 +0600
Subject: [PATCH 103/415] [youporn] Make description optional
Some videos does not contain any description
---
youtube_dl/extractor/youporn.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/youporn.py b/youtube_dl/extractor/youporn.py
index d10ebb0bf..db5d049d2 100644
--- a/youtube_dl/extractor/youporn.py
+++ b/youtube_dl/extractor/youporn.py
@@ -92,7 +92,7 @@ class YouPornIE(InfoExtractor):
description = self._html_search_regex(
r'(?s)<div[^>]+class=["\']video-description["\'][^>]*>(.+?)</div>',
- webpage, 'description', fatal=False)
+ webpage, 'description', default=None)
thumbnail = self._search_regex(
r'(?:imageurl\s*=|poster\s*:)\s*(["\'])(?P<thumbnail>.+?)\1',
webpage, 'thumbnail', fatal=False, group='thumbnail')
From 4f13f8f798be06bc2b3c0c42818bb0785e4cde64 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 25 Oct 2015 23:12:12 +0600
Subject: [PATCH 104/415] [youporn] Improve uploader extraction
---
youtube_dl/extractor/youporn.py | 32 +++++++++++++++++++++++++++-----
1 file changed, 27 insertions(+), 5 deletions(-)
diff --git a/youtube_dl/extractor/youporn.py b/youtube_dl/extractor/youporn.py
index db5d049d2..b39fbb5fc 100644
--- a/youtube_dl/extractor/youporn.py
+++ b/youtube_dl/extractor/youporn.py
@@ -15,8 +15,9 @@ from ..aes import aes_decrypt_text
class YouPornIE(InfoExtractor):
_VALID_URL = r'https?://(?:www\.)?youporn\.com/watch/(?P<id>\d+)/(?P<display_id>[^/?#&]+)'
- _TEST = {
+ _TESTS = [{
'url': 'http://www.youporn.com/watch/505835/sex-ed-is-it-safe-to-masturbate-daily/',
+ 'md5': '71ec5fcfddacf80f495efa8b6a8d9a89',
'info_dict': {
'id': '505835',
'display_id': 'sex-ed-is-it-safe-to-masturbate-daily',
@@ -31,8 +32,29 @@ class YouPornIE(InfoExtractor):
'categories': list,
'tags': list,
'age_limit': 18,
- }
- }
+ },
+ }, {
+ # Anonymous User uploader
+ 'url': 'http://www.youporn.com/watch/561726/big-tits-awesome-brunette-on-amazing-webcam-show/?from=related3&al=2&from_id=561726&pos=4',
+ 'info_dict': {
+ 'id': '561726',
+ 'display_id': 'big-tits-awesome-brunette-on-amazing-webcam-show',
+ 'ext': 'mp4',
+ 'title': 'Big Tits Awesome Brunette On amazing webcam show',
+ 'description': 'http://sweetlivegirls.com Big Tits Awesome Brunette On amazing webcam show.mp4',
+ 'thumbnail': 're:^https?://.*\.jpg$',
+ 'uploader': 'Anonymous User',
+ 'upload_date': '20111125',
+ 'average_rating': int,
+ 'view_count': int,
+ 'categories': list,
+ 'tags': list,
+ 'age_limit': 18,
+ },
+ 'params': {
+ 'skip_download': True,
+ },
+ }]
def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url)
@@ -97,8 +119,8 @@ class YouPornIE(InfoExtractor):
r'(?:imageurl\s*=|poster\s*:)\s*(["\'])(?P<thumbnail>.+?)\1',
webpage, 'thumbnail', fatal=False, group='thumbnail')
- uploader = self._search_regex(
- r'<div[^>]+class=["\']videoInfoBy["\'][^>]*>\s*By:\s*</div>\s*<a[^>]+href="[^"]*">([^<]+)</a>',
+ uploader = self._html_search_regex(
+ r'(?s)<div[^>]+class=["\']videoInfoBy["\'][^>]*>\s*By:\s*</div>(.+?)</(?:a|div)>',
webpage, 'uploader', fatal=False)
upload_date = unified_strdate(self._html_search_regex(
r'(?s)<div[^>]+class=["\']videoInfoTime["\'][^>]*>(.+?)</div>',
From 7b3a19e5339344037a872574780c39f334cea90e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 25 Oct 2015 23:17:23 +0600
Subject: [PATCH 105/415] [stitcher] Remove origEpisodeURL
It's always 404
---
youtube_dl/extractor/stitcher.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/stitcher.py b/youtube_dl/extractor/stitcher.py
index 971a1c466..d5c852f52 100644
--- a/youtube_dl/extractor/stitcher.py
+++ b/youtube_dl/extractor/stitcher.py
@@ -64,7 +64,7 @@ class StitcherIE(InfoExtractor):
'url': episode[episode_key],
'ext': determine_ext(episode[episode_key]) or 'mp3',
'vcodec': 'none',
- } for episode_key in ('origEpisodeURL', 'episodeURL') if episode.get(episode_key)]
+ } for episode_key in ('episodeURL',) if episode.get(episode_key)]
description = self._search_regex(
r'Episode Info:\s*</span>([^<]+)<', webpage, 'description', fatal=False)
duration = int_or_none(episode.get('duration'))
From 755ff8d22ca5607400c1232b194e20a004e4e9eb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 25 Oct 2015 23:41:10 +0600
Subject: [PATCH 106/415] [youporn] Extract comment count
---
youtube_dl/extractor/youporn.py | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/youtube_dl/extractor/youporn.py b/youtube_dl/extractor/youporn.py
index b39fbb5fc..9bf8d1eeb 100644
--- a/youtube_dl/extractor/youporn.py
+++ b/youtube_dl/extractor/youporn.py
@@ -29,6 +29,7 @@ class YouPornIE(InfoExtractor):
'upload_date': '20101221',
'average_rating': int,
'view_count': int,
+ 'comment_count': int,
'categories': list,
'tags': list,
'age_limit': 18,
@@ -47,6 +48,7 @@ class YouPornIE(InfoExtractor):
'upload_date': '20111125',
'average_rating': int,
'view_count': int,
+ 'comment_count': int,
'categories': list,
'tags': list,
'age_limit': 18,
@@ -135,6 +137,9 @@ class YouPornIE(InfoExtractor):
view_count = str_to_int(self._search_regex(
r'(?s)<div[^>]+class=["\']videoInfoViews["\'][^>]*>.*?([\d,.]+)\s*</div>',
webpage, 'view count', fatal=False))
+ comment_count = str_to_int(self._search_regex(
+ r'>All [Cc]omments? \(([\d,.]+)\)',
+ webpage, 'comment count', fatal=False))
def extract_tag_box(title):
tag_box = self._search_regex(
@@ -158,6 +163,7 @@ class YouPornIE(InfoExtractor):
'upload_date': upload_date,
'average_rating': average_rating,
'view_count': view_count,
+ 'comment_count': comment_count,
'categories': categories,
'tags': tags,
'age_limit': age_limit,
From 36e6f62cd0883f0f486d1666d010e5d9e6d515bd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
<jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 25 Oct 2015 20:04:55 +0100
Subject: [PATCH 107/415] Use a wrapper around xml.etree.ElementTree.fromstring
in python 2.x (#7178)
Attributes aren't unicode objects, so they couldn't be directly used in info_dict fields (for example '--write-description' doesn't work with bytes).
---
test/test_compat.py | 7 +++++++
test/test_utils.py | 11 +++++++----
youtube_dl/compat.py | 25 +++++++++++++++++++++++++
youtube_dl/downloader/f4m.py | 4 ++--
youtube_dl/extractor/bbc.py | 8 +++++---
youtube_dl/extractor/bilibili.py | 6 ++++--
youtube_dl/extractor/brightcove.py | 4 ++--
youtube_dl/extractor/common.py | 4 ++--
youtube_dl/extractor/crunchyroll.py | 4 ++--
youtube_dl/extractor/vevo.py | 6 +++---
youtube_dl/utils.py | 3 ++-
11 files changed, 61 insertions(+), 21 deletions(-)
diff --git a/test/test_compat.py b/test/test_compat.py
index 4ee0dc99d..2b0860479 100644
--- a/test/test_compat.py
+++ b/test/test_compat.py
@@ -13,8 +13,10 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from youtube_dl.utils import get_filesystem_encoding
from youtube_dl.compat import (
compat_getenv,
+ compat_etree_fromstring,
compat_expanduser,
compat_shlex_split,
+ compat_str,
compat_urllib_parse_unquote,
compat_urllib_parse_unquote_plus,
)
@@ -71,5 +73,10 @@ class TestCompat(unittest.TestCase):
def test_compat_shlex_split(self):
self.assertEqual(compat_shlex_split('-option "one two"'), ['-option', 'one two'])
+ def test_compat_etree_fromstring(self):
+ xml = '<el foo="bar"></el>'
+ doc = compat_etree_fromstring(xml.encode('utf-8'))
+ self.assertTrue(isinstance(doc.attrib['foo'], compat_str))
+
if __name__ == '__main__':
unittest.main()
diff --git a/test/test_utils.py b/test/test_utils.py
index 918a7a9ef..a9e0fed7e 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -68,6 +68,9 @@ from youtube_dl.utils import (
cli_valueless_option,
cli_bool_option,
)
+from youtube_dl.compat import (
+ compat_etree_fromstring,
+)
class TestUtil(unittest.TestCase):
@@ -242,7 +245,7 @@ class TestUtil(unittest.TestCase):
<node x="b" y="d" />
<node x="" />
</root>'''
- doc = xml.etree.ElementTree.fromstring(testxml)
+ doc = compat_etree_fromstring(testxml)
self.assertEqual(find_xpath_attr(doc, './/fourohfour', 'n'), None)
self.assertEqual(find_xpath_attr(doc, './/fourohfour', 'n', 'v'), None)
@@ -263,7 +266,7 @@ class TestUtil(unittest.TestCase):
<url>http://server.com/download.mp3</url>
</media:song>
</root>'''
- doc = xml.etree.ElementTree.fromstring(testxml)
+ doc = compat_etree_fromstring(testxml)
find = lambda p: doc.find(xpath_with_ns(p, {'media': 'http://example.com/'}))
self.assertTrue(find('media:song') is not None)
self.assertEqual(find('media:song/media:author').text, 'The Author')
@@ -285,7 +288,7 @@ class TestUtil(unittest.TestCase):
<p>Foo</p>
</div>
</root>'''
- doc = xml.etree.ElementTree.fromstring(testxml)
+ doc = compat_etree_fromstring(testxml)
self.assertEqual(xpath_text(doc, 'div/p'), 'Foo')
self.assertEqual(xpath_text(doc, 'div/bar', default='default'), 'default')
self.assertTrue(xpath_text(doc, 'div/bar') is None)
@@ -297,7 +300,7 @@ class TestUtil(unittest.TestCase):
<p x="a">Foo</p>
</div>
</root>'''
- doc = xml.etree.ElementTree.fromstring(testxml)
+ doc = compat_etree_fromstring(testxml)
self.assertEqual(xpath_attr(doc, 'div/p', 'x'), 'a')
self.assertEqual(xpath_attr(doc, 'div/bar', 'x'), None)
self.assertEqual(xpath_attr(doc, 'div/p', 'y'), None)
diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index d103ab9ad..cf10835ca 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -14,6 +14,7 @@ import socket
import subprocess
import sys
import itertools
+import xml.etree.ElementTree
try:
@@ -212,6 +213,29 @@ try:
except ImportError: # Python 2.6
from xml.parsers.expat import ExpatError as compat_xml_parse_error
+if sys.version_info[0] >= 3:
+ compat_etree_fromstring = xml.etree.ElementTree.fromstring
+else:
+ # on python 2.x the the attributes of a node are str objects instead of
+ # unicode
+ etree = xml.etree.ElementTree
+
+ # on 2.6 XML doesn't have a parser argument, function copied from CPython
+ # 2.7 source
+ def _XML(text, parser=None):
+ if not parser:
+ parser = etree.XMLParser(target=etree.TreeBuilder())
+ parser.feed(text)
+ return parser.close()
+
+ def _element_factory(*args, **kwargs):
+ el = etree.Element(*args, **kwargs)
+ for k, v in el.items():
+ el.set(k, v.decode('utf-8'))
+ return el
+
+ def compat_etree_fromstring(text):
+ return _XML(text, parser=etree.XMLParser(target=etree.TreeBuilder(element_factory=_element_factory)))
try:
from urllib.parse import parse_qs as compat_parse_qs
@@ -507,6 +531,7 @@ __all__ = [
'compat_chr',
'compat_cookiejar',
'compat_cookies',
+ 'compat_etree_fromstring',
'compat_expanduser',
'compat_get_terminal_size',
'compat_getenv',
diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index 7f6143954..6170cc155 100644
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -5,10 +5,10 @@ import io
import itertools
import os
import time
-import xml.etree.ElementTree as etree
from .fragment import FragmentFD
from ..compat import (
+ compat_etree_fromstring,
compat_urlparse,
compat_urllib_error,
compat_urllib_parse_urlparse,
@@ -290,7 +290,7 @@ class F4mFD(FragmentFD):
man_url = urlh.geturl()
manifest = urlh.read()
- doc = etree.fromstring(manifest)
+ doc = compat_etree_fromstring(manifest)
formats = [(int(f.attrib.get('bitrate', -1)), f)
for f in self._get_unencrypted_media(doc)]
if requested_bitrate is None:
diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 2cdce1eb9..a55a6dbc9 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -2,7 +2,6 @@
from __future__ import unicode_literals
import re
-import xml.etree.ElementTree
from .common import InfoExtractor
from ..utils import (
@@ -14,7 +13,10 @@ from ..utils import (
remove_end,
unescapeHTML,
)
-from ..compat import compat_HTTPError
+from ..compat import (
+ compat_etree_fromstring,
+ compat_HTTPError,
+)
class BBCCoUkIE(InfoExtractor):
@@ -344,7 +346,7 @@ class BBCCoUkIE(InfoExtractor):
url, programme_id, 'Downloading media selection XML')
except ExtractorError as ee:
if isinstance(ee.cause, compat_HTTPError) and ee.cause.code == 403:
- media_selection = xml.etree.ElementTree.fromstring(ee.cause.read().decode('utf-8'))
+ media_selection = compat_etree_fromstring(ee.cause.read().decode('utf-8'))
else:
raise
return self._process_media_selector(media_selection, programme_id)
diff --git a/youtube_dl/extractor/bilibili.py b/youtube_dl/extractor/bilibili.py
index ecc17ebeb..6c66a1236 100644
--- a/youtube_dl/extractor/bilibili.py
+++ b/youtube_dl/extractor/bilibili.py
@@ -4,9 +4,11 @@ from __future__ import unicode_literals
import re
import itertools
import json
-import xml.etree.ElementTree as ET
from .common import InfoExtractor
+from ..compat import (
+ compat_etree_fromstring,
+)
from ..utils import (
int_or_none,
unified_strdate,
@@ -88,7 +90,7 @@ class BiliBiliIE(InfoExtractor):
except ValueError:
pass
- lq_doc = ET.fromstring(lq_page)
+ lq_doc = compat_etree_fromstring(lq_page)
lq_durls = lq_doc.findall('./durl')
hq_doc = self._download_xml(
diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 4721c2293..1686cdde1 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -3,10 +3,10 @@ from __future__ import unicode_literals
import re
import json
-import xml.etree.ElementTree
from .common import InfoExtractor
from ..compat import (
+ compat_etree_fromstring,
compat_parse_qs,
compat_str,
compat_urllib_parse,
@@ -119,7 +119,7 @@ class BrightcoveIE(InfoExtractor):
object_str = fix_xml_ampersands(object_str)
try:
- object_doc = xml.etree.ElementTree.fromstring(object_str.encode('utf-8'))
+ object_doc = compat_etree_fromstring(object_str.encode('utf-8'))
except compat_xml_parse_error:
return
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 10c0d5d1f..52523d7b2 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -10,7 +10,6 @@ import re
import socket
import sys
import time
-import xml.etree.ElementTree
from ..compat import (
compat_cookiejar,
@@ -23,6 +22,7 @@ from ..compat import (
compat_urllib_request,
compat_urlparse,
compat_str,
+ compat_etree_fromstring,
)
from ..utils import (
NO_DEFAULT,
@@ -461,7 +461,7 @@ class InfoExtractor(object):
return xml_string
if transform_source:
xml_string = transform_source(xml_string)
- return xml.etree.ElementTree.fromstring(xml_string.encode('utf-8'))
+ return compat_etree_fromstring(xml_string.encode('utf-8'))
def _download_json(self, url_or_request, video_id,
note='Downloading JSON metadata',
diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index f8ce10111..0c9b8ca02 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -5,12 +5,12 @@ import re
import json
import base64
import zlib
-import xml.etree.ElementTree
from hashlib import sha1
from math import pow, sqrt, floor
from .common import InfoExtractor
from ..compat import (
+ compat_etree_fromstring,
compat_urllib_parse,
compat_urllib_parse_unquote,
compat_urllib_request,
@@ -234,7 +234,7 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
return output
def _extract_subtitles(self, subtitle):
- sub_root = xml.etree.ElementTree.fromstring(subtitle)
+ sub_root = compat_etree_fromstring(subtitle)
return [{
'ext': 'srt',
'data': self._convert_subtitles_to_srt(sub_root),
diff --git a/youtube_dl/extractor/vevo.py b/youtube_dl/extractor/vevo.py
index c17094f81..4c0de354f 100644
--- a/youtube_dl/extractor/vevo.py
+++ b/youtube_dl/extractor/vevo.py
@@ -1,10 +1,10 @@
from __future__ import unicode_literals
import re
-import xml.etree.ElementTree
from .common import InfoExtractor
from ..compat import (
+ compat_etree_fromstring,
compat_urllib_request,
)
from ..utils import (
@@ -97,7 +97,7 @@ class VevoIE(InfoExtractor):
if last_version['version'] == -1:
raise ExtractorError('Unable to extract last version of the video')
- renditions = xml.etree.ElementTree.fromstring(last_version['data'])
+ renditions = compat_etree_fromstring(last_version['data'])
formats = []
# Already sorted from worst to best quality
for rend in renditions.findall('rendition'):
@@ -114,7 +114,7 @@ class VevoIE(InfoExtractor):
def _formats_from_smil(self, smil_xml):
formats = []
- smil_doc = xml.etree.ElementTree.fromstring(smil_xml.encode('utf-8'))
+ smil_doc = compat_etree_fromstring(smil_xml.encode('utf-8'))
els = smil_doc.findall('.//{http://www.w3.org/2001/SMIL20/Language}video')
for el in els:
src = el.attrib['src']
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index a61e47646..7d846d680 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -36,6 +36,7 @@ import zlib
from .compat import (
compat_basestring,
compat_chr,
+ compat_etree_fromstring,
compat_html_entities,
compat_http_client,
compat_kwargs,
@@ -1974,7 +1975,7 @@ def dfxp2srt(dfxp_data):
return out
- dfxp = xml.etree.ElementTree.fromstring(dfxp_data.encode('utf-8'))
+ dfxp = compat_etree_fromstring(dfxp_data.encode('utf-8'))
out = []
paras = dfxp.findall(_x('.//ttml:p')) or dfxp.findall(_x('.//ttaf1:p')) or dfxp.findall('.//p')
From 387db16a789fea25795433538d80513c18d0f699 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
<jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 25 Oct 2015 20:30:54 +0100
Subject: [PATCH 108/415] [compat] compat_etree_fromstring: only decode bytes
objects
---
test/test_compat.py | 3 ++-
youtube_dl/compat.py | 6 +++---
2 files changed, 5 insertions(+), 4 deletions(-)
diff --git a/test/test_compat.py b/test/test_compat.py
index 2b0860479..834f4bc55 100644
--- a/test/test_compat.py
+++ b/test/test_compat.py
@@ -74,9 +74,10 @@ class TestCompat(unittest.TestCase):
self.assertEqual(compat_shlex_split('-option "one two"'), ['-option', 'one two'])
def test_compat_etree_fromstring(self):
- xml = '<el foo="bar"></el>'
+ xml = '<el foo="bar" spam="中文"></el>'
doc = compat_etree_fromstring(xml.encode('utf-8'))
self.assertTrue(isinstance(doc.attrib['foo'], compat_str))
+ self.assertTrue(isinstance(doc.attrib['spam'], compat_str))
if __name__ == '__main__':
unittest.main()
diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index cf10835ca..f39d4e9a9 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -216,8 +216,7 @@ except ImportError: # Python 2.6
if sys.version_info[0] >= 3:
compat_etree_fromstring = xml.etree.ElementTree.fromstring
else:
- # on python 2.x the the attributes of a node are str objects instead of
- # unicode
+ # on python 2.x the the attributes of a node aren't always unicode objects
etree = xml.etree.ElementTree
# on 2.6 XML doesn't have a parser argument, function copied from CPython
@@ -231,7 +230,8 @@ else:
def _element_factory(*args, **kwargs):
el = etree.Element(*args, **kwargs)
for k, v in el.items():
- el.set(k, v.decode('utf-8'))
+ if isinstance(v, bytes):
+ el.set(k, v.decode('utf-8'))
return el
def compat_etree_fromstring(text):
From 5f9f87c06fa819c75e59f5d0d491d191c229abbc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Lalinsk=C3=BD?= <lukas@oxygene.sk>
Date: Mon, 26 Oct 2015 14:42:17 +0100
Subject: [PATCH 109/415] [vidme] Check for deleted videos
---
youtube_dl/extractor/vidme.py | 9 +++++++++
1 file changed, 9 insertions(+)
diff --git a/youtube_dl/extractor/vidme.py b/youtube_dl/extractor/vidme.py
index eb5cde761..3d63ed4f0 100644
--- a/youtube_dl/extractor/vidme.py
+++ b/youtube_dl/extractor/vidme.py
@@ -101,6 +101,10 @@ class VidmeIE(InfoExtractor):
# suspended
'url': 'https://vid.me/Ox3G',
'only_matching': True,
+ }, {
+ # deleted
+ 'url': 'https://vid.me/KTPm',
+ 'only_matching': True,
}, {
# no formats in the API response
'url': 'https://vid.me/e5g',
@@ -143,6 +147,11 @@ class VidmeIE(InfoExtractor):
video = response['video']
+ if video.get('state') == 'deleted':
+ raise ExtractorError(
+ 'Vidme said: Sorry, this video has been deleted.',
+ expected=True)
+
if video.get('state') in ('user-disabled', 'suspended'):
raise ExtractorError(
'Vidme said: This video has been suspended either due to a copyright claim, '
From 5dadae079bd053c822353b081e94d9daff333208 Mon Sep 17 00:00:00 2001
From: Frans de Jonge <fransdejonge@gmail.com>
Date: Mon, 26 Oct 2015 15:11:09 +0100
Subject: [PATCH 110/415] [francetv] Add subtitles support
---
youtube_dl/extractor/francetv.py | 11 +++++++++++
1 file changed, 11 insertions(+)
diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index 129984a5f..eaaa43958 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -83,6 +83,16 @@ class FranceTVBaseInfoExtractor(InfoExtractor):
if subtitle:
title += ' - %s' % subtitle
+ subtitles = {}
+ for subtitle_accessibilite in info['subtitles']:
+ if subtitle_accessibilite['url'] is not '':
+ if not subtitles:
+ subtitles['fr'] = []
+ subtitles['fr'].append({
+ 'ext': subtitle_accessibilite['format'],
+ 'url': subtitle_accessibilite['url'],
+ })
+
return {
'id': video_id,
'title': title,
@@ -91,6 +101,7 @@ class FranceTVBaseInfoExtractor(InfoExtractor):
'duration': int_or_none(info.get('real_duration')) or parse_duration(info['duree']),
'timestamp': int_or_none(info['diffusion']['timestamp']),
'formats': formats,
+ 'subtitles': subtitles,
}
From 6e4b8b28916aaafc6d1b4b4d69a6f667e35d413f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 26 Oct 2015 20:35:28 +0600
Subject: [PATCH 111/415] [francetv] Make subtitles more robust (Closes #7298)
---
youtube_dl/extractor/francetv.py | 14 ++++++--------
1 file changed, 6 insertions(+), 8 deletions(-)
diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index eaaa43958..07115b9d4 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -84,14 +84,12 @@ class FranceTVBaseInfoExtractor(InfoExtractor):
title += ' - %s' % subtitle
subtitles = {}
- for subtitle_accessibilite in info['subtitles']:
- if subtitle_accessibilite['url'] is not '':
- if not subtitles:
- subtitles['fr'] = []
- subtitles['fr'].append({
- 'ext': subtitle_accessibilite['format'],
- 'url': subtitle_accessibilite['url'],
- })
+ subtitles_list = [{
+ 'url': subtitle['url'],
+ 'ext': subtitle.get('format'),
+ } for subtitle in info.get('subtitles', []) if subtitle.get('url')]
+ if subtitles_list:
+ subtitles['fr'] = subtitles_list
return {
'id': video_id,
From c137cc0d33fee8369b857c8f12a9116379248127 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 26 Oct 2015 20:35:45 +0600
Subject: [PATCH 112/415] [francetv] Add subtitles test
---
youtube_dl/extractor/francetv.py | 3 +++
1 file changed, 3 insertions(+)
diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index 07115b9d4..a31cc3c97 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -129,6 +129,9 @@ class FranceTvInfoIE(FranceTVBaseInfoExtractor):
'title': 'Soir 3',
'upload_date': '20130826',
'timestamp': 1377548400,
+ 'subtitles': {
+ 'fr': 'mincount:2',
+ },
},
}, {
'url': 'http://www.francetvinfo.fr/elections/europeennes/direct-europeennes-regardez-le-debat-entre-les-candidats-a-la-presidence-de-la-commission_600639.html',
From f78546272cf7c4b10c8003870728ab69bec982fc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
<jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 26 Oct 2015 16:41:24 +0100
Subject: [PATCH 113/415] [compat] compat_etree_fromstring: also decode the
text attribute
Deletes parse_xml from utils, because it also does it.
---
test/test_compat.py | 11 ++++++++++-
youtube_dl/compat.py | 18 ++++++++++++++++--
youtube_dl/extractor/ard.py | 4 ++--
youtube_dl/extractor/generic.py | 4 ++--
youtube_dl/utils.py | 23 -----------------------
5 files changed, 30 insertions(+), 30 deletions(-)
diff --git a/test/test_compat.py b/test/test_compat.py
index 834f4bc55..b6bfad05e 100644
--- a/test/test_compat.py
+++ b/test/test_compat.py
@@ -74,10 +74,19 @@ class TestCompat(unittest.TestCase):
self.assertEqual(compat_shlex_split('-option "one two"'), ['-option', 'one two'])
def test_compat_etree_fromstring(self):
- xml = '<el foo="bar" spam="中文"></el>'
+ xml = '''
+ <root foo="bar" spam="中文">
+ <normal>foo</normal>
+ <chinese>中文</chinese>
+ <foo><bar>spam</bar></foo>
+ </root>
+ '''
doc = compat_etree_fromstring(xml.encode('utf-8'))
self.assertTrue(isinstance(doc.attrib['foo'], compat_str))
self.assertTrue(isinstance(doc.attrib['spam'], compat_str))
+ self.assertTrue(isinstance(doc.find('normal').text, compat_str))
+ self.assertTrue(isinstance(doc.find('chinese').text, compat_str))
+ self.assertTrue(isinstance(doc.find('foo/bar').text, compat_str))
if __name__ == '__main__':
unittest.main()
diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index f39d4e9a9..2d43ec852 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -216,9 +216,19 @@ except ImportError: # Python 2.6
if sys.version_info[0] >= 3:
compat_etree_fromstring = xml.etree.ElementTree.fromstring
else:
- # on python 2.x the the attributes of a node aren't always unicode objects
+ # on python 2.x the attributes and text of a node aren't always unicode
+ # objects
etree = xml.etree.ElementTree
+ try:
+ _etree_iter = etree.Element.iter
+ except AttributeError: # Python <=2.6
+ def _etree_iter(root):
+ for el in root.findall('*'):
+ yield el
+ for sub in _etree_iter(el):
+ yield sub
+
# on 2.6 XML doesn't have a parser argument, function copied from CPython
# 2.7 source
def _XML(text, parser=None):
@@ -235,7 +245,11 @@ else:
return el
def compat_etree_fromstring(text):
- return _XML(text, parser=etree.XMLParser(target=etree.TreeBuilder(element_factory=_element_factory)))
+ doc = _XML(text, parser=etree.XMLParser(target=etree.TreeBuilder(element_factory=_element_factory)))
+ for el in _etree_iter(doc):
+ if el.text is not None and isinstance(el.text, bytes):
+ el.text = el.text.decode('utf-8')
+ return doc
try:
from urllib.parse import parse_qs as compat_parse_qs
diff --git a/youtube_dl/extractor/ard.py b/youtube_dl/extractor/ard.py
index 6f465789b..73be6d204 100644
--- a/youtube_dl/extractor/ard.py
+++ b/youtube_dl/extractor/ard.py
@@ -14,8 +14,8 @@ from ..utils import (
parse_duration,
unified_strdate,
xpath_text,
- parse_xml,
)
+from ..compat import compat_etree_fromstring
class ARDMediathekIE(InfoExtractor):
@@ -161,7 +161,7 @@ class ARDMediathekIE(InfoExtractor):
raise ExtractorError('This program is only suitable for those aged 12 and older. Video %s is therefore only available between 20 pm and 6 am.' % video_id, expected=True)
if re.search(r'[\?&]rss($|[=&])', url):
- doc = parse_xml(webpage)
+ doc = compat_etree_fromstring(webpage.encode('utf-8'))
if doc.tag == 'rss':
return GenericIE()._extract_rss(url, video_id, doc)
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index ca5fbafb2..1de96b268 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -9,6 +9,7 @@ import sys
from .common import InfoExtractor
from .youtube import YoutubeIE
from ..compat import (
+ compat_etree_fromstring,
compat_urllib_parse_unquote,
compat_urllib_request,
compat_urlparse,
@@ -21,7 +22,6 @@ from ..utils import (
HEADRequest,
is_html,
orderedSet,
- parse_xml,
smuggle_url,
unescapeHTML,
unified_strdate,
@@ -1237,7 +1237,7 @@ class GenericIE(InfoExtractor):
# Is it an RSS feed, a SMIL file or a XSPF playlist?
try:
- doc = parse_xml(webpage)
+ doc = compat_etree_fromstring(webpage.encode('utf-8'))
if doc.tag == 'rss':
return self._extract_rss(url, video_id, doc)
elif re.match(r'^(?:{[^}]+})?smil$', doc.tag):
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 7d846d680..c761ea22a 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1652,29 +1652,6 @@ def encode_dict(d, encoding='utf-8'):
return dict((k.encode(encoding), v.encode(encoding)) for k, v in d.items())
-try:
- etree_iter = xml.etree.ElementTree.Element.iter
-except AttributeError: # Python <=2.6
- etree_iter = lambda n: n.findall('.//*')
-
-
-def parse_xml(s):
- class TreeBuilder(xml.etree.ElementTree.TreeBuilder):
- def doctype(self, name, pubid, system):
- pass # Ignore doctypes
-
- parser = xml.etree.ElementTree.XMLParser(target=TreeBuilder())
- kwargs = {'parser': parser} if sys.version_info >= (2, 7) else {}
- tree = xml.etree.ElementTree.XML(s.encode('utf-8'), **kwargs)
- # Fix up XML parser in Python 2.x
- if sys.version_info < (3, 0):
- for n in etree_iter(tree):
- if n.text is not None:
- if not isinstance(n.text, compat_str):
- n.text = n.text.decode('utf-8')
- return tree
-
-
US_RATINGS = {
'G': 0,
'PG': 10,
From a526167d40983e47231d10c09c9f9064e0298604 Mon Sep 17 00:00:00 2001
From: Pierre Fenoll <pierrefenoll@gmail.com>
Date: Tue, 27 Oct 2015 11:58:59 +0100
Subject: [PATCH 114/415] [francetv] Accept mobile URLs
---
youtube_dl/extractor/francetv.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index a31cc3c97..d63dc4d7c 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -105,7 +105,7 @@ class FranceTVBaseInfoExtractor(InfoExtractor):
class PluzzIE(FranceTVBaseInfoExtractor):
IE_NAME = 'pluzz.francetv.fr'
- _VALID_URL = r'https?://pluzz\.francetv\.fr/videos/(.*?)\.html'
+ _VALID_URL = r'https?://(?:m\.)?pluzz\.francetv\.fr/videos/(.*?)\.html'
# Can't use tests, videos expire in 7 days
From 0a192fbea798c843ad6fef37106901d431f39b6e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 27 Oct 2015 21:43:29 +0600
Subject: [PATCH 115/415] [pluzz] Fix mobile support and modernize (Closes
#7305)
---
youtube_dl/extractor/francetv.py | 16 +++++++++++-----
1 file changed, 11 insertions(+), 5 deletions(-)
diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index d63dc4d7c..00a80ba61 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -105,15 +105,21 @@ class FranceTVBaseInfoExtractor(InfoExtractor):
class PluzzIE(FranceTVBaseInfoExtractor):
IE_NAME = 'pluzz.francetv.fr'
- _VALID_URL = r'https?://(?:m\.)?pluzz\.francetv\.fr/videos/(.*?)\.html'
+ _VALID_URL = r'https?://(?:m\.)?pluzz\.francetv\.fr/videos/(?P<id>.+?)\.html'
# Can't use tests, videos expire in 7 days
def _real_extract(self, url):
- title = re.match(self._VALID_URL, url).group(1)
- webpage = self._download_webpage(url, title)
- video_id = self._search_regex(
- r'data-diffusion="(\d+)"', webpage, 'ID')
+ display_id = self._match_id(url)
+
+ webpage = self._download_webpage(url, display_id)
+
+ video_id = self._html_search_meta(
+ 'id_video', webpage, 'video id', default=None)
+ if not video_id:
+ video_id = self._search_regex(
+ r'data-diffusion=["\'](\d+)', webpage, 'video id')
+
return self._extract_video(video_id, 'Pluzz')
From 7ccb2b84ddb65f41d01037b5b62301886be9d22c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
<jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 28 Oct 2015 08:22:04 +0100
Subject: [PATCH 116/415] [francetv] fix style issues reported by flake8
* Don't redefine variable in list comprehension
* Line missing indentation
---
youtube_dl/extractor/francetv.py | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index 00a80ba61..8e60cf60f 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -85,9 +85,9 @@ class FranceTVBaseInfoExtractor(InfoExtractor):
subtitles = {}
subtitles_list = [{
- 'url': subtitle['url'],
- 'ext': subtitle.get('format'),
- } for subtitle in info.get('subtitles', []) if subtitle.get('url')]
+ 'url': subformat['url'],
+ 'ext': subformat.get('format'),
+ } for subformat in info.get('subtitles', []) if subformat.get('url')]
if subtitles_list:
subtitles['fr'] = subtitles_list
@@ -118,7 +118,7 @@ class PluzzIE(FranceTVBaseInfoExtractor):
'id_video', webpage, 'video id', default=None)
if not video_id:
video_id = self._search_regex(
- r'data-diffusion=["\'](\d+)', webpage, 'video id')
+ r'data-diffusion=["\'](\d+)', webpage, 'video id')
return self._extract_video(video_id, 'Pluzz')
From 4e16c1f80b009001aaea6f8baca5dfbfa9b58c11 Mon Sep 17 00:00:00 2001
From: Cian Ruane <CianLR@users.noreply.github.com>
Date: Fri, 16 Oct 2015 01:23:09 +0100
Subject: [PATCH 117/415] [clyp] Add extractor
Update __init__.py
[clyp.it] Extract ID idiomatically and make duration and description optional
---
youtube_dl/extractor/__init__.py | 1 +
youtube_dl/extractor/clyp.py | 57 ++++++++++++++++++++++++++++++++
2 files changed, 58 insertions(+)
create mode 100644 youtube_dl/extractor/clyp.py
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 6318ac4a2..f98e6487e 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -90,6 +90,7 @@ from .cliphunter import CliphunterIE
from .clipsyndicate import ClipsyndicateIE
from .cloudy import CloudyIE
from .clubic import ClubicIE
+from .clyp import ClypIE
from .cmt import CMTIE
from .cnet import CNETIE
from .cnn import (
diff --git a/youtube_dl/extractor/clyp.py b/youtube_dl/extractor/clyp.py
new file mode 100644
index 000000000..906729b30
--- /dev/null
+++ b/youtube_dl/extractor/clyp.py
@@ -0,0 +1,57 @@
+# coding: utf-8
+
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+
+class ClypIE(InfoExtractor):
+ _VALID_URL = r'https?://(?:www\.)?clyp\.it/(?P<id>[a-z0-9]+)'
+
+ _TESTS = [{
+ 'url': 'https://clyp.it/ojz2wfah',
+ 'md5': '1d4961036c41247ecfdcc439c0cddcbb',
+ 'info_dict': {
+ 'id': 'ojz2wfah',
+ 'ext': 'mp3',
+ 'title': 'Krisson80 - bits wip wip',
+ 'description': '#Krisson80BitsWipWip #chiptune\n#wip',
+ },
+ }, {
+ 'url': 'https://clyp.it/ojz2wfah',
+ 'only_matching': True,
+ }]
+
+ def _real_extract(self, url):
+ audio_id = self._match_id(url)
+ api_url = 'https://api.clyp.it/' + audio_id
+ metadata = self._download_json(api_url, audio_id)
+
+ title = metadata['Title']
+
+ description = None
+ if metadata['Description']: description = metadata['Description']
+
+ duration = None
+ if metadata['Duration']: duration = int(metadata['Duration'])
+
+ formats = [
+ {
+ 'url': metadata['OggUrl'],
+ 'format_id': 'ogg',
+ 'preference': -2
+ },{
+ 'url': metadata['Mp3Url'],
+ 'format_id': 'mp3',
+ 'preference': -1
+ }]
+
+ return {
+ 'id': audio_id,
+ 'title': title,
+ 'formats': formats,
+ 'description': description,
+ 'duration': duration
+ }
From 52c3a6e49d2cbc1932992d816d28bbed629daadc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 28 Oct 2015 21:40:22 +0600
Subject: [PATCH 118/415] [utils] Improve parse_iso8601
---
test/test_utils.py | 2 ++
youtube_dl/utils.py | 13 +++++++++----
2 files changed, 11 insertions(+), 4 deletions(-)
diff --git a/test/test_utils.py b/test/test_utils.py
index 918a7a9ef..0c34f0e55 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -425,6 +425,8 @@ class TestUtil(unittest.TestCase):
self.assertEqual(parse_iso8601('2014-03-23T22:04:26+0000'), 1395612266)
self.assertEqual(parse_iso8601('2014-03-23T22:04:26Z'), 1395612266)
self.assertEqual(parse_iso8601('2014-03-23T22:04:26.1234Z'), 1395612266)
+ self.assertEqual(parse_iso8601('2015-09-29T08:27:31.727'), 1443515251)
+ self.assertEqual(parse_iso8601('2015-09-29T08-27-31.727'), None)
def test_strip_jsonp(self):
stripped = strip_jsonp('cb ([ {"id":"532cb",\n\n\n"x":\n3}\n]\n);')
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index a61e47646..558c9c7d5 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -814,9 +814,11 @@ def parse_iso8601(date_str, delimiter='T', timezone=None):
if date_str is None:
return None
+ date_str = re.sub(r'\.[0-9]+', '', date_str)
+
if timezone is None:
m = re.search(
- r'(\.[0-9]+)?(?:Z$| ?(?P<sign>\+|-)(?P<hours>[0-9]{2}):?(?P<minutes>[0-9]{2})$)',
+ r'(?:Z$| ?(?P<sign>\+|-)(?P<hours>[0-9]{2}):?(?P<minutes>[0-9]{2})$)',
date_str)
if not m:
timezone = datetime.timedelta()
@@ -829,9 +831,12 @@ def parse_iso8601(date_str, delimiter='T', timezone=None):
timezone = datetime.timedelta(
hours=sign * int(m.group('hours')),
minutes=sign * int(m.group('minutes')))
- date_format = '%Y-%m-%d{0}%H:%M:%S'.format(delimiter)
- dt = datetime.datetime.strptime(date_str, date_format) - timezone
- return calendar.timegm(dt.timetuple())
+ try:
+ date_format = '%Y-%m-%d{0}%H:%M:%S'.format(delimiter)
+ dt = datetime.datetime.strptime(date_str, date_format) - timezone
+ return calendar.timegm(dt.timetuple())
+ except ValueError:
+ pass
def unified_strdate(date_str, day_first=True):
From 03c2c162f9eaac6b474a1be9e985621f5b7b8c10 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 28 Oct 2015 21:42:01 +0600
Subject: [PATCH 119/415] [clyp] Improve and cleanup (Closes #7194)
---
youtube_dl/extractor/clyp.py | 62 ++++++++++++++++++------------------
1 file changed, 31 insertions(+), 31 deletions(-)
diff --git a/youtube_dl/extractor/clyp.py b/youtube_dl/extractor/clyp.py
index 906729b30..57e643799 100644
--- a/youtube_dl/extractor/clyp.py
+++ b/youtube_dl/extractor/clyp.py
@@ -1,16 +1,15 @@
-# coding: utf-8
-
from __future__ import unicode_literals
-import re
-
from .common import InfoExtractor
+from ..utils import (
+ float_or_none,
+ parse_iso8601,
+)
class ClypIE(InfoExtractor):
_VALID_URL = r'https?://(?:www\.)?clyp\.it/(?P<id>[a-z0-9]+)'
-
- _TESTS = [{
+ _TEST = {
'url': 'https://clyp.it/ojz2wfah',
'md5': '1d4961036c41247ecfdcc439c0cddcbb',
'info_dict': {
@@ -18,40 +17,41 @@ class ClypIE(InfoExtractor):
'ext': 'mp3',
'title': 'Krisson80 - bits wip wip',
'description': '#Krisson80BitsWipWip #chiptune\n#wip',
+ 'duration': 263.21,
+ 'timestamp': 1443515251,
+ 'upload_date': '20150929',
},
- }, {
- 'url': 'https://clyp.it/ojz2wfah',
- 'only_matching': True,
- }]
+ }
def _real_extract(self, url):
audio_id = self._match_id(url)
- api_url = 'https://api.clyp.it/' + audio_id
- metadata = self._download_json(api_url, audio_id)
+
+ metadata = self._download_json(
+ 'https://api.clyp.it/%s' % audio_id, audio_id)
+
+ formats = []
+ for secure in ('', 'Secure'):
+ for ext in ('Ogg', 'Mp3'):
+ format_id = '%s%s' % (secure, ext)
+ format_url = metadata.get('%sUrl' % format_id)
+ if format_url:
+ formats.append({
+ 'url': format_url,
+ 'format_id': format_id,
+ 'vcodec': 'none',
+ })
+ self._sort_formats(formats)
title = metadata['Title']
-
- description = None
- if metadata['Description']: description = metadata['Description']
-
- duration = None
- if metadata['Duration']: duration = int(metadata['Duration'])
-
- formats = [
- {
- 'url': metadata['OggUrl'],
- 'format_id': 'ogg',
- 'preference': -2
- },{
- 'url': metadata['Mp3Url'],
- 'format_id': 'mp3',
- 'preference': -1
- }]
+ description = metadata.get('Description')
+ duration = float_or_none(metadata.get('Duration'))
+ timestamp = parse_iso8601(metadata.get('DateCreated'))
return {
'id': audio_id,
'title': title,
- 'formats': formats,
'description': description,
- 'duration': duration
+ 'duration': duration,
+ 'timestamp': timestamp,
+ 'formats': formats,
}
From ae37338e681319a28d98dc551253d9fa1830969a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
<jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 29 Oct 2015 13:58:40 +0100
Subject: [PATCH 120/415] [compat] compat_etree_fromstring: clarify comment
---
youtube_dl/compat.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 2d43ec852..a3e85264a 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -216,8 +216,8 @@ except ImportError: # Python 2.6
if sys.version_info[0] >= 3:
compat_etree_fromstring = xml.etree.ElementTree.fromstring
else:
- # on python 2.x the attributes and text of a node aren't always unicode
- # objects
+ # python 2.x tries to encode unicode strings with ascii (see the
+ # XMLParser._fixtext method)
etree = xml.etree.ElementTree
try:
From 6fb8ace671db2f2bdcc9cd7ac6b9f81fbd356791 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 29 Oct 2015 22:44:01 +0600
Subject: [PATCH 121/415] [moniker] Add support for builtin embedded videos
(Closes #7244)
---
youtube_dl/extractor/moniker.py | 35 ++++++++++++++++++++++-----------
1 file changed, 24 insertions(+), 11 deletions(-)
diff --git a/youtube_dl/extractor/moniker.py b/youtube_dl/extractor/moniker.py
index 69e4bcd1a..204c03c4a 100644
--- a/youtube_dl/extractor/moniker.py
+++ b/youtube_dl/extractor/moniker.py
@@ -17,7 +17,7 @@ from ..utils import (
class MonikerIE(InfoExtractor):
IE_DESC = 'allmyvideos.net and vidspot.net'
- _VALID_URL = r'https?://(?:www\.)?(?:allmyvideos|vidspot)\.net/(?P<id>[a-zA-Z0-9_-]+)'
+ _VALID_URL = r'https?://(?:www\.)?(?:allmyvideos|vidspot)\.net/(?:(?:2|v)/v-)?(?P<id>[a-zA-Z0-9_-]+)'
_TESTS = [{
'url': 'http://allmyvideos.net/jih3nce3x6wn',
@@ -64,18 +64,30 @@ class MonikerIE(InfoExtractor):
raise ExtractorError(
'%s returned error: %s' % (self.IE_NAME, error), expected=True)
- fields = re.findall(r'type="hidden" name="(.+?)"\s* value="?(.+?)">', orig_webpage)
- data = dict(fields)
+ builtin_url = self._search_regex(
+ r'<iframe[^>]+src=(["\'])(?P<url>.+?/builtin-.+?)\1',
+ orig_webpage, 'builtin URL', default=None, group='url')
- post = compat_urllib_parse.urlencode(data)
- headers = {
- b'Content-Type': b'application/x-www-form-urlencoded',
- }
- req = compat_urllib_request.Request(url, post, headers)
- webpage = self._download_webpage(
- req, video_id, note='Downloading video page ...')
+ if builtin_url:
+ req = compat_urllib_request.Request(builtin_url)
+ req.add_header('Referer', url)
+ webpage = self._download_webpage(req, video_id, 'Downloading builtin page')
+ title = self._og_search_title(orig_webpage).strip()
+ description = self._og_search_description(orig_webpage).strip()
+ else:
+ fields = re.findall(r'type="hidden" name="(.+?)"\s* value="?(.+?)">', orig_webpage)
+ data = dict(fields)
- title = os.path.splitext(data['fname'])[0]
+ post = compat_urllib_parse.urlencode(data)
+ headers = {
+ b'Content-Type': b'application/x-www-form-urlencoded',
+ }
+ req = compat_urllib_request.Request(url, post, headers)
+ webpage = self._download_webpage(
+ req, video_id, note='Downloading video page ...')
+
+ title = os.path.splitext(data['fname'])[0]
+ description = None
# Could be several links with different quality
links = re.findall(r'"file" : "?(.+?)",', webpage)
@@ -89,5 +101,6 @@ class MonikerIE(InfoExtractor):
return {
'id': video_id,
'title': title,
+ 'description': description,
'formats': formats,
}
From 721f5a277ca0012ee72c9d4b3e5550e52a0a596d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 29 Oct 2015 22:47:18 +0600
Subject: [PATCH 122/415] [moniker] Add tests for #7244
---
youtube_dl/extractor/moniker.py | 12 ++++++++++++
1 file changed, 12 insertions(+)
diff --git a/youtube_dl/extractor/moniker.py b/youtube_dl/extractor/moniker.py
index 204c03c4a..7c0c4e50e 100644
--- a/youtube_dl/extractor/moniker.py
+++ b/youtube_dl/extractor/moniker.py
@@ -46,6 +46,18 @@ class MonikerIE(InfoExtractor):
}, {
'url': 'https://www.vidspot.net/l2ngsmhs8ci5',
'only_matching': True,
+ }, {
+ 'url': 'http://vidspot.net/2/v-ywDf99',
+ 'md5': '5f8254ce12df30479428b0152fb8e7ba',
+ 'info_dict': {
+ 'id': 'ywDf99',
+ 'ext': 'mp4',
+ 'title': 'IL FAIT LE MALIN EN PORSHE CAYENNE ( mais pas pour longtemps)',
+ 'description': 'IL FAIT LE MALIN EN PORSHE CAYENNE.',
+ },
+ }, {
+ 'url': 'http://allmyvideos.net/v/v-HXZm5t',
+ 'only_matching': True,
}]
def _real_extract(self, url):
From 6722ebd43720e836af8217fa078fa1a604b98229 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 30 Oct 2015 21:00:36 +0600
Subject: [PATCH 123/415] [anitube] Relax key regex (Closes #7303)
Another variant seen http://anitubebr.xpg.uol.com.br/embed/
---
youtube_dl/extractor/anitube.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/youtube_dl/extractor/anitube.py b/youtube_dl/extractor/anitube.py
index 31f0d417c..23f942ae2 100644
--- a/youtube_dl/extractor/anitube.py
+++ b/youtube_dl/extractor/anitube.py
@@ -26,8 +26,8 @@ class AnitubeIE(InfoExtractor):
video_id = mobj.group('id')
webpage = self._download_webpage(url, video_id)
- key = self._html_search_regex(
- r'http://www\.anitube\.se/embed/([A-Za-z0-9_-]*)', webpage, 'key')
+ key = self._search_regex(
+ r'src=["\']https?://[^/]+/embed/([A-Za-z0-9_-]+)', webpage, 'key')
config_xml = self._download_xml(
'http://www.anitube.se/nuevo/econfig.php?key=%s' % key, key)
From 47f2d01a5ac074f6959aa12e8bc00310f18a54e8 Mon Sep 17 00:00:00 2001
From: Lucas <mikotosc@gmail.com>
Date: Thu, 24 Sep 2015 22:19:09 +0200
Subject: [PATCH 124/415] Add new extractor
---
youtube_dl/extractor/__init__.py | 1 +
youtube_dl/extractor/kika.py | 115 +++++++++++++++++++++++++++++++
2 files changed, 116 insertions(+)
create mode 100644 youtube_dl/extractor/kika.py
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index f98e6487e..5ad4e9c36 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -274,6 +274,7 @@ from .karrierevideos import KarriereVideosIE
from .keezmovies import KeezMoviesIE
from .khanacademy import KhanAcademyIE
from .kickstarter import KickStarterIE
+from .kika import KikaIE
from .keek import KeekIE
from .kontrtube import KontrTubeIE
from .krasview import KrasViewIE
diff --git a/youtube_dl/extractor/kika.py b/youtube_dl/extractor/kika.py
new file mode 100644
index 000000000..db0f333ff
--- /dev/null
+++ b/youtube_dl/extractor/kika.py
@@ -0,0 +1,115 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import ExtractorError
+
+
+class KikaIE(InfoExtractor):
+ _VALID_URL = r'https?://(?:www\.)?kika\.de/(?:[a-z-]+/)*(?:video|sendung)(?P<id>\d+).*'
+
+ _TESTS = [
+ {
+ 'url': 'http://www.kika.de/baumhaus/videos/video9572.html',
+ 'md5': '94fc748cf5d64916571d275a07ffe2d5',
+ 'info_dict': {
+ 'id': '9572',
+ 'ext': 'mp4',
+ 'title': 'Baumhaus vom 29. Oktober 2014',
+ 'description': None
+ }
+ },
+ {
+ 'url': 'http://www.kika.de/sendungen/einzelsendungen/weihnachtsprogramm/videos/video8182.html',
+ 'md5': '5fe9c4dd7d71e3b238f04b8fdd588357',
+ 'info_dict': {
+ 'id': '8182',
+ 'ext': 'mp4',
+ 'title': 'Beutolomäus und der geheime Weihnachtswunsch',
+ 'description': 'md5:b69d32d7b2c55cbe86945ab309d39bbd'
+ }
+ },
+ {
+ 'url': 'http://www.kika.de/videos/allevideos/video9572_zc-32ca94ad_zs-3f535991.html',
+ 'md5': '94fc748cf5d64916571d275a07ffe2d5',
+ 'info_dict': {
+ 'id': '9572',
+ 'ext': 'mp4',
+ 'title': 'Baumhaus vom 29. Oktober 2014',
+ 'description': None
+ }
+ },
+ {
+ 'url': 'http://www.kika.de/sendungen/einzelsendungen/weihnachtsprogramm/videos/sendung81244_zc-81d703f8_zs-f82d5e31.html',
+ 'md5': '5fe9c4dd7d71e3b238f04b8fdd588357',
+ 'info_dict': {
+ 'id': '8182',
+ 'ext': 'mp4',
+ 'title': 'Beutolomäus und der geheime Weihnachtswunsch',
+ 'description': 'md5:b69d32d7b2c55cbe86945ab309d39bbd'
+ }
+ }
+ ]
+
+ def _real_extract(self, url):
+ # broadcast_id may be the same as the video_id
+ broadcast_id = self._match_id(url)
+ webpage = self._download_webpage(url, broadcast_id)
+
+ xml_re = r'sectionArticle[ "](?:(?!sectionA[ "])(?:.|\n))*?dataURL:\'(?:/[a-z-]+?)*?/video(\d+)-avCustom\.xml'
+ video_id = self._search_regex(xml_re, webpage, "xml_url", default=None)
+ if not video_id:
+ # Video is not available online
+ err_msg = 'Video %s is not available online' % broadcast_id
+ raise ExtractorError(err_msg, expected=True)
+
+ xml_url = 'http://www.kika.de/video%s-avCustom.xml' % (video_id)
+ xml_tree = self._download_xml(xml_url, video_id)
+
+ title = xml_tree.find('title').text
+ webpage_url = xml_tree.find('htmlUrl').text
+
+ # Try to get the description, not available for all videos
+ try:
+ broadcast_elem = xml_tree.find('broadcast')
+ description = broadcast_elem.find('broadcastDescription').text
+ except AttributeError:
+ # No description available
+ description = None
+
+ # duration string format is mm:ss (even if it is >= 1 hour, e.g. 78:42)
+ tmp = xml_tree.find('duration').text.split(':')
+ duration = int(tmp[0]) * 60 + int(tmp[1])
+
+ formats_list = []
+ for elem in xml_tree.find('assets'):
+ format_dict = {}
+ format_dict['url'] = elem.find('progressiveDownloadUrl').text
+ format_dict['ext'] = elem.find('mediaType').text.lower()
+ format_dict['format'] = elem.find('profileName').text
+ width = int(elem.find('frameWidth').text)
+ height = int(elem.find('frameHeight').text)
+ format_dict['width'] = width
+ format_dict['height'] = height
+ format_dict['resolution'] = '%dx%d' % (width, height)
+ format_dict['abr'] = int(elem.find('bitrateAudio').text)
+ format_dict['vbr'] = int(elem.find('bitrateVideo').text)
+ format_dict['tbr'] = format_dict['abr'] + format_dict['vbr']
+ format_dict['filesize'] = int(elem.find('fileSize').text)
+
+ # append resolution and dict for sorting by resolution
+ formats_list.append((width * height, format_dict))
+
+ # Sort by resolution (=quality)
+ formats_list.sort()
+
+ out_list = [x[1] for x in formats_list]
+
+ return {
+ 'id': video_id,
+ 'title': title,
+ 'description': description,
+ 'formats': out_list,
+ 'duration': duration,
+ 'webpage_url': webpage_url
+ }
From 892015b088fa21915270b0a05937fcc7063ccdd2 Mon Sep 17 00:00:00 2001
From: Lucas <mikotosc@gmail.com>
Date: Mon, 28 Sep 2015 22:00:56 +0200
Subject: [PATCH 125/415] replaced inefficient code
---
youtube_dl/extractor/kika.py | 18 +++++++-----------
1 file changed, 7 insertions(+), 11 deletions(-)
diff --git a/youtube_dl/extractor/kika.py b/youtube_dl/extractor/kika.py
index db0f333ff..871e4ea44 100644
--- a/youtube_dl/extractor/kika.py
+++ b/youtube_dl/extractor/kika.py
@@ -87,29 +87,25 @@ class KikaIE(InfoExtractor):
format_dict['url'] = elem.find('progressiveDownloadUrl').text
format_dict['ext'] = elem.find('mediaType').text.lower()
format_dict['format'] = elem.find('profileName').text
- width = int(elem.find('frameWidth').text)
- height = int(elem.find('frameHeight').text)
- format_dict['width'] = width
- format_dict['height'] = height
- format_dict['resolution'] = '%dx%d' % (width, height)
+ format_dict['width'] = int(elem.find('frameWidth').text)
+ format_dict['height'] = int(elem.find('frameHeight').text)
+ format_dict['resolution'] = '%dx%d' % (format_dict['width'],
+ format_dict['height'])
format_dict['abr'] = int(elem.find('bitrateAudio').text)
format_dict['vbr'] = int(elem.find('bitrateVideo').text)
format_dict['tbr'] = format_dict['abr'] + format_dict['vbr']
format_dict['filesize'] = int(elem.find('fileSize').text)
- # append resolution and dict for sorting by resolution
- formats_list.append((width * height, format_dict))
+ formats_list.append(format_dict)
# Sort by resolution (=quality)
- formats_list.sort()
-
- out_list = [x[1] for x in formats_list]
+ formats_list.sort(key=lambda x: x['width'] * x['height'])
return {
'id': video_id,
'title': title,
'description': description,
- 'formats': out_list,
+ 'formats': formats_list,
'duration': duration,
'webpage_url': webpage_url
}
From 78d7ee19dc417b16b26fe2fa1101124866727a85 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 31 Oct 2015 22:21:52 +0800
Subject: [PATCH 126/415] [democracynow] Fix _TESTS
---
youtube_dl/extractor/democracynow.py | 7 +++----
1 file changed, 3 insertions(+), 4 deletions(-)
diff --git a/youtube_dl/extractor/democracynow.py b/youtube_dl/extractor/democracynow.py
index 973bb437b..05cfc7502 100644
--- a/youtube_dl/extractor/democracynow.py
+++ b/youtube_dl/extractor/democracynow.py
@@ -36,10 +36,9 @@ class DemocracynowIE(InfoExtractor):
if display_id == '':
display_id = 'home'
webpage = self._download_webpage(url, display_id)
- re_desc = re.search(r'<meta property=.og:description. content=(["\'])(.+?)\1', webpage, re.DOTALL)
- description = re_desc.group(2) if re_desc else ''
+ description = self._og_search_description(webpage)
- jstr = self._search_regex(r'({.+?"related_video_xml".+?})', webpage, 'json', default=None)
+ jstr = self._search_regex(r'<script[^>]+type="text/json"[^>]*>\s*({[^>]+})', webpage, 'json')
js = self._parse_json(jstr, display_id)
video_id = None
formats = []
@@ -56,7 +55,7 @@ class DemocracynowIE(InfoExtractor):
'ext': ext,
'url': url,
}]
- for key in ('file', 'audio'):
+ for key in ('file', 'audio', 'video'):
url = js.get(key, '')
if url == '' or url is None:
continue
From 8c1aa28c27af204c4996260cdc70359e83c2c3d6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
<jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 31 Oct 2015 16:14:36 +0100
Subject: [PATCH 127/415] [kika] Replace non working tests and recognize
'einzelsendung' urls.
---
youtube_dl/extractor/kika.py | 20 ++++++++++----------
1 file changed, 10 insertions(+), 10 deletions(-)
diff --git a/youtube_dl/extractor/kika.py b/youtube_dl/extractor/kika.py
index 871e4ea44..c9169076a 100644
--- a/youtube_dl/extractor/kika.py
+++ b/youtube_dl/extractor/kika.py
@@ -6,16 +6,16 @@ from ..utils import ExtractorError
class KikaIE(InfoExtractor):
- _VALID_URL = r'https?://(?:www\.)?kika\.de/(?:[a-z-]+/)*(?:video|sendung)(?P<id>\d+).*'
+ _VALID_URL = r'https?://(?:www\.)?kika\.de/(?:[a-z-]+/)*(?:video|(?:einzel)?sendung)(?P<id>\d+).*'
_TESTS = [
{
- 'url': 'http://www.kika.de/baumhaus/videos/video9572.html',
- 'md5': '94fc748cf5d64916571d275a07ffe2d5',
+ 'url': 'http://www.kika.de/baumhaus/videos/video19636.html',
+ 'md5': '4930515e36b06c111213e80d1e4aad0e',
'info_dict': {
- 'id': '9572',
+ 'id': '19636',
'ext': 'mp4',
- 'title': 'Baumhaus vom 29. Oktober 2014',
+ 'title': 'Baumhaus vom 30. Oktober 2015',
'description': None
}
},
@@ -30,17 +30,17 @@ class KikaIE(InfoExtractor):
}
},
{
- 'url': 'http://www.kika.de/videos/allevideos/video9572_zc-32ca94ad_zs-3f535991.html',
- 'md5': '94fc748cf5d64916571d275a07ffe2d5',
+ 'url': 'http://www.kika.de/baumhaus/sendungen/video19636_zc-fea7f8a0_zs-4bf89c60.html',
+ 'md5': '4930515e36b06c111213e80d1e4aad0e',
'info_dict': {
- 'id': '9572',
+ 'id': '19636',
'ext': 'mp4',
- 'title': 'Baumhaus vom 29. Oktober 2014',
+ 'title': 'Baumhaus vom 30. Oktober 2015',
'description': None
}
},
{
- 'url': 'http://www.kika.de/sendungen/einzelsendungen/weihnachtsprogramm/videos/sendung81244_zc-81d703f8_zs-f82d5e31.html',
+ 'url': 'http://www.kika.de/sendungen/einzelsendungen/weihnachtsprogramm/einzelsendung2534.html',
'md5': '5fe9c4dd7d71e3b238f04b8fdd588357',
'info_dict': {
'id': '8182',
From c3040bd00a43e111dab0d1ab903df03ac7d19a00 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
<jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 31 Oct 2015 16:32:35 +0100
Subject: [PATCH 128/415] [kika] Cleanup
Closes #6957.
---
youtube_dl/extractor/kika.py | 54 +++++++++++++++---------------------
1 file changed, 22 insertions(+), 32 deletions(-)
diff --git a/youtube_dl/extractor/kika.py b/youtube_dl/extractor/kika.py
index c9169076a..5337ac439 100644
--- a/youtube_dl/extractor/kika.py
+++ b/youtube_dl/extractor/kika.py
@@ -16,8 +16,8 @@ class KikaIE(InfoExtractor):
'id': '19636',
'ext': 'mp4',
'title': 'Baumhaus vom 30. Oktober 2015',
- 'description': None
- }
+ 'description': None,
+ },
},
{
'url': 'http://www.kika.de/sendungen/einzelsendungen/weihnachtsprogramm/videos/video8182.html',
@@ -26,8 +26,8 @@ class KikaIE(InfoExtractor):
'id': '8182',
'ext': 'mp4',
'title': 'Beutolomäus und der geheime Weihnachtswunsch',
- 'description': 'md5:b69d32d7b2c55cbe86945ab309d39bbd'
- }
+ 'description': 'md5:b69d32d7b2c55cbe86945ab309d39bbd',
+ },
},
{
'url': 'http://www.kika.de/baumhaus/sendungen/video19636_zc-fea7f8a0_zs-4bf89c60.html',
@@ -36,8 +36,8 @@ class KikaIE(InfoExtractor):
'id': '19636',
'ext': 'mp4',
'title': 'Baumhaus vom 30. Oktober 2015',
- 'description': None
- }
+ 'description': None,
+ },
},
{
'url': 'http://www.kika.de/sendungen/einzelsendungen/weihnachtsprogramm/einzelsendung2534.html',
@@ -46,9 +46,9 @@ class KikaIE(InfoExtractor):
'id': '8182',
'ext': 'mp4',
'title': 'Beutolomäus und der geheime Weihnachtswunsch',
- 'description': 'md5:b69d32d7b2c55cbe86945ab309d39bbd'
- }
- }
+ 'description': 'md5:b69d32d7b2c55cbe86945ab309d39bbd',
+ },
+ },
]
def _real_extract(self, url):
@@ -59,7 +59,6 @@ class KikaIE(InfoExtractor):
xml_re = r'sectionArticle[ "](?:(?!sectionA[ "])(?:.|\n))*?dataURL:\'(?:/[a-z-]+?)*?/video(\d+)-avCustom\.xml'
video_id = self._search_regex(xml_re, webpage, "xml_url", default=None)
if not video_id:
- # Video is not available online
err_msg = 'Video %s is not available online' % broadcast_id
raise ExtractorError(err_msg, expected=True)
@@ -74,38 +73,29 @@ class KikaIE(InfoExtractor):
broadcast_elem = xml_tree.find('broadcast')
description = broadcast_elem.find('broadcastDescription').text
except AttributeError:
- # No description available
description = None
# duration string format is mm:ss (even if it is >= 1 hour, e.g. 78:42)
tmp = xml_tree.find('duration').text.split(':')
duration = int(tmp[0]) * 60 + int(tmp[1])
- formats_list = []
- for elem in xml_tree.find('assets'):
- format_dict = {}
- format_dict['url'] = elem.find('progressiveDownloadUrl').text
- format_dict['ext'] = elem.find('mediaType').text.lower()
- format_dict['format'] = elem.find('profileName').text
- format_dict['width'] = int(elem.find('frameWidth').text)
- format_dict['height'] = int(elem.find('frameHeight').text)
- format_dict['resolution'] = '%dx%d' % (format_dict['width'],
- format_dict['height'])
- format_dict['abr'] = int(elem.find('bitrateAudio').text)
- format_dict['vbr'] = int(elem.find('bitrateVideo').text)
- format_dict['tbr'] = format_dict['abr'] + format_dict['vbr']
- format_dict['filesize'] = int(elem.find('fileSize').text)
-
- formats_list.append(format_dict)
-
- # Sort by resolution (=quality)
- formats_list.sort(key=lambda x: x['width'] * x['height'])
+ formats = [{
+ 'url': elem.find('progressiveDownloadUrl').text,
+ 'ext': elem.find('mediaType').text.lower(),
+ 'format': elem.find('profileName').text,
+ 'width': int(elem.find('frameWidth').text),
+ 'height': int(elem.find('frameHeight').text),
+ 'abr': int(elem.find('bitrateAudio').text),
+ 'vbr': int(elem.find('bitrateVideo').text),
+ 'filesize': int(elem.find('fileSize').text),
+ } for elem in xml_tree.find('assets')]
+ self._sort_formats(formats)
return {
'id': video_id,
'title': title,
'description': description,
- 'formats': formats_list,
+ 'formats': formats,
'duration': duration,
- 'webpage_url': webpage_url
+ 'webpage_url': webpage_url,
}
From 2b1b2d83cacfdce19cae5eea2f9bbfd142efc7f1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 31 Oct 2015 22:17:09 +0600
Subject: [PATCH 129/415] [mdr] Modernize and include kika.de
---
youtube_dl/extractor/__init__.py | 1 -
youtube_dl/extractor/kika.py | 101 ------------------
youtube_dl/extractor/mdr.py | 172 +++++++++++++++++++++++--------
3 files changed, 131 insertions(+), 143 deletions(-)
delete mode 100644 youtube_dl/extractor/kika.py
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 5ad4e9c36..f98e6487e 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -274,7 +274,6 @@ from .karrierevideos import KarriereVideosIE
from .keezmovies import KeezMoviesIE
from .khanacademy import KhanAcademyIE
from .kickstarter import KickStarterIE
-from .kika import KikaIE
from .keek import KeekIE
from .kontrtube import KontrTubeIE
from .krasview import KrasViewIE
diff --git a/youtube_dl/extractor/kika.py b/youtube_dl/extractor/kika.py
deleted file mode 100644
index 5337ac439..000000000
--- a/youtube_dl/extractor/kika.py
+++ /dev/null
@@ -1,101 +0,0 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
-from .common import InfoExtractor
-from ..utils import ExtractorError
-
-
-class KikaIE(InfoExtractor):
- _VALID_URL = r'https?://(?:www\.)?kika\.de/(?:[a-z-]+/)*(?:video|(?:einzel)?sendung)(?P<id>\d+).*'
-
- _TESTS = [
- {
- 'url': 'http://www.kika.de/baumhaus/videos/video19636.html',
- 'md5': '4930515e36b06c111213e80d1e4aad0e',
- 'info_dict': {
- 'id': '19636',
- 'ext': 'mp4',
- 'title': 'Baumhaus vom 30. Oktober 2015',
- 'description': None,
- },
- },
- {
- 'url': 'http://www.kika.de/sendungen/einzelsendungen/weihnachtsprogramm/videos/video8182.html',
- 'md5': '5fe9c4dd7d71e3b238f04b8fdd588357',
- 'info_dict': {
- 'id': '8182',
- 'ext': 'mp4',
- 'title': 'Beutolomäus und der geheime Weihnachtswunsch',
- 'description': 'md5:b69d32d7b2c55cbe86945ab309d39bbd',
- },
- },
- {
- 'url': 'http://www.kika.de/baumhaus/sendungen/video19636_zc-fea7f8a0_zs-4bf89c60.html',
- 'md5': '4930515e36b06c111213e80d1e4aad0e',
- 'info_dict': {
- 'id': '19636',
- 'ext': 'mp4',
- 'title': 'Baumhaus vom 30. Oktober 2015',
- 'description': None,
- },
- },
- {
- 'url': 'http://www.kika.de/sendungen/einzelsendungen/weihnachtsprogramm/einzelsendung2534.html',
- 'md5': '5fe9c4dd7d71e3b238f04b8fdd588357',
- 'info_dict': {
- 'id': '8182',
- 'ext': 'mp4',
- 'title': 'Beutolomäus und der geheime Weihnachtswunsch',
- 'description': 'md5:b69d32d7b2c55cbe86945ab309d39bbd',
- },
- },
- ]
-
- def _real_extract(self, url):
- # broadcast_id may be the same as the video_id
- broadcast_id = self._match_id(url)
- webpage = self._download_webpage(url, broadcast_id)
-
- xml_re = r'sectionArticle[ "](?:(?!sectionA[ "])(?:.|\n))*?dataURL:\'(?:/[a-z-]+?)*?/video(\d+)-avCustom\.xml'
- video_id = self._search_regex(xml_re, webpage, "xml_url", default=None)
- if not video_id:
- err_msg = 'Video %s is not available online' % broadcast_id
- raise ExtractorError(err_msg, expected=True)
-
- xml_url = 'http://www.kika.de/video%s-avCustom.xml' % (video_id)
- xml_tree = self._download_xml(xml_url, video_id)
-
- title = xml_tree.find('title').text
- webpage_url = xml_tree.find('htmlUrl').text
-
- # Try to get the description, not available for all videos
- try:
- broadcast_elem = xml_tree.find('broadcast')
- description = broadcast_elem.find('broadcastDescription').text
- except AttributeError:
- description = None
-
- # duration string format is mm:ss (even if it is >= 1 hour, e.g. 78:42)
- tmp = xml_tree.find('duration').text.split(':')
- duration = int(tmp[0]) * 60 + int(tmp[1])
-
- formats = [{
- 'url': elem.find('progressiveDownloadUrl').text,
- 'ext': elem.find('mediaType').text.lower(),
- 'format': elem.find('profileName').text,
- 'width': int(elem.find('frameWidth').text),
- 'height': int(elem.find('frameHeight').text),
- 'abr': int(elem.find('bitrateAudio').text),
- 'vbr': int(elem.find('bitrateVideo').text),
- 'filesize': int(elem.find('fileSize').text),
- } for elem in xml_tree.find('assets')]
- self._sort_formats(formats)
-
- return {
- 'id': video_id,
- 'title': title,
- 'description': description,
- 'formats': formats,
- 'duration': duration,
- 'webpage_url': webpage_url,
- }
diff --git a/youtube_dl/extractor/mdr.py b/youtube_dl/extractor/mdr.py
index fc7499958..541ddd909 100644
--- a/youtube_dl/extractor/mdr.py
+++ b/youtube_dl/extractor/mdr.py
@@ -1,64 +1,154 @@
+# coding: utf-8
from __future__ import unicode_literals
-import re
-
from .common import InfoExtractor
+from ..compat import compat_urlparse
+from ..utils import (
+ determine_ext,
+ int_or_none,
+ parse_duration,
+ parse_iso8601,
+ xpath_text,
+)
class MDRIE(InfoExtractor):
- _VALID_URL = r'^(?P<domain>https?://(?:www\.)?mdr\.de)/(?:.*)/(?P<type>video|audio)(?P<video_id>[^/_]+)(?:_|\.html)'
+ IE_DESC = 'MDR.DE and KiKA'
+ _VALID_URL = r'https?://(?:www\.)?(?:mdr|kika)\.de/(?:.*)/[a-z]+(?P<id>\d+)(?:_.+?)?\.html'
- # No tests, MDR regularily deletes its videos
- _TEST = {
+ _TESTS = [{
+ # MDR regularily deletes its videos
'url': 'http://www.mdr.de/fakt/video189002.html',
'only_matching': True,
- }
+ }, {
+ 'url': 'http://www.kika.de/baumhaus/videos/video19636.html',
+ 'md5': '4930515e36b06c111213e80d1e4aad0e',
+ 'info_dict': {
+ 'id': '19636',
+ 'ext': 'mp4',
+ 'title': 'Baumhaus vom 30. Oktober 2015',
+ 'duration': 134,
+ 'uploader': 'KIKA',
+ },
+ }, {
+ 'url': 'http://www.kika.de/sendungen/einzelsendungen/weihnachtsprogramm/videos/video8182.html',
+ 'md5': '5fe9c4dd7d71e3b238f04b8fdd588357',
+ 'info_dict': {
+ 'id': '8182',
+ 'ext': 'mp4',
+ 'title': 'Beutolomäus und der geheime Weihnachtswunsch',
+ 'description': 'md5:b69d32d7b2c55cbe86945ab309d39bbd',
+ 'timestamp': 1419047100,
+ 'upload_date': '20141220',
+ 'duration': 4628,
+ 'uploader': 'KIKA',
+ },
+ }, {
+ 'url': 'http://www.kika.de/baumhaus/sendungen/video19636_zc-fea7f8a0_zs-4bf89c60.html',
+ 'only_matching': True,
+ }, {
+ 'url': 'http://www.kika.de/sendungen/einzelsendungen/weihnachtsprogramm/einzelsendung2534.html',
+ 'only_matching': True,
+ }]
def _real_extract(self, url):
- m = re.match(self._VALID_URL, url)
- video_id = m.group('video_id')
- domain = m.group('domain')
+ video_id = self._match_id(url)
- # determine title and media streams from webpage
- html = self._download_webpage(url, video_id)
+ webpage = self._download_webpage(url, video_id)
- title = self._html_search_regex(r'<h[12]>(.*?)</h[12]>', html, 'title')
- xmlurl = self._search_regex(
- r'dataURL:\'(/(?:.+)/(?:video|audio)[0-9]+-avCustom.xml)', html, 'XML URL')
+ data_url = self._search_regex(
+ r'dataURL\s*:\s*(["\'])(?P<url>/.+/(?:video|audio)[0-9]+-avCustom\.xml)\1',
+ webpage, 'data url', group='url')
+
+ doc = self._download_xml(
+ compat_urlparse.urljoin(url, data_url), video_id)
+
+ title = (xpath_text(doc, './title', 'title', default=None) or
+ xpath_text(doc, './broadcast/broadcastName', 'title'))
- doc = self._download_xml(domain + xmlurl, video_id)
formats = []
- for a in doc.findall('./assets/asset'):
- url_el = a.find('./progressiveDownloadUrl')
- if url_el is None:
- continue
- abr = int(a.find('bitrateAudio').text) // 1000
- media_type = a.find('mediaType').text
- format = {
- 'abr': abr,
- 'filesize': int(a.find('fileSize').text),
- 'url': url_el.text,
- }
+ processed_urls = []
+ for asset in doc.findall('./assets/asset'):
+ for source in (
+ 'progressiveDownload',
+ 'dynamicHttpStreamingRedirector',
+ 'adaptiveHttpStreamingRedirector'):
+ url_el = asset.find('./%sUrl' % source)
+ if url_el is None:
+ continue
- vbr_el = a.find('bitrateVideo')
- if vbr_el is None:
- format.update({
- 'vcodec': 'none',
- 'format_id': '%s-%d' % (media_type, abr),
- })
- else:
- vbr = int(vbr_el.text) // 1000
- format.update({
- 'vbr': vbr,
- 'width': int(a.find('frameWidth').text),
- 'height': int(a.find('frameHeight').text),
- 'format_id': '%s-%d' % (media_type, vbr),
- })
- formats.append(format)
+ video_url = url_el.text
+ if video_url in processed_urls:
+ continue
+
+ processed_urls.append(video_url)
+
+ vbr = int_or_none(xpath_text(asset, './bitrateVideo', 'vbr'), 1000)
+ abr = int_or_none(xpath_text(asset, './bitrateAudio', 'abr'), 1000)
+
+ url_formats = []
+
+ ext = determine_ext(url_el.text)
+ if ext == 'm3u8':
+ url_formats = self._extract_m3u8_formats(
+ video_url, video_id, 'mp4', entry_protocol='m3u8_native',
+ preference=0, m3u8_id='HLS', fatal=False)
+ elif ext == 'f4m':
+ url_formats = self._extract_f4m_formats(
+ video_url + '?hdcore=3.7.0&plugin=aasp-3.7.0.39.44', video_id,
+ preference=0, f4m_id='HDS', fatal=False)
+ else:
+ media_type = xpath_text(asset, './mediaType', 'media type', default='MP4')
+ vbr = int_or_none(xpath_text(asset, './bitrateVideo', 'vbr'), 1000)
+ abr = int_or_none(xpath_text(asset, './bitrateAudio', 'abr'), 1000)
+ filesize = int_or_none(xpath_text(asset, './fileSize', 'file size'))
+
+ f = {
+ 'url': video_url,
+ 'format_id': '%s-%d' % (media_type, vbr or abr),
+ 'filesize': filesize,
+ 'abr': abr,
+ 'preference': 1,
+ }
+
+ if vbr:
+ width = int_or_none(xpath_text(asset, './frameWidth', 'width'))
+ height = int_or_none(xpath_text(asset, './frameHeight', 'height'))
+ f.update({
+ 'vbr': vbr,
+ 'width': width,
+ 'height': height,
+ })
+
+ url_formats.append(f)
+
+ if not vbr:
+ for f in url_formats:
+ abr = f.get('tbr') or abr
+ if 'tbr' in f:
+ del f['tbr']
+ f.update({
+ 'abr': abr,
+ 'vcodec': 'none',
+ })
+
+ if url_formats:
+ formats.extend(url_formats)
self._sort_formats(formats)
+ description = xpath_text(doc, './broadcast/broadcastDescription', 'description')
+ timestamp = parse_iso8601(
+ xpath_text(doc, './broadcast/broadcastDate', 'timestamp', default=None) or
+ xpath_text(doc, './broadcast/broadcastStartDate', 'timestamp', default=None))
+ duration = parse_duration(xpath_text(doc, './duration', 'duration'))
+ uploader = xpath_text(doc, './rights', 'uploader')
+
return {
'id': video_id,
'title': title,
+ 'description': description,
+ 'timestamp': timestamp,
+ 'duration': duration,
+ 'uploader': uploader,
'formats': formats,
}
From 8cdb5c845336ad3dc48c85a0558a38bd42972b00 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 31 Oct 2015 22:24:21 +0600
Subject: [PATCH 130/415] [mdr] Add audio test
---
youtube_dl/extractor/mdr.py | 11 +++++++++++
1 file changed, 11 insertions(+)
diff --git a/youtube_dl/extractor/mdr.py b/youtube_dl/extractor/mdr.py
index 541ddd909..e05577496 100644
--- a/youtube_dl/extractor/mdr.py
+++ b/youtube_dl/extractor/mdr.py
@@ -20,6 +20,17 @@ class MDRIE(InfoExtractor):
# MDR regularily deletes its videos
'url': 'http://www.mdr.de/fakt/video189002.html',
'only_matching': True,
+ }, {
+ # audio
+ 'url': 'http://www.mdr.de/kultur/audio1312272_zc-15948bad_zs-86171fdd.html',
+ 'md5': '64c4ee50f0a791deb9479cd7bbe9d2fa',
+ 'info_dict': {
+ 'id': '1312272',
+ 'ext': 'mp3',
+ 'title': 'Feuilleton vom 30. Oktober 2015',
+ 'duration': 250,
+ 'uploader': 'MITTELDEUTSCHER RUNDFUNK',
+ },
}, {
'url': 'http://www.kika.de/baumhaus/videos/video19636.html',
'md5': '4930515e36b06c111213e80d1e4aad0e',
From 578c074575f45ffdfd032d7b84f6fe449614f511 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 31 Oct 2015 22:39:44 +0600
Subject: [PATCH 131/415] [utils] Support list of xpath in xpath_element
---
test/test_utils.py | 7 +++++++
youtube_dl/utils.py | 15 ++++++++++++---
2 files changed, 19 insertions(+), 3 deletions(-)
diff --git a/test/test_utils.py b/test/test_utils.py
index 0c34f0e55..5a56ad776 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -275,9 +275,16 @@ class TestUtil(unittest.TestCase):
p = xml.etree.ElementTree.SubElement(div, 'p')
p.text = 'Foo'
self.assertEqual(xpath_element(doc, 'div/p'), p)
+ self.assertEqual(xpath_element(doc, ['div/p']), p)
+ self.assertEqual(xpath_element(doc, ['div/bar', 'div/p']), p)
self.assertEqual(xpath_element(doc, 'div/bar', default='default'), 'default')
+ self.assertEqual(xpath_element(doc, ['div/bar'], default='default'), 'default')
self.assertTrue(xpath_element(doc, 'div/bar') is None)
+ self.assertTrue(xpath_element(doc, ['div/bar']) is None)
+ self.assertTrue(xpath_element(doc, ['div/bar'], 'div/baz') is None)
self.assertRaises(ExtractorError, xpath_element, doc, 'div/bar', fatal=True)
+ self.assertRaises(ExtractorError, xpath_element, doc, ['div/bar'], fatal=True)
+ self.assertRaises(ExtractorError, xpath_element, doc, ['div/bar', 'div/baz'], fatal=True)
def test_xpath_text(self):
testxml = '''<root>
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 558c9c7d5..89c88a4d3 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -178,10 +178,19 @@ def xpath_with_ns(path, ns_map):
def xpath_element(node, xpath, name=None, fatal=False, default=NO_DEFAULT):
- if sys.version_info < (2, 7): # Crazy 2.6
- xpath = xpath.encode('ascii')
+ def _find_xpath(xpath):
+ if sys.version_info < (2, 7): # Crazy 2.6
+ xpath = xpath.encode('ascii')
+ return node.find(xpath)
+
+ if isinstance(xpath, (str, compat_str)):
+ n = _find_xpath(xpath)
+ else:
+ for xp in xpath:
+ n = _find_xpath(xp)
+ if n is not None:
+ break
- n = node.find(xpath)
if n is None:
if default is not NO_DEFAULT:
return default
From 11465da70257663ee52c7be50debe1c1e825ec67 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 31 Oct 2015 22:45:45 +0600
Subject: [PATCH 132/415] [mdr] Simplify xpath
---
youtube_dl/extractor/mdr.py | 11 +++++++----
1 file changed, 7 insertions(+), 4 deletions(-)
diff --git a/youtube_dl/extractor/mdr.py b/youtube_dl/extractor/mdr.py
index e05577496..a63257c56 100644
--- a/youtube_dl/extractor/mdr.py
+++ b/youtube_dl/extractor/mdr.py
@@ -74,8 +74,7 @@ class MDRIE(InfoExtractor):
doc = self._download_xml(
compat_urlparse.urljoin(url, data_url), video_id)
- title = (xpath_text(doc, './title', 'title', default=None) or
- xpath_text(doc, './broadcast/broadcastName', 'title'))
+ title = xpath_text(doc, ['./title', './broadcast/broadcastName'], 'title', fatal=True)
formats = []
processed_urls = []
@@ -149,8 +148,12 @@ class MDRIE(InfoExtractor):
description = xpath_text(doc, './broadcast/broadcastDescription', 'description')
timestamp = parse_iso8601(
- xpath_text(doc, './broadcast/broadcastDate', 'timestamp', default=None) or
- xpath_text(doc, './broadcast/broadcastStartDate', 'timestamp', default=None))
+ xpath_text(
+ doc, [
+ './broadcast/broadcastDate',
+ './broadcast/broadcastStartDate',
+ './broadcast/broadcastEndDate'],
+ 'timestamp', default=None))
duration = parse_duration(xpath_text(doc, './duration', 'duration'))
uploader = xpath_text(doc, './rights', 'uploader')
From 82b69a5cbb1455d31916be4f19ab327ae63f313c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 31 Oct 2015 23:00:36 +0600
Subject: [PATCH 133/415] [mdr] PEP 8
---
youtube_dl/extractor/mdr.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/mdr.py b/youtube_dl/extractor/mdr.py
index a63257c56..a566c6a2c 100644
--- a/youtube_dl/extractor/mdr.py
+++ b/youtube_dl/extractor/mdr.py
@@ -20,7 +20,7 @@ class MDRIE(InfoExtractor):
# MDR regularily deletes its videos
'url': 'http://www.mdr.de/fakt/video189002.html',
'only_matching': True,
- }, {
+ }, {
# audio
'url': 'http://www.mdr.de/kultur/audio1312272_zc-15948bad_zs-86171fdd.html',
'md5': '64c4ee50f0a791deb9479cd7bbe9d2fa',
From e327b736ca6a6a1c880b93e09a3b310c354c2c7c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 31 Oct 2015 23:05:30 +0600
Subject: [PATCH 134/415] [generic] Update test
---
youtube_dl/extractor/generic.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index ca5fbafb2..a84135032 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -141,6 +141,7 @@ class GenericIE(InfoExtractor):
'ext': 'mp4',
'title': 'Automatics, robotics and biocybernetics',
'description': 'md5:815fc1deb6b3a2bff99de2d5325be482',
+ 'upload_date': '20130627',
'formats': 'mincount:16',
'subtitles': 'mincount:1',
},
From ae12bc3ebb4cb377c2b4337ec255e652b36f5143 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 31 Oct 2015 23:07:37 +0600
Subject: [PATCH 135/415] [utils] Make unified_strdate always return unicode
string
---
youtube_dl/utils.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 89c88a4d3..764a89cca 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -910,7 +910,7 @@ def unified_strdate(date_str, day_first=True):
timetuple = email.utils.parsedate_tz(date_str)
if timetuple:
upload_date = datetime.datetime(*timetuple[:6]).strftime('%Y%m%d')
- return upload_date
+ return compat_str(upload_date)
def determine_ext(url, default_ext='unknown_video'):
From dc519b5421366a8cac681455a817ae25f7f4aa83 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 31 Oct 2015 23:12:57 +0600
Subject: [PATCH 136/415] [extractor/common] Make ie_key and IE_NAME return
unicode string
---
youtube_dl/extractor/common.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 10c0d5d1f..59c3fa8dc 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -310,11 +310,11 @@ class InfoExtractor(object):
@classmethod
def ie_key(cls):
"""A string for getting the InfoExtractor with get_info_extractor"""
- return cls.__name__[:-2]
+ return compat_str(cls.__name__[:-2])
@property
def IE_NAME(self):
- return type(self).__name__[:-2]
+ return compat_str(type(self).__name__[:-2])
def _request_webpage(self, url_or_request, video_id, note=None, errnote=None, fatal=True):
""" Returns the response handle """
From 76f0c50d3d3e2eb5903b61da08829699e902916d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 1 Nov 2015 00:01:08 +0600
Subject: [PATCH 137/415] [mdr] Fix failed formats processing
---
youtube_dl/extractor/mdr.py | 11 ++++++-----
1 file changed, 6 insertions(+), 5 deletions(-)
diff --git a/youtube_dl/extractor/mdr.py b/youtube_dl/extractor/mdr.py
index a566c6a2c..88334889e 100644
--- a/youtube_dl/extractor/mdr.py
+++ b/youtube_dl/extractor/mdr.py
@@ -96,8 +96,6 @@ class MDRIE(InfoExtractor):
vbr = int_or_none(xpath_text(asset, './bitrateVideo', 'vbr'), 1000)
abr = int_or_none(xpath_text(asset, './bitrateAudio', 'abr'), 1000)
- url_formats = []
-
ext = determine_ext(url_el.text)
if ext == 'm3u8':
url_formats = self._extract_m3u8_formats(
@@ -130,7 +128,10 @@ class MDRIE(InfoExtractor):
'height': height,
})
- url_formats.append(f)
+ url_formats = [f]
+
+ if not url_formats:
+ continue
if not vbr:
for f in url_formats:
@@ -142,8 +143,8 @@ class MDRIE(InfoExtractor):
'vcodec': 'none',
})
- if url_formats:
- formats.extend(url_formats)
+ formats.extend(url_formats)
+
self._sort_formats(formats)
description = xpath_text(doc, './broadcast/broadcastDescription', 'description')
From dbd82a1d4fff1655920e111cc25a7fd526d7bf9a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 1 Nov 2015 00:01:34 +0600
Subject: [PATCH 138/415] [extractor/common] Fix m3u8 extraction on failure
---
youtube_dl/extractor/common.py | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 59c3fa8dc..1f09fbb47 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -943,13 +943,14 @@ class InfoExtractor(object):
if re.match(r'^https?://', u)
else compat_urlparse.urljoin(m3u8_url, u))
- m3u8_doc, urlh = self._download_webpage_handle(
+ res = self._download_webpage_handle(
m3u8_url, video_id,
note=note or 'Downloading m3u8 information',
errnote=errnote or 'Failed to download m3u8 information',
fatal=fatal)
- if m3u8_doc is False:
- return m3u8_doc
+ if res is False:
+ return res
+ m3u8_doc, urlh = res
m3u8_url = urlh.geturl()
last_info = None
last_media = None
From 9550ca506fccf9c9d795816cc0a7817ff262ef45 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 31 Oct 2015 19:36:04 +0100
Subject: [PATCH 139/415] [utils] change extract_attributes to work in python 2
---
youtube_dl/extractor/brightcove.py | 3 +--
youtube_dl/utils.py | 3 ++-
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index b41cee91b..c6ad1d065 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -383,8 +383,7 @@ class BrightcoveInPageEmbedIE(InfoExtractor):
return None
def _real_extract(self, url):
- mobj = re.match(self._VALID_URL, url)
- account_id, player_id, embed, video_id = mobj.groups()
+ account_id, player_id, embed, video_id = re.match(self._VALID_URL, url).groups()
webpage = self._download_webpage('http://players.brightcove.net/%s/%s_%s/index.min.js' % (account_id, player_id, embed), video_id)
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index bcebf9cc5..518cea98b 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -252,7 +252,8 @@ def extract_attributes(attributes_str, attributes_regex=r'(?s)\s*([^\s=]+)\s*=\s
attributes = re.findall(attributes_regex, attributes_str)
attributes_dict = {}
if attributes:
- attributes_dict = {attribute_name: attribute_value for (attribute_name, attribute_value) in attributes}
+ for (attribute_name, attribute_value) in attributes:
+ attributes_dict[attribute_name] = attribute_value
return attributes_dict
From 80dcee5cd5cbe623a53e0c582e3e3ae170c63e8d Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 31 Oct 2015 04:02:49 +0100
Subject: [PATCH 140/415] [eitb] fix info extraction
---
youtube_dl/extractor/eitb.py | 65 ++++++++++++++++++++++++------------
1 file changed, 44 insertions(+), 21 deletions(-)
diff --git a/youtube_dl/extractor/eitb.py b/youtube_dl/extractor/eitb.py
index 2cba82532..fc8f15544 100644
--- a/youtube_dl/extractor/eitb.py
+++ b/youtube_dl/extractor/eitb.py
@@ -1,39 +1,62 @@
# encoding: utf-8
from __future__ import unicode_literals
-import re
-
from .common import InfoExtractor
-from .brightcove import BrightcoveIE
-from ..utils import ExtractorError
+from ..compat import compat_urllib_request
+from ..utils import (
+ int_or_none,
+ unified_strdate,
+)
class EitbIE(InfoExtractor):
IE_NAME = 'eitb.tv'
- _VALID_URL = r'https?://www\.eitb\.tv/(eu/bideoa|es/video)/[^/]+/(?P<playlist_id>\d+)/(?P<chapter_id>\d+)'
+ _VALID_URL = r'https?://www\.eitb\.tv/(eu/bideoa|es/video)/[^/]+/\d+/(?P<id>\d+)'
_TEST = {
- 'add_ie': ['Brightcove'],
- 'url': 'http://www.eitb.tv/es/video/60-minutos-60-minutos-2013-2014/2677100210001/2743577154001/lasa-y-zabala-30-anos/',
+ 'url': 'http://www.eitb.tv/es/video/60-minutos-60-minutos-2013-2014/4104995148001/4090227752001/lasa-y-zabala-30-anos/',
'md5': 'edf4436247185adee3ea18ce64c47998',
'info_dict': {
- 'id': '2743577154001',
+ 'id': '4090227752001',
'ext': 'mp4',
'title': '60 minutos (Lasa y Zabala, 30 años)',
- # All videos from eitb has this description in the brightcove info
- 'description': '.',
- 'uploader': 'Euskal Telebista',
+ 'description': '',
+ 'duration': 3996760,
+ 'upload_date': '20131014',
},
}
def _real_extract(self, url):
- mobj = re.match(self._VALID_URL, url)
- chapter_id = mobj.group('chapter_id')
- webpage = self._download_webpage(url, chapter_id)
- bc_url = BrightcoveIE._extract_brightcove_url(webpage)
- if bc_url is None:
- raise ExtractorError('Could not extract the Brightcove url')
- # The BrightcoveExperience object doesn't contain the video id, we set
- # it manually
- bc_url += '&%40videoPlayer={0}'.format(chapter_id)
- return self.url_result(bc_url, BrightcoveIE.ie_key())
+ video_id = self._match_id(url)
+ video_data = self._download_json('http://mam.eitb.eus/mam/REST/ServiceMultiweb/Video/MULTIWEBTV/%s/' % video_id, video_id)['web_media'][0]
+
+ formats = []
+ for rendition in video_data['RENDITIONS']:
+ formats.append({
+ 'url': rendition['PMD_URL'],
+ 'width': int_or_none(rendition.get('FRAME_WIDTH')),
+ 'height': int_or_none(rendition.get('FRAME_HEIGHT')),
+ 'tbr': int_or_none(rendition.get('ENCODING_RATE')),
+ })
+
+ # TODO: parse f4m manifest
+ request = compat_urllib_request.Request(
+ 'http://mam.eitb.eus/mam/REST/ServiceMultiweb/DomainRestrictedSecurity/TokenAuth/',
+ headers={'Referer': url})
+ token_data = self._download_json(request, video_id, fatal=False)
+ if token_data:
+ m3u8_formats = self._extract_m3u8_formats('%s?hdnts=%s' % (video_data['HLS_SURL'], token_data['token']), video_id, m3u8_id='hls', fatal=False)
+ if m3u8_formats:
+ formats.extend(m3u8_formats)
+
+ self._sort_formats(formats)
+
+ return {
+ 'id': video_id,
+ 'title': video_data['NAME_ES'],
+ 'description': video_data.get('SHORT_DESC_ES'),
+ 'thumbnail': video_data.get('STILL_URL'),
+ 'duration': int_or_none(video_data.get('LENGTH')),
+ 'upload_date': unified_strdate(video_data.get('BROADCST_DATE')),
+ 'formats': formats,
+ }
From 8a06999ba0f9c948f8d2a1ef89c73eedbfb09cfb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 1 Nov 2015 01:52:33 +0600
Subject: [PATCH 141/415] [eitb] Improve, make more robust and extract f4m
formats (Closes #7328)
---
youtube_dl/extractor/eitb.py | 71 +++++++++++++++++++++++++-----------
1 file changed, 50 insertions(+), 21 deletions(-)
diff --git a/youtube_dl/extractor/eitb.py b/youtube_dl/extractor/eitb.py
index fc8f15544..0de8d3dc6 100644
--- a/youtube_dl/extractor/eitb.py
+++ b/youtube_dl/extractor/eitb.py
@@ -4,14 +4,15 @@ from __future__ import unicode_literals
from .common import InfoExtractor
from ..compat import compat_urllib_request
from ..utils import (
+ float_or_none,
int_or_none,
- unified_strdate,
+ parse_iso8601,
)
class EitbIE(InfoExtractor):
IE_NAME = 'eitb.tv'
- _VALID_URL = r'https?://www\.eitb\.tv/(eu/bideoa|es/video)/[^/]+/\d+/(?P<id>\d+)'
+ _VALID_URL = r'https?://(?:www\.)?eitb\.tv/(?:eu/bideoa|es/video)/[^/]+/\d+/(?P<id>\d+)'
_TEST = {
'url': 'http://www.eitb.tv/es/video/60-minutos-60-minutos-2013-2014/4104995148001/4090227752001/lasa-y-zabala-30-anos/',
@@ -20,43 +21,71 @@ class EitbIE(InfoExtractor):
'id': '4090227752001',
'ext': 'mp4',
'title': '60 minutos (Lasa y Zabala, 30 años)',
- 'description': '',
- 'duration': 3996760,
+ 'description': 'Programa de reportajes de actualidad.',
+ 'duration': 3996.76,
+ 'timestamp': 1381789200,
'upload_date': '20131014',
+ 'tags': list,
},
}
def _real_extract(self, url):
video_id = self._match_id(url)
- video_data = self._download_json('http://mam.eitb.eus/mam/REST/ServiceMultiweb/Video/MULTIWEBTV/%s/' % video_id, video_id)['web_media'][0]
+
+ video = self._download_json(
+ 'http://mam.eitb.eus/mam/REST/ServiceMultiweb/Video/MULTIWEBTV/%s/' % video_id,
+ video_id, 'Downloading video JSON')
+
+ media = video['web_media'][0]
formats = []
- for rendition in video_data['RENDITIONS']:
+ for rendition in media['RENDITIONS']:
+ video_url = rendition.get('PMD_URL')
+ if not video_url:
+ continue
+ tbr = float_or_none(rendition.get('ENCODING_RATE'), 1000)
+ format_id = 'http'
+ if tbr:
+ format_id += '-%d' % int(tbr)
formats.append({
'url': rendition['PMD_URL'],
+ 'format_id': format_id,
'width': int_or_none(rendition.get('FRAME_WIDTH')),
'height': int_or_none(rendition.get('FRAME_HEIGHT')),
- 'tbr': int_or_none(rendition.get('ENCODING_RATE')),
+ 'tbr': tbr,
})
- # TODO: parse f4m manifest
- request = compat_urllib_request.Request(
- 'http://mam.eitb.eus/mam/REST/ServiceMultiweb/DomainRestrictedSecurity/TokenAuth/',
- headers={'Referer': url})
- token_data = self._download_json(request, video_id, fatal=False)
- if token_data:
- m3u8_formats = self._extract_m3u8_formats('%s?hdnts=%s' % (video_data['HLS_SURL'], token_data['token']), video_id, m3u8_id='hls', fatal=False)
- if m3u8_formats:
- formats.extend(m3u8_formats)
+ hls_url = media.get('HLS_SURL')
+ if hls_url:
+ request = compat_urllib_request.Request(
+ 'http://mam.eitb.eus/mam/REST/ServiceMultiweb/DomainRestrictedSecurity/TokenAuth/',
+ headers={'Referer': url})
+ token_data = self._download_json(
+ request, video_id, 'Downloading auth token', fatal=False)
+ if token_data:
+ token = token_data.get('token')
+ if token:
+ m3u8_formats = self._extract_m3u8_formats(
+ '%s?hdnts=%s' % (hls_url, token), video_id, m3u8_id='hls', fatal=False)
+ if m3u8_formats:
+ formats.extend(m3u8_formats)
+
+ hds_url = media.get('HDS_SURL').replace('euskalsvod', 'euskalvod')
+ if hds_url:
+ f4m_formats = self._extract_f4m_formats(
+ '%s?hdcore=3.7.0' % hds_url, video_id, f4m_id='hds', fatal=False)
+ if f4m_formats:
+ formats.extend(f4m_formats)
self._sort_formats(formats)
return {
'id': video_id,
- 'title': video_data['NAME_ES'],
- 'description': video_data.get('SHORT_DESC_ES'),
- 'thumbnail': video_data.get('STILL_URL'),
- 'duration': int_or_none(video_data.get('LENGTH')),
- 'upload_date': unified_strdate(video_data.get('BROADCST_DATE')),
+ 'title': media.get('NAME_ES') or media.get('name') or media['NAME_EU'],
+ 'description': media.get('SHORT_DESC_ES') or video.get('desc_group') or media.get('SHORT_DESC_EU'),
+ 'thumbnail': media.get('STILL_URL') or media.get('THUMBNAIL_URL'),
+ 'duration': float_or_none(media.get('LENGTH'), 1000),
+ 'timestamp': parse_iso8601(media.get('BROADCST_DATE'), ' '),
+ 'tags': media.get('TAGS'),
'formats': formats,
}
From 999079b4543b4cd5e71a235865fbfefd349eb064 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 1 Nov 2015 15:49:11 +0600
Subject: [PATCH 142/415] [eitb] Improve hds extraction
---
youtube_dl/extractor/eitb.py | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/youtube_dl/extractor/eitb.py b/youtube_dl/extractor/eitb.py
index 0de8d3dc6..357a2196c 100644
--- a/youtube_dl/extractor/eitb.py
+++ b/youtube_dl/extractor/eitb.py
@@ -70,10 +70,11 @@ class EitbIE(InfoExtractor):
if m3u8_formats:
formats.extend(m3u8_formats)
- hds_url = media.get('HDS_SURL').replace('euskalsvod', 'euskalvod')
+ hds_url = media.get('HDS_SURL')
if hds_url:
f4m_formats = self._extract_f4m_formats(
- '%s?hdcore=3.7.0' % hds_url, video_id, f4m_id='hds', fatal=False)
+ '%s?hdcore=3.7.0' % hds_url.replace('euskalsvod', 'euskalvod'),
+ video_id, f4m_id='hds', fatal=False)
if f4m_formats:
formats.extend(f4m_formats)
From ab6ca0480280abb2a35a54e1b380bbae07a48863 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 1 Nov 2015 14:20:10 +0100
Subject: [PATCH 143/415] release 2015.11.01
---
docs/supportedsites.md | 3 ++-
youtube_dl/version.py | 2 +-
2 files changed, 3 insertions(+), 2 deletions(-)
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 03561b87d..805af14a0 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -93,6 +93,7 @@
- **Clipsyndicate**
- **Cloudy**
- **Clubic**
+ - **Clyp**
- **cmt.com**
- **CNET**
- **CNN**
@@ -281,7 +282,7 @@
- **macgamestore**: MacGameStore trailers
- **mailru**: Видео@Mail.Ru
- **Malemotion**
- - **MDR**
+ - **MDR**: MDR.DE and KiKA
- **media.ccc.de**
- **metacafe**
- **Metacritic**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 125e8ccf5..006b973c0 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
from __future__ import unicode_literals
-__version__ = '2015.10.24'
+__version__ = '2015.11.01'
From c90d16cf36d8edf03f4dc923ee9dbeadca910844 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 2 Nov 2015 04:26:20 +0600
Subject: [PATCH 144/415] [utils:sanitize_path] Disallow trailing whitespace in
path segment (Closes #7332)
---
youtube_dl/utils.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index efd5f4ae1..7b3f79141 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -366,7 +366,7 @@ def sanitize_path(s):
if drive_or_unc:
norm_path.pop(0)
sanitized_path = [
- path_part if path_part in ['.', '..'] else re.sub('(?:[/<>:"\\|\\\\?\\*]|\.$)', '#', path_part)
+ path_part if path_part in ['.', '..'] else re.sub('(?:[/<>:"\\|\\\\?\\*]|[\s.]$)', '#', path_part)
for path_part in norm_path]
if drive_or_unc:
sanitized_path.insert(0, drive_or_unc + os.path.sep)
From eb97f46e8bd9cb04f0fe5f8a5c13aeeaabeefef5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
<jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 2 Nov 2015 12:46:10 +0100
Subject: [PATCH 145/415] [mitele] Fix extraction and update test checksum
(fixes #7343)
---
youtube_dl/extractor/mitele.py | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
diff --git a/youtube_dl/extractor/mitele.py b/youtube_dl/extractor/mitele.py
index 3142fcde2..c595f2077 100644
--- a/youtube_dl/extractor/mitele.py
+++ b/youtube_dl/extractor/mitele.py
@@ -1,7 +1,10 @@
from __future__ import unicode_literals
from .common import InfoExtractor
-from ..compat import compat_urllib_parse
+from ..compat import (
+ compat_urllib_parse,
+ compat_urlparse,
+)
from ..utils import (
encode_dict,
get_element_by_attribute,
@@ -15,7 +18,7 @@ class MiTeleIE(InfoExtractor):
_TESTS = [{
'url': 'http://www.mitele.es/programas-tv/diario-de/la-redaccion/programa-144/',
- 'md5': '757b0b66cbd7e0a97226d7d3156cb3e9',
+ 'md5': '0ff1a13aebb35d9bc14081ff633dd324',
'info_dict': {
'id': '0NF1jJnxS1Wu3pHrmvFyw2',
'display_id': 'programa-144',
@@ -34,6 +37,7 @@ class MiTeleIE(InfoExtractor):
config_url = self._search_regex(
r'data-config\s*=\s*"([^"]+)"', webpage, 'data config url')
+ config_url = compat_urlparse.urljoin(url, config_url)
config = self._download_json(
config_url, display_id, 'Downloading config JSON')
From c514b0ec655b23e7804eb18df04daa863d973f32 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sun, 1 Nov 2015 22:12:20 +0100
Subject: [PATCH 146/415] [videofy.me] fix info extraction
Closes #7339.
---
youtube_dl/extractor/videofyme.py | 40 ++++++++++++++++---------------
1 file changed, 21 insertions(+), 19 deletions(-)
diff --git a/youtube_dl/extractor/videofyme.py b/youtube_dl/extractor/videofyme.py
index 94f9e9be9..cd3f50a63 100644
--- a/youtube_dl/extractor/videofyme.py
+++ b/youtube_dl/extractor/videofyme.py
@@ -2,8 +2,8 @@ from __future__ import unicode_literals
from .common import InfoExtractor
from ..utils import (
- find_xpath_attr,
int_or_none,
+ parse_iso8601,
)
@@ -18,33 +18,35 @@ class VideofyMeIE(InfoExtractor):
'id': '1100701',
'ext': 'mp4',
'title': 'This is VideofyMe',
- 'description': None,
+ 'description': '',
+ 'upload_date': '20130326',
+ 'timestamp': 1364288959,
'uploader': 'VideofyMe',
'uploader_id': 'thisisvideofyme',
'view_count': int,
+ 'likes': int,
+ 'comment_count': int,
},
-
}
def _real_extract(self, url):
video_id = self._match_id(url)
- config = self._download_xml('http://sunshine.videofy.me/?videoId=%s' % video_id,
- video_id)
- video = config.find('video')
- sources = video.find('sources')
- url_node = next(node for node in [find_xpath_attr(sources, 'source', 'id', 'HQ %s' % key)
- for key in ['on', 'av', 'off']] if node is not None)
- video_url = url_node.find('url').text
- view_count = int_or_none(self._search_regex(
- r'([0-9]+)', video.find('views').text, 'view count', fatal=False))
+
+ config = self._download_json('http://vf-player-info-loader.herokuapp.com/%s.json' % video_id, video_id)['videoinfo']
+
+ video = config.get('video')
+ blog = config.get('blog', {})
return {
'id': video_id,
- 'title': video.find('title').text,
- 'url': video_url,
- 'thumbnail': video.find('thumb').text,
- 'description': video.find('description').text,
- 'uploader': config.find('blog/name').text,
- 'uploader_id': video.find('identifier').text,
- 'view_count': view_count,
+ 'title': video['title'],
+ 'url': video['sources']['source']['url'],
+ 'thumbnail': video.get('thumb'),
+ 'description': video.get('description'),
+ 'timestamp': parse_iso8601(video.get('date')),
+ 'uploader': blog.get('name'),
+ 'uploader_id': blog.get('identifier'),
+ 'view_count': int_or_none(self._search_regex(r'([0-9]+)', video.get('views'), 'view count', fatal=False)),
+ 'likes': int_or_none(video.get('likes')),
+ 'comment_count': int_or_none(video.get('nrOfComments')),
}
From 6a750402787dfc1f39a9ad347f2d78ae1c94c52c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
<jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 2 Nov 2015 14:08:38 +0100
Subject: [PATCH 147/415] [utils] unified_strdate: Return None if the date
format can't be recognized (fixes #7340)
This issue was introduced with ae12bc3ebb4cb377c2b4337ec255e652b36f5143, it returned 'None'.
---
test/test_utils.py | 1 +
youtube_dl/utils.py | 3 ++-
2 files changed, 3 insertions(+), 1 deletion(-)
diff --git a/test/test_utils.py b/test/test_utils.py
index 3298315d2..01829f71e 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -236,6 +236,7 @@ class TestUtil(unittest.TestCase):
unified_strdate('2/2/2015 6:47:40 PM', day_first=False),
'20150202')
self.assertEqual(unified_strdate('25-09-2014'), '20140925')
+ self.assertEqual(unified_strdate('UNKNOWN DATE FORMAT'), None)
def test_find_xpath_attr(self):
testxml = '''<root>
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 7b3f79141..d39f313a4 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -911,7 +911,8 @@ def unified_strdate(date_str, day_first=True):
timetuple = email.utils.parsedate_tz(date_str)
if timetuple:
upload_date = datetime.datetime(*timetuple[:6]).strftime('%Y%m%d')
- return compat_str(upload_date)
+ if upload_date is not None:
+ return compat_str(upload_date)
def determine_ext(url, default_ext='unknown_video'):
From a230068ff7427c19e29331fc0f2bb17d50003bca Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 2 Nov 2015 16:18:54 +0100
Subject: [PATCH 148/415] release 2015.11.02
---
youtube_dl/version.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 006b973c0..6ef482b78 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
from __future__ import unicode_literals
-__version__ = '2015.11.01'
+__version__ = '2015.11.02'
From dde9fe9788f23f168e0bddaf8ab0470f469165fa Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 3 Nov 2015 18:36:54 +0800
Subject: [PATCH 149/415] [democracynow] Simplify
---
youtube_dl/extractor/democracynow.py | 86 ++++++++++++++--------------
1 file changed, 44 insertions(+), 42 deletions(-)
diff --git a/youtube_dl/extractor/democracynow.py b/youtube_dl/extractor/democracynow.py
index 05cfc7502..824b8e2c5 100644
--- a/youtube_dl/extractor/democracynow.py
+++ b/youtube_dl/extractor/democracynow.py
@@ -2,11 +2,18 @@
from __future__ import unicode_literals
import re
+import os.path
+
from .common import InfoExtractor
+from ..compat import compat_urlparse
+from ..utils import (
+ url_basename,
+ remove_start,
+)
class DemocracynowIE(InfoExtractor):
- _VALID_URL = r'https?://(?:www\.)?democracynow.org/?(?P<id>[^\?]*)'
+ _VALID_URL = r'https?://(?:www\.)?democracynow.org/(?P<id>[^\?]*)'
IE_NAME = 'democracynow'
_TESTS = [{
'url': 'http://www.democracynow.org/shows/2015/7/3',
@@ -14,9 +21,7 @@ class DemocracynowIE(InfoExtractor):
'id': '2015-0703-001',
'ext': 'mp4',
'title': 'July 03, 2015 - Democracy Now!',
- 'description': 'A daily independent global news hour with Amy Goodman & Juan Gonz\xe1lez "What to the Slave is 4th of July?": James Earl Jones Reads Frederick Douglass\u2019 Historic Speech : "This Flag Comes Down Today": Bree Newsome Scales SC Capitol Flagpole, Takes Down Confederate Flag : "We Shall Overcome": Remembering Folk Icon, Activist Pete Seeger in His Own Words & Songs',
- 'uploader': 'Democracy Now',
- 'upload_date': None,
+ 'description': 'A daily independent global news hour with Amy Goodman & Juan González "What to the Slave is 4th of July?": James Earl Jones Reads Frederick Douglass\u2019 Historic Speech : "This Flag Comes Down Today": Bree Newsome Scales SC Capitol Flagpole, Takes Down Confederate Flag : "We Shall Overcome": Remembering Folk Icon, Activist Pete Seeger in His Own Words & Songs',
},
}, {
'url': 'http://www.democracynow.org/2015/7/3/this_flag_comes_down_today_bree',
@@ -25,60 +30,57 @@ class DemocracynowIE(InfoExtractor):
'ext': 'mp4',
'title': '"This Flag Comes Down Today": Bree Newsome Scales SC Capitol Flagpole, Takes Down Confederate Flag',
'description': 'md5:4d2bc4f0d29f5553c2210a4bc7761a21',
- 'uploader': 'Democracy Now',
- 'upload_date': None,
},
}]
def _real_extract(self, url):
display_id = self._match_id(url)
- base_host = re.search(r'^(.+?://[^/]+)', url).group(1)
- if display_id == '':
- display_id = 'home'
webpage = self._download_webpage(url, display_id)
description = self._og_search_description(webpage)
- jstr = self._search_regex(r'<script[^>]+type="text/json"[^>]*>\s*({[^>]+})', webpage, 'json')
- js = self._parse_json(jstr, display_id)
+ js = self._parse_json(self._search_regex(
+ r'<script[^>]+type="text/json"[^>]*>\s*({[^>]+})', webpage, 'json'),
+ display_id)
video_id = None
formats = []
+
+ default_lang = 'en'
+
subtitles = {}
- for key in ('caption_file', '.......'):
- # ....... = pending vtt support that doesn't clobber srt 'chapter_file':
- url = js.get(key, '')
- if url == '' or url is None:
- continue
- if not re.match(r'^https?://', url):
- url = base_host + url
- ext = re.search(r'\.([^\.]+)$', url).group(1)
- subtitles['eng'] = [{
- 'ext': ext,
- 'url': url,
- }]
- for key in ('file', 'audio', 'video'):
- url = js.get(key, '')
- if url == '' or url is None:
- continue
- if not re.match(r'^https?://', url):
- url = base_host + url
- purl = re.search(r'/(?P<dir>[^/]+)/(?:dn)?(?P<fn>[^/]+?)\.(?P<ext>[^\.\?]+)(?P<hasparams>\?|$)', url)
- if video_id is None:
- video_id = purl.group('fn')
- if js.get('start') is not None:
- url += '&' if purl.group('hasparams') == '?' else '?'
- url = url + 'start=' + str(js.get('start'))
- formats.append({
- 'format_id': purl.group('dir'),
- 'ext': purl.group('ext'),
- 'url': url,
+
+ def add_subtitle_item(lang, info_dict):
+ if lang not in subtitles:
+ subtitles[lang] = []
+ subtitles[lang].append(info_dict)
+
+ # chapter_file are not subtitles
+ if 'caption_file' in js:
+ add_subtitle_item(default_lang, {
+ 'url': compat_urlparse.urljoin(url, js['caption_file']),
})
+
+ for subtitle_item in js.get('captions', []):
+ lang = subtitle_item.get('language', '').lower() or default_lang
+ add_subtitle_item(lang, {
+ 'url': compat_urlparse.urljoin(url, subtitle_item['url']),
+ })
+
+ for key in ('file', 'audio', 'video'):
+ media_url = js.get(key, '')
+ if not media_url:
+ continue
+ media_url = re.sub(r'\?.*', '', compat_urlparse.urljoin(url, media_url))
+ video_id = video_id or remove_start(os.path.splitext(url_basename(media_url))[0], 'dn')
+ formats.append({
+ 'url': media_url,
+ })
+
self._sort_formats(formats)
- ret = {
+
+ return {
'id': video_id,
'title': js.get('title'),
'description': description,
- 'uploader': 'Democracy Now',
'subtitles': subtitles,
'formats': formats,
}
- return ret
From fc68d52bb95dc81ed3d05a5c5397cd3f35ee093a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 3 Nov 2015 21:24:10 +0800
Subject: [PATCH 150/415] [democracynow] Add MD5 sums
---
youtube_dl/extractor/democracynow.py | 2 ++
1 file changed, 2 insertions(+)
diff --git a/youtube_dl/extractor/democracynow.py b/youtube_dl/extractor/democracynow.py
index 824b8e2c5..70c364e8b 100644
--- a/youtube_dl/extractor/democracynow.py
+++ b/youtube_dl/extractor/democracynow.py
@@ -17,6 +17,7 @@ class DemocracynowIE(InfoExtractor):
IE_NAME = 'democracynow'
_TESTS = [{
'url': 'http://www.democracynow.org/shows/2015/7/3',
+ 'md5': 'fbb8fe3d7a56a5e12431ce2f9b2fab0d',
'info_dict': {
'id': '2015-0703-001',
'ext': 'mp4',
@@ -25,6 +26,7 @@ class DemocracynowIE(InfoExtractor):
},
}, {
'url': 'http://www.democracynow.org/2015/7/3/this_flag_comes_down_today_bree',
+ 'md5': 'fbb8fe3d7a56a5e12431ce2f9b2fab0d',
'info_dict': {
'id': '2015-0703-001',
'ext': 'mp4',
From 852fad922ffa931b3c90b0b9fdb2fa1c7f965ab4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 3 Nov 2015 20:53:17 +0600
Subject: [PATCH 151/415] [vimeo] Fix non-ASCII video passwords (Closes #7352)
---
youtube_dl/extractor/vimeo.py | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 2437ae1eb..cc0d337e8 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -13,6 +13,7 @@ from ..compat import (
compat_urlparse,
)
from ..utils import (
+ encode_dict,
ExtractorError,
InAdvancePagedList,
int_or_none,
@@ -208,10 +209,10 @@ class VimeoIE(VimeoBaseInfoExtractor):
if password is None:
raise ExtractorError('This video is protected by a password, use the --video-password option', expected=True)
token, vuid = self._extract_xsrft_and_vuid(webpage)
- data = urlencode_postdata({
+ data = urlencode_postdata(encode_dict({
'password': password,
'token': token,
- })
+ }))
if url.startswith('http://'):
# vimeo only supports https now, but the user can give an http url
url = url.replace('http://', 'https://')
From 0a0110fc6bbd21850e25541fd0bd4b602ce194e2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 3 Nov 2015 21:01:09 +0600
Subject: [PATCH 152/415] [vimeo] Fix non-ASCII video passwords (2)
---
youtube_dl/extractor/vimeo.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index cc0d337e8..fa07bd59c 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -228,7 +228,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
password = self._downloader.params.get('videopassword', None)
if password is None:
raise ExtractorError('This video is protected by a password, use the --video-password option')
- data = compat_urllib_parse.urlencode({'password': password})
+ data = urlencode_postdata(encode_dict({'password': password}))
pass_url = url + '/check-password'
password_request = compat_urllib_request.Request(pass_url, data)
password_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
From 3fa3ff1bc36aaf82ac0a5e880304cb7aae217b9a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 3 Nov 2015 21:06:36 +0600
Subject: [PATCH 153/415] [vimeo] Fix non-ASCII login
---
youtube_dl/extractor/vimeo.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index fa07bd59c..46fb36f21 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -41,13 +41,13 @@ class VimeoBaseInfoExtractor(InfoExtractor):
self.report_login()
webpage = self._download_webpage(self._LOGIN_URL, None, False)
token, vuid = self._extract_xsrft_and_vuid(webpage)
- data = urlencode_postdata({
+ data = urlencode_postdata(encode_dict({
'action': 'login',
'email': username,
'password': password,
'service': 'vimeo',
'token': token,
- })
+ }))
login_request = compat_urllib_request.Request(self._LOGIN_URL, data)
login_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
login_request.add_header('Cookie', 'vuid=%s' % vuid)
From bfdf891fd36811909aa5d83dc0614eacbb634fcf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 3 Nov 2015 21:09:24 +0600
Subject: [PATCH 154/415] [vimeo] Fix non-ASCII album passwords
---
youtube_dl/extractor/vimeo.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 46fb36f21..b608740b8 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -489,7 +489,7 @@ class VimeoChannelIE(VimeoBaseInfoExtractor):
token, vuid = self._extract_xsrft_and_vuid(webpage)
fields['token'] = token
fields['password'] = password
- post = urlencode_postdata(fields)
+ post = urlencode_postdata(encode_dict(fields))
password_path = self._search_regex(
r'action="([^"]+)"', login_form, 'password URL')
password_url = compat_urlparse.urljoin(page_url, password_path)
From fd8102820c4d14fdb1ff7e090553211717012f67 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 4 Nov 2015 00:09:55 +0800
Subject: [PATCH 155/415] [democracynow] Rename js to json_data
---
youtube_dl/extractor/democracynow.py | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
diff --git a/youtube_dl/extractor/democracynow.py b/youtube_dl/extractor/democracynow.py
index 70c364e8b..72fc75d80 100644
--- a/youtube_dl/extractor/democracynow.py
+++ b/youtube_dl/extractor/democracynow.py
@@ -40,7 +40,7 @@ class DemocracynowIE(InfoExtractor):
webpage = self._download_webpage(url, display_id)
description = self._og_search_description(webpage)
- js = self._parse_json(self._search_regex(
+ json_data = self._parse_json(self._search_regex(
r'<script[^>]+type="text/json"[^>]*>\s*({[^>]+})', webpage, 'json'),
display_id)
video_id = None
@@ -56,19 +56,19 @@ class DemocracynowIE(InfoExtractor):
subtitles[lang].append(info_dict)
# chapter_file are not subtitles
- if 'caption_file' in js:
+ if 'caption_file' in json_data:
add_subtitle_item(default_lang, {
- 'url': compat_urlparse.urljoin(url, js['caption_file']),
+ 'url': compat_urlparse.urljoin(url, json_data['caption_file']),
})
- for subtitle_item in js.get('captions', []):
+ for subtitle_item in json_data.get('captions', []):
lang = subtitle_item.get('language', '').lower() or default_lang
add_subtitle_item(lang, {
'url': compat_urlparse.urljoin(url, subtitle_item['url']),
})
for key in ('file', 'audio', 'video'):
- media_url = js.get(key, '')
+ media_url = json_data.get(key, '')
if not media_url:
continue
media_url = re.sub(r'\?.*', '', compat_urlparse.urljoin(url, media_url))
@@ -81,7 +81,7 @@ class DemocracynowIE(InfoExtractor):
return {
'id': video_id,
- 'title': js.get('title'),
+ 'title': json_data.get('title'),
'description': description,
'subtitles': subtitles,
'formats': formats,
From 0aeb9a106e1aad37967e0ee666ed816a7d5eb7c2 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 4 Nov 2015 00:13:00 +0800
Subject: [PATCH 156/415] [democracynow] Prevent required fields to be None
---
youtube_dl/extractor/democracynow.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/youtube_dl/extractor/democracynow.py b/youtube_dl/extractor/democracynow.py
index 72fc75d80..6cd395e11 100644
--- a/youtube_dl/extractor/democracynow.py
+++ b/youtube_dl/extractor/democracynow.py
@@ -80,8 +80,8 @@ class DemocracynowIE(InfoExtractor):
self._sort_formats(formats)
return {
- 'id': video_id,
- 'title': json_data.get('title'),
+ 'id': video_id or display_id,
+ 'title': json_data['title'],
'description': description,
'subtitles': subtitles,
'formats': formats,
From 66d041f250f7d3e0c4d501e3b98721f2c6588c35 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 4 Nov 2015 00:53:30 +0800
Subject: [PATCH 157/415] [test/subtitles] Add test for DemocracynowIE
---
test/test_subtitles.py | 21 +++++++++++++++++++++
1 file changed, 21 insertions(+)
diff --git a/test/test_subtitles.py b/test/test_subtitles.py
index 0343967d9..75f0ea75f 100644
--- a/test/test_subtitles.py
+++ b/test/test_subtitles.py
@@ -28,6 +28,7 @@ from youtube_dl.extractor import (
ThePlatformFeedIE,
RTVEALaCartaIE,
FunnyOrDieIE,
+ DemocracynowIE,
)
@@ -346,5 +347,25 @@ class TestFunnyOrDieSubtitles(BaseTestSubtitles):
self.assertEqual(md5(subtitles['en']), 'c5593c193eacd353596c11c2d4f9ecc4')
+class TestDemocracynowSubtitles(BaseTestSubtitles):
+ url = 'http://www.democracynow.org/shows/2015/7/3'
+ IE = DemocracynowIE
+
+ def test_allsubtitles(self):
+ self.DL.params['writesubtitles'] = True
+ self.DL.params['allsubtitles'] = True
+ subtitles = self.getSubtitles()
+ self.assertEqual(set(subtitles.keys()), set(['en']))
+ self.assertEqual(md5(subtitles['en']), 'acaca989e24a9e45a6719c9b3d60815c')
+
+ def test_subtitles_in_page(self):
+ self.url = 'http://www.democracynow.org/2015/7/3/this_flag_comes_down_today_bree'
+ self.DL.params['writesubtitles'] = True
+ self.DL.params['allsubtitles'] = True
+ subtitles = self.getSubtitles()
+ self.assertEqual(set(subtitles.keys()), set(['en']))
+ self.assertEqual(md5(subtitles['en']), 'acaca989e24a9e45a6719c9b3d60815c')
+
+
if __name__ == '__main__':
unittest.main()
From ad607563a2fbb5275ea39f7a052c09ffa232e271 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 4 Nov 2015 16:46:26 +0600
Subject: [PATCH 158/415] [globo] Separate article extractor
---
youtube_dl/extractor/__init__.py | 5 +-
youtube_dl/extractor/globo.py | 140 +++++++++++++++++--------------
2 files changed, 79 insertions(+), 66 deletions(-)
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 10286aa88..94150a28f 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -212,7 +212,10 @@ from .gfycat import GfycatIE
from .giantbomb import GiantBombIE
from .giga import GigaIE
from .glide import GlideIE
-from .globo import GloboIE
+from .globo import (
+ GloboIE,
+ GloboArticleIE,
+)
from .godtube import GodTubeIE
from .goldenmoustache import GoldenMoustacheIE
from .golem import GolemIE
diff --git a/youtube_dl/extractor/globo.py b/youtube_dl/extractor/globo.py
index 33d6432a6..828e40d76 100644
--- a/youtube_dl/extractor/globo.py
+++ b/youtube_dl/extractor/globo.py
@@ -18,75 +18,52 @@ from ..utils import (
class GloboIE(InfoExtractor):
- _VALID_URL = 'https?://.+?\.globo\.com/(?P<id>.+)'
+ _VALID_URL = '(?:globo:|https?://.+?\.globo\.com/(?:[^/]+/)*(?:v/(?:[^/]+/)?|videos/))(?P<id>\d{7,})'
_API_URL_TEMPLATE = 'http://api.globovideos.com/videos/%s/playlist'
_SECURITY_URL_TEMPLATE = 'http://security.video.globo.com/videos/%s/hash?player=flash&version=17.0.0.132&resource_id=%s'
- _VIDEOID_REGEXES = [
- r'\bdata-video-id="(\d+)"',
- r'\bdata-player-videosids="(\d+)"',
- r'<div[^>]+\bid="(\d+)"',
- ]
-
_RESIGN_EXPIRATION = 86400
- _TESTS = [
- {
- 'url': 'http://globotv.globo.com/sportv/futebol-nacional/v/os-gols-de-atletico-mg-3-x-2-santos-pela-24a-rodada-do-brasileirao/3654973/',
- 'md5': '03ebf41cb7ade43581608b7d9b71fab0',
- 'info_dict': {
- 'id': '3654973',
- 'ext': 'mp4',
- 'title': 'Os gols de Atlético-MG 3 x 2 Santos pela 24ª rodada do Brasileirão',
- 'duration': 251.585,
- 'uploader': 'SporTV',
- 'uploader_id': 698,
- 'like_count': int,
- }
- },
- {
- 'url': 'http://g1.globo.com/carros/autoesporte/videos/t/exclusivos-do-g1/v/mercedes-benz-gla-passa-por-teste-de-colisao-na-europa/3607726/',
- 'md5': 'b3ccc801f75cd04a914d51dadb83a78d',
- 'info_dict': {
- 'id': '3607726',
- 'ext': 'mp4',
- 'title': 'Mercedes-Benz GLA passa por teste de colisão na Europa',
- 'duration': 103.204,
- 'uploader': 'Globo.com',
- 'uploader_id': 265,
- 'like_count': int,
- }
- },
- {
- 'url': 'http://g1.globo.com/jornal-nacional/noticia/2014/09/novidade-na-fiscalizacao-de-bagagem-pela-receita-provoca-discussoes.html',
- 'md5': '307fdeae4390ccfe6ba1aa198cf6e72b',
- 'info_dict': {
- 'id': '3652183',
- 'ext': 'mp4',
- 'title': 'Receita Federal explica como vai fiscalizar bagagens de quem retorna ao Brasil de avião',
- 'duration': 110.711,
- 'uploader': 'Rede Globo',
- 'uploader_id': 196,
- 'like_count': int,
- }
- },
- {
- 'url': 'http://globotv.globo.com/canal-brasil/sangue-latino/t/todos-os-videos/v/ator-e-diretor-argentino-ricado-darin-fala-sobre-utopias-e-suas-perdas/3928201/',
- 'md5': 'c1defca721ce25b2354e927d3e4b3dec',
- 'info_dict': {
- 'id': '3928201',
- 'ext': 'mp4',
- 'title': 'Ator e diretor argentino, Ricado Darín fala sobre utopias e suas perdas',
- 'duration': 1472.906,
- 'uploader': 'Canal Brasil',
- 'uploader_id': 705,
- 'like_count': int,
- }
- },
- ]
+ _TESTS = [{
+ 'url': 'http://globotv.globo.com/sportv/futebol-nacional/v/os-gols-de-atletico-mg-3-x-2-santos-pela-24a-rodada-do-brasileirao/3654973/',
+ 'md5': '03ebf41cb7ade43581608b7d9b71fab0',
+ 'info_dict': {
+ 'id': '3654973',
+ 'ext': 'mp4',
+ 'title': 'Os gols de Atlético-MG 3 x 2 Santos pela 24ª rodada do Brasileirão',
+ 'duration': 251.585,
+ 'uploader': 'SporTV',
+ 'uploader_id': 698,
+ 'like_count': int,
+ }
+ }, {
+ 'url': 'http://g1.globo.com/carros/autoesporte/videos/t/exclusivos-do-g1/v/mercedes-benz-gla-passa-por-teste-de-colisao-na-europa/3607726/',
+ 'md5': 'b3ccc801f75cd04a914d51dadb83a78d',
+ 'info_dict': {
+ 'id': '3607726',
+ 'ext': 'mp4',
+ 'title': 'Mercedes-Benz GLA passa por teste de colisão na Europa',
+ 'duration': 103.204,
+ 'uploader': 'Globo.com',
+ 'uploader_id': 265,
+ 'like_count': int,
+ }
+ }, {
+ 'url': 'http://globotv.globo.com/canal-brasil/sangue-latino/t/todos-os-videos/v/ator-e-diretor-argentino-ricado-darin-fala-sobre-utopias-e-suas-perdas/3928201/',
+ 'md5': 'c1defca721ce25b2354e927d3e4b3dec',
+ 'info_dict': {
+ 'id': '3928201',
+ 'ext': 'mp4',
+ 'title': 'Ator e diretor argentino, Ricado Darín fala sobre utopias e suas perdas',
+ 'duration': 1472.906,
+ 'uploader': 'Canal Brasil',
+ 'uploader_id': 705,
+ 'like_count': int,
+ }
+ }]
- class MD5():
+ class MD5:
HEX_FORMAT_LOWERCASE = 0
HEX_FORMAT_UPPERCASE = 1
BASE64_PAD_CHARACTER_DEFAULT_COMPLIANCE = ''
@@ -353,9 +330,6 @@ class GloboIE(InfoExtractor):
def _real_extract(self, url):
video_id = self._match_id(url)
- webpage = self._download_webpage(url, video_id)
- video_id = self._search_regex(self._VIDEOID_REGEXES, webpage, 'video id')
-
video = self._download_json(
self._API_URL_TEMPLATE % video_id, video_id)['videos'][0]
@@ -417,3 +391,39 @@ class GloboIE(InfoExtractor):
'like_count': like_count,
'formats': formats
}
+
+
+class GloboArticleIE(InfoExtractor):
+ _VALID_URL = 'https?://.+?\.globo\.com/(?:[^/]+/)*(?P<id>[^/]+)\.html'
+
+ _VIDEOID_REGEXES = [
+ r'\bdata-video-id=["\'](\d{7,})',
+ r'\bdata-player-videosids=["\'](\d{7,})',
+ r'\bvideosIDs\s*:\s*["\'](\d{7,})',
+ r'\bdata-id=["\'](\d{7,})',
+ r'<div[^>]+\bid=["\'](\d{7,})',
+ ]
+
+ _TEST = {
+ 'url': 'http://g1.globo.com/jornal-nacional/noticia/2014/09/novidade-na-fiscalizacao-de-bagagem-pela-receita-provoca-discussoes.html',
+ 'md5': '307fdeae4390ccfe6ba1aa198cf6e72b',
+ 'info_dict': {
+ 'id': '3652183',
+ 'ext': 'mp4',
+ 'title': 'Receita Federal explica como vai fiscalizar bagagens de quem retorna ao Brasil de avião',
+ 'duration': 110.711,
+ 'uploader': 'Rede Globo',
+ 'uploader_id': 196,
+ 'like_count': int,
+ }
+ }
+
+ @classmethod
+ def suitable(cls, url):
+ return False if GloboIE.suitable(url) else super(GloboArticleIE, cls).suitable(url)
+
+ def _real_extract(self, url):
+ display_id = self._match_id(url)
+ webpage = self._download_webpage(url, display_id)
+ video_id = self._search_regex(self._VIDEOID_REGEXES, webpage, 'video id')
+ return self.url_result('globo:%s' % video_id, 'Globo')
From e3778cce0e912f803ea10cb806406f7fcafe840f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 4 Nov 2015 16:51:19 +0600
Subject: [PATCH 159/415] [globo] Improve m3u8 extraction
---
youtube_dl/extractor/globo.py | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/globo.py b/youtube_dl/extractor/globo.py
index 828e40d76..c28899011 100644
--- a/youtube_dl/extractor/globo.py
+++ b/youtube_dl/extractor/globo.py
@@ -367,7 +367,10 @@ class GloboIE(InfoExtractor):
resource_url = resource['url']
signed_url = '%s?h=%s&k=%s' % (resource_url, signed_hash, 'flash')
if resource_id.endswith('m3u8') or resource_url.endswith('.m3u8'):
- formats.extend(self._extract_m3u8_formats(signed_url, resource_id, 'mp4'))
+ m3u8_formats = self._extract_m3u8_formats(
+ signed_url, resource_id, 'mp4', m3u8_id='hls', fatal=False)
+ if m3u8_formats:
+ formats.extend(m3u8_formats)
else:
formats.append({
'url': signed_url,
From c3459d24f16056e8ae8f982db2a10871ef18e80a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 4 Nov 2015 16:53:21 +0600
Subject: [PATCH 160/415] [globo] Skip unsupported smooth streaming
---
youtube_dl/extractor/globo.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/globo.py b/youtube_dl/extractor/globo.py
index c28899011..ec451bb07 100644
--- a/youtube_dl/extractor/globo.py
+++ b/youtube_dl/extractor/globo.py
@@ -338,7 +338,7 @@ class GloboIE(InfoExtractor):
formats = []
for resource in video['resources']:
resource_id = resource.get('_id')
- if not resource_id:
+ if not resource_id or resource_id.endswith('manifest'):
continue
security = self._download_json(
From 5d235ca7f66af1f82c1a4d753d238f48fc3afa40 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 4 Nov 2015 16:55:39 +0600
Subject: [PATCH 161/415] [globo] Prefer native m3u8
---
youtube_dl/extractor/globo.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/globo.py b/youtube_dl/extractor/globo.py
index ec451bb07..2a805cbb2 100644
--- a/youtube_dl/extractor/globo.py
+++ b/youtube_dl/extractor/globo.py
@@ -368,7 +368,8 @@ class GloboIE(InfoExtractor):
signed_url = '%s?h=%s&k=%s' % (resource_url, signed_hash, 'flash')
if resource_id.endswith('m3u8') or resource_url.endswith('.m3u8'):
m3u8_formats = self._extract_m3u8_formats(
- signed_url, resource_id, 'mp4', m3u8_id='hls', fatal=False)
+ signed_url, resource_id, 'mp4', entry_protocol='m3u8_native',
+ m3u8_id='hls', fatal=False)
if m3u8_formats:
formats.extend(m3u8_formats)
else:
From b4ef6a0038657c1adde565df947e42ad1e1b4195 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 4 Nov 2015 17:01:27 +0600
Subject: [PATCH 162/415] [globo] Remove non available test
---
youtube_dl/extractor/globo.py | 12 ------------
1 file changed, 12 deletions(-)
diff --git a/youtube_dl/extractor/globo.py b/youtube_dl/extractor/globo.py
index 2a805cbb2..8aada01dc 100644
--- a/youtube_dl/extractor/globo.py
+++ b/youtube_dl/extractor/globo.py
@@ -26,18 +26,6 @@ class GloboIE(InfoExtractor):
_RESIGN_EXPIRATION = 86400
_TESTS = [{
- 'url': 'http://globotv.globo.com/sportv/futebol-nacional/v/os-gols-de-atletico-mg-3-x-2-santos-pela-24a-rodada-do-brasileirao/3654973/',
- 'md5': '03ebf41cb7ade43581608b7d9b71fab0',
- 'info_dict': {
- 'id': '3654973',
- 'ext': 'mp4',
- 'title': 'Os gols de Atlético-MG 3 x 2 Santos pela 24ª rodada do Brasileirão',
- 'duration': 251.585,
- 'uploader': 'SporTV',
- 'uploader_id': 698,
- 'like_count': int,
- }
- }, {
'url': 'http://g1.globo.com/carros/autoesporte/videos/t/exclusivos-do-g1/v/mercedes-benz-gla-passa-por-teste-de-colisao-na-europa/3607726/',
'md5': 'b3ccc801f75cd04a914d51dadb83a78d',
'info_dict': {
From aebb42d32b608eaffb424e5e7c22f1b68a491e3d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 4 Nov 2015 17:01:55 +0600
Subject: [PATCH 163/415] [globo] Remove like count
It's no longer provided
---
youtube_dl/extractor/globo.py | 5 -----
1 file changed, 5 deletions(-)
diff --git a/youtube_dl/extractor/globo.py b/youtube_dl/extractor/globo.py
index 8aada01dc..dc89e46ac 100644
--- a/youtube_dl/extractor/globo.py
+++ b/youtube_dl/extractor/globo.py
@@ -35,7 +35,6 @@ class GloboIE(InfoExtractor):
'duration': 103.204,
'uploader': 'Globo.com',
'uploader_id': 265,
- 'like_count': int,
}
}, {
'url': 'http://globotv.globo.com/canal-brasil/sangue-latino/t/todos-os-videos/v/ator-e-diretor-argentino-ricado-darin-fala-sobre-utopias-e-suas-perdas/3928201/',
@@ -47,7 +46,6 @@ class GloboIE(InfoExtractor):
'duration': 1472.906,
'uploader': 'Canal Brasil',
'uploader_id': 705,
- 'like_count': int,
}
}]
@@ -370,7 +368,6 @@ class GloboIE(InfoExtractor):
self._sort_formats(formats)
duration = float_or_none(video.get('duration'), 1000)
- like_count = int_or_none(video.get('likes'))
uploader = video.get('channel')
uploader_id = video.get('channel_id')
@@ -380,7 +377,6 @@ class GloboIE(InfoExtractor):
'duration': duration,
'uploader': uploader,
'uploader_id': uploader_id,
- 'like_count': like_count,
'formats': formats
}
@@ -406,7 +402,6 @@ class GloboArticleIE(InfoExtractor):
'duration': 110.711,
'uploader': 'Rede Globo',
'uploader_id': 196,
- 'like_count': int,
}
}
From a4a6b7b80f18680ee0a8bba50a24c58edd3f2a73 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 4 Nov 2015 17:03:45 +0600
Subject: [PATCH 164/415] [globo] Improve http formats
---
youtube_dl/extractor/globo.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/youtube_dl/extractor/globo.py b/youtube_dl/extractor/globo.py
index dc89e46ac..64622aa5c 100644
--- a/youtube_dl/extractor/globo.py
+++ b/youtube_dl/extractor/globo.py
@@ -361,8 +361,8 @@ class GloboIE(InfoExtractor):
else:
formats.append({
'url': signed_url,
- 'format_id': resource_id,
- 'height': resource.get('height'),
+ 'format_id': 'http-%s' % resource_id,
+ 'height': int_or_none(resource.get('height')),
})
self._sort_formats(formats)
From 264cd00fff4f6d7063d43e1d476de46901bd9c5b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 4 Nov 2015 17:10:45 +0600
Subject: [PATCH 165/415] [globo] Update tests
---
youtube_dl/extractor/globo.py | 32 ++++++++++++++++++++++----------
1 file changed, 22 insertions(+), 10 deletions(-)
diff --git a/youtube_dl/extractor/globo.py b/youtube_dl/extractor/globo.py
index 64622aa5c..0337256ed 100644
--- a/youtube_dl/extractor/globo.py
+++ b/youtube_dl/extractor/globo.py
@@ -35,18 +35,30 @@ class GloboIE(InfoExtractor):
'duration': 103.204,
'uploader': 'Globo.com',
'uploader_id': 265,
- }
+ },
+ }, {
+ 'url': 'http://globoplay.globo.com/v/4581987/',
+ 'md5': 'f36a1ecd6a50da1577eee6dd17f67eff',
+ 'info_dict': {
+ 'id': '4581987',
+ 'ext': 'mp4',
+ 'title': 'Acidentes de trânsito estão entre as maiores causas de queda de energia em SP',
+ 'duration': 137.973,
+ 'uploader': 'Rede Globo',
+ 'uploader_id': 196,
+ },
+ }, {
+ 'url': 'http://canalbrasil.globo.com/programas/sangue-latino/videos/3928201.html',
+ 'only_matching': True,
+ }, {
+ 'url': 'http://globosatplay.globo.com/globonews/v/4472924/',
+ 'only_matching': True,
+ }, {
+ 'url': 'http://globotv.globo.com/t/programa/v/clipe-sexo-e-as-negas-adeus/3836166/',
+ 'only_matching': True,
}, {
'url': 'http://globotv.globo.com/canal-brasil/sangue-latino/t/todos-os-videos/v/ator-e-diretor-argentino-ricado-darin-fala-sobre-utopias-e-suas-perdas/3928201/',
- 'md5': 'c1defca721ce25b2354e927d3e4b3dec',
- 'info_dict': {
- 'id': '3928201',
- 'ext': 'mp4',
- 'title': 'Ator e diretor argentino, Ricado Darín fala sobre utopias e suas perdas',
- 'duration': 1472.906,
- 'uploader': 'Canal Brasil',
- 'uploader_id': 705,
- }
+ 'only_matching': True,
}]
class MD5:
From e7d34c03f200e178e9d6dfe4ae3f6856e382a4b9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 4 Nov 2015 17:12:42 +0600
Subject: [PATCH 166/415] [globo] Force uploader id to be string
---
youtube_dl/extractor/globo.py | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/youtube_dl/extractor/globo.py b/youtube_dl/extractor/globo.py
index 0337256ed..6c0fc54de 100644
--- a/youtube_dl/extractor/globo.py
+++ b/youtube_dl/extractor/globo.py
@@ -14,6 +14,7 @@ from ..utils import (
ExtractorError,
float_or_none,
int_or_none,
+ str_or_none,
)
@@ -34,7 +35,7 @@ class GloboIE(InfoExtractor):
'title': 'Mercedes-Benz GLA passa por teste de colisão na Europa',
'duration': 103.204,
'uploader': 'Globo.com',
- 'uploader_id': 265,
+ 'uploader_id': '265',
},
}, {
'url': 'http://globoplay.globo.com/v/4581987/',
@@ -45,7 +46,7 @@ class GloboIE(InfoExtractor):
'title': 'Acidentes de trânsito estão entre as maiores causas de queda de energia em SP',
'duration': 137.973,
'uploader': 'Rede Globo',
- 'uploader_id': 196,
+ 'uploader_id': '196',
},
}, {
'url': 'http://canalbrasil.globo.com/programas/sangue-latino/videos/3928201.html',
@@ -381,7 +382,7 @@ class GloboIE(InfoExtractor):
duration = float_or_none(video.get('duration'), 1000)
uploader = video.get('channel')
- uploader_id = video.get('channel_id')
+ uploader_id = str_or_none(video.get('channel_id'))
return {
'id': video_id,
From c13722480bebfb1fc33169516790df2e99b3e499 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 4 Nov 2015 17:13:35 +0600
Subject: [PATCH 167/415] [globo:article] Fix test
---
youtube_dl/extractor/globo.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/globo.py b/youtube_dl/extractor/globo.py
index 6c0fc54de..5883be704 100644
--- a/youtube_dl/extractor/globo.py
+++ b/youtube_dl/extractor/globo.py
@@ -414,7 +414,7 @@ class GloboArticleIE(InfoExtractor):
'title': 'Receita Federal explica como vai fiscalizar bagagens de quem retorna ao Brasil de avião',
'duration': 110.711,
'uploader': 'Rede Globo',
- 'uploader_id': 196,
+ 'uploader_id': '196',
}
}
From 5d501a0901c36695c9d6ca3958ac4ccfdea90954 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 4 Nov 2015 17:42:11 +0600
Subject: [PATCH 168/415] [globo] Add more tests
---
youtube_dl/extractor/globo.py | 13 +++++++++++--
1 file changed, 11 insertions(+), 2 deletions(-)
diff --git a/youtube_dl/extractor/globo.py b/youtube_dl/extractor/globo.py
index 5883be704..c65ef6bcf 100644
--- a/youtube_dl/extractor/globo.py
+++ b/youtube_dl/extractor/globo.py
@@ -60,6 +60,9 @@ class GloboIE(InfoExtractor):
}, {
'url': 'http://globotv.globo.com/canal-brasil/sangue-latino/t/todos-os-videos/v/ator-e-diretor-argentino-ricado-darin-fala-sobre-utopias-e-suas-perdas/3928201/',
'only_matching': True,
+ }, {
+ 'url': 'http://canaloff.globo.com/programas/desejar-profundo/videos/4518560.html',
+ 'only_matching': True,
}]
class MD5:
@@ -405,7 +408,7 @@ class GloboArticleIE(InfoExtractor):
r'<div[^>]+\bid=["\'](\d{7,})',
]
- _TEST = {
+ _TESTS = [{
'url': 'http://g1.globo.com/jornal-nacional/noticia/2014/09/novidade-na-fiscalizacao-de-bagagem-pela-receita-provoca-discussoes.html',
'md5': '307fdeae4390ccfe6ba1aa198cf6e72b',
'info_dict': {
@@ -416,7 +419,13 @@ class GloboArticleIE(InfoExtractor):
'uploader': 'Rede Globo',
'uploader_id': '196',
}
- }
+ }, {
+ 'url': 'http://gq.globo.com/Prazeres/Poder/noticia/2015/10/all-o-desafio-assista-ao-segundo-capitulo-da-serie.html',
+ 'only_matching': True,
+ }, {
+ 'url': 'http://gshow.globo.com/programas/tv-xuxa/O-Programa/noticia/2014/01/xuxa-e-junno-namoram-muuuito-em-luau-de-zeze-di-camargo-e-luciano.html',
+ 'only_matching': True,
+ }]
@classmethod
def suitable(cls, url):
From 17d1900581ffd12866e56640080ce340d99149a2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 4 Nov 2015 17:57:46 +0600
Subject: [PATCH 169/415] [vk] Fix view count extraction (Closes #7353)
---
youtube_dl/extractor/vk.py | 10 +++++++---
1 file changed, 7 insertions(+), 3 deletions(-)
diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 765e9e6fd..01960b827 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -281,9 +281,13 @@ class VKIE(InfoExtractor):
mobj.group(1) + ' ' + mobj.group(2)
upload_date = unified_strdate(mobj.group(1) + ' ' + mobj.group(2))
- view_count = str_to_int(self._search_regex(
- r'"mv_views_count_number"[^>]*>([\d,.]+) views<',
- info_page, 'view count', fatal=False))
+ view_count = None
+ views = self._html_search_regex(
+ r'"mv_views_count_number"[^>]*>(.+?\bviews?)<',
+ info_page, 'view count', fatal=False)
+ if views:
+ view_count = str_to_int(self._search_regex(
+ r'([\d,.]+)', views, 'view count', fatal=False))
formats = [{
'format_id': k,
From cb5a470635ea2ad91f18d33e391979aabb0755fb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
<jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 4 Nov 2015 16:18:51 +0100
Subject: [PATCH 170/415] [vimeo] Remove unused import
---
youtube_dl/extractor/vimeo.py | 1 -
1 file changed, 1 deletion(-)
diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index b608740b8..ca716c8f5 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -8,7 +8,6 @@ import itertools
from .common import InfoExtractor
from ..compat import (
compat_HTTPError,
- compat_urllib_parse,
compat_urllib_request,
compat_urlparse,
)
From 44b2264feae331eeb34e83eed1387def3d61a437 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 4 Nov 2015 22:12:24 +0600
Subject: [PATCH 171/415] [youtube] Prefer video_info with token available
---
youtube_dl/extractor/youtube.py | 2 ++
1 file changed, 2 insertions(+)
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index d7eda7aa7..5eeb3c663 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1107,6 +1107,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
if not video_info:
video_info = get_video_info
if 'token' in get_video_info:
+ if 'token' not in video_info:
+ video_info = get_video_info
break
if 'token' not in video_info:
if 'reason' in video_info:
From 89ea063eebae84792a7ccb968533ff8bf6a41d56 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 4 Nov 2015 22:49:23 +0600
Subject: [PATCH 172/415] [youtube] Clarify rationale for preferring a video
info with token (#7362)
---
youtube_dl/extractor/youtube.py | 9 +++++++++
1 file changed, 9 insertions(+)
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 5eeb3c663..e2a43299f 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1107,6 +1107,15 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
if not video_info:
video_info = get_video_info
if 'token' in get_video_info:
+ # Different get_video_info requests may report different results, e.g.
+ # some may report video unavailability, but some may serve it without
+ # any complaint (see https://github.com/rg3/youtube-dl/issues/7362,
+ # the original webpage as well as el=info and el=embedded get_video_info
+ # requests report video unavailability due to geo restriction while
+ # el=detailpage succeeds and returns valid data). This is probably
+ # due to YouTube measures against IP ranges of hosting providers.
+ # Working around by preferring the first succeeded video_info containing
+ # the token if no such video_info yet was found.
if 'token' not in video_info:
video_info = get_video_info
break
From f93ded98522cc1272a8d2210738937132292afc9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 5 Nov 2015 01:54:49 +0600
Subject: [PATCH 173/415] [prosiebensat1] Add support for .ch domains (Closes
#7365)
---
youtube_dl/extractor/prosiebensat1.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/prosiebensat1.py b/youtube_dl/extractor/prosiebensat1.py
index effcf1db3..baa54a3af 100644
--- a/youtube_dl/extractor/prosiebensat1.py
+++ b/youtube_dl/extractor/prosiebensat1.py
@@ -20,7 +20,7 @@ from ..utils import (
class ProSiebenSat1IE(InfoExtractor):
IE_NAME = 'prosiebensat1'
IE_DESC = 'ProSiebenSat.1 Digital'
- _VALID_URL = r'https?://(?:www\.)?(?:(?:prosieben|prosiebenmaxx|sixx|sat1|kabeleins|the-voice-of-germany)\.(?:de|at)|ran\.de|fem\.com)/(?P<id>.+)'
+ _VALID_URL = r'https?://(?:www\.)?(?:(?:prosieben|prosiebenmaxx|sixx|sat1|kabeleins|the-voice-of-germany)\.(?:de|at|ch)|ran\.de|fem\.com)/(?P<id>.+)'
_TESTS = [
{
From b15c44cd36831f175e9dd4081b82beb8075790b3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 5 Nov 2015 02:51:30 +0600
Subject: [PATCH 174/415] [periscope] Add support for videos with broadcast_id
(Closes #7359)
---
youtube_dl/extractor/periscope.py | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/youtube_dl/extractor/periscope.py b/youtube_dl/extractor/periscope.py
index 8ad936758..0f9d7576f 100644
--- a/youtube_dl/extractor/periscope.py
+++ b/youtube_dl/extractor/periscope.py
@@ -27,9 +27,10 @@ class PeriscopeIE(InfoExtractor):
'skip': 'Expires in 24 hours',
}
- def _call_api(self, method, token):
+ def _call_api(self, method, value):
+ attribute = 'token' if len(value) > 13 else 'broadcast_id'
return self._download_json(
- 'https://api.periscope.tv/api/v2/%s?token=%s' % (method, token), token)
+ 'https://api.periscope.tv/api/v2/%s?%s=%s' % (method, attribute, value), value)
def _real_extract(self, url):
token = self._match_id(url)
From 2549e113b8750a493917436d4fd15ed74a1a4983 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 5 Nov 2015 02:55:53 +0600
Subject: [PATCH 175/415] [periscope] Add test for broadcast_id based URL
---
youtube_dl/extractor/periscope.py | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git a/youtube_dl/extractor/periscope.py b/youtube_dl/extractor/periscope.py
index 0f9d7576f..7621d9e99 100644
--- a/youtube_dl/extractor/periscope.py
+++ b/youtube_dl/extractor/periscope.py
@@ -12,7 +12,7 @@ from ..utils import parse_iso8601
class PeriscopeIE(InfoExtractor):
IE_DESC = 'Periscope'
_VALID_URL = r'https?://(?:www\.)?periscope\.tv/w/(?P<id>[^/?#]+)'
- _TEST = {
+ _TESTS = [{
'url': 'https://www.periscope.tv/w/aJUQnjY3MjA3ODF8NTYxMDIyMDl2zCg2pECBgwTqRpQuQD352EMPTKQjT4uqlM3cgWFA-g==',
'md5': '65b57957972e503fcbbaeed8f4fa04ca',
'info_dict': {
@@ -25,7 +25,10 @@ class PeriscopeIE(InfoExtractor):
'uploader_id': '1465763',
},
'skip': 'Expires in 24 hours',
- }
+ }, {
+ 'url': 'https://www.periscope.tv/w/1ZkKzPbMVggJv',
+ 'only_matching': True,
+ }]
def _call_api(self, method, value):
attribute = 'token' if len(value) > 13 else 'broadcast_id'
From 53472df85793cc89deb779c2ffc3ae1f47292fd4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 5 Nov 2015 02:56:44 +0600
Subject: [PATCH 176/415] [periscope] Add note on where to find alive example
URLs
---
youtube_dl/extractor/periscope.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/youtube_dl/extractor/periscope.py b/youtube_dl/extractor/periscope.py
index 7621d9e99..887c8020d 100644
--- a/youtube_dl/extractor/periscope.py
+++ b/youtube_dl/extractor/periscope.py
@@ -12,6 +12,7 @@ from ..utils import parse_iso8601
class PeriscopeIE(InfoExtractor):
IE_DESC = 'Periscope'
_VALID_URL = r'https?://(?:www\.)?periscope\.tv/w/(?P<id>[^/?#]+)'
+ # Alive example URLs can be found here http://onperiscope.com/
_TESTS = [{
'url': 'https://www.periscope.tv/w/aJUQnjY3MjA3ODF8NTYxMDIyMDl2zCg2pECBgwTqRpQuQD352EMPTKQjT4uqlM3cgWFA-g==',
'md5': '65b57957972e503fcbbaeed8f4fa04ca',
From b3613d36da14ab527166326707c0f911d192144d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 5 Nov 2015 04:37:51 +0600
Subject: [PATCH 177/415] [YoutubeDL] Sanitize path after output template
substitution (Closes #7367)
---
youtube_dl/YoutubeDL.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 12977bf80..1783ce01b 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -572,7 +572,7 @@ class YoutubeDL(object):
if v is not None)
template_dict = collections.defaultdict(lambda: 'NA', template_dict)
- outtmpl = sanitize_path(self.params.get('outtmpl', DEFAULT_OUTTMPL))
+ outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
tmpl = compat_expanduser(outtmpl)
filename = tmpl % template_dict
# Temporary fix for #4787
@@ -580,7 +580,7 @@ class YoutubeDL(object):
# to workaround encoding issues with subprocess on python2 @ Windows
if sys.version_info < (3, 0) and sys.platform == 'win32':
filename = encodeFilename(filename, True).decode(preferredencoding())
- return filename
+ return sanitize_path(filename)
except ValueError as err:
self.report_error('Error in output template: ' + str(err) + ' (encoding: ' + repr(preferredencoding()) + ')')
return None
From 6953d8e95a78e83f087693b7353baab96b09fbdd Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 6 Nov 2015 02:09:55 +0100
Subject: [PATCH 178/415] [miomio] fix info extraction (fixes #7366)
---
youtube_dl/extractor/miomio.py | 9 ++++++---
1 file changed, 6 insertions(+), 3 deletions(-)
diff --git a/youtube_dl/extractor/miomio.py b/youtube_dl/extractor/miomio.py
index a784fc5fb..3f812e005 100644
--- a/youtube_dl/extractor/miomio.py
+++ b/youtube_dl/extractor/miomio.py
@@ -4,6 +4,7 @@ from __future__ import unicode_literals
import random
from .common import InfoExtractor
+from ..compat import compat_urllib_request
from ..utils import (
xpath_text,
int_or_none,
@@ -60,10 +61,12 @@ class MioMioIE(InfoExtractor):
'http://www.miomio.tv/mioplayer/mioplayerconfigfiles/xml.php?id=%s&r=%s' % (id, random.randint(100, 999)),
video_id)
- # the following xml contains the actual configuration information on the video file(s)
- vid_config = self._download_xml(
+ vid_config_request = compat_urllib_request.Request(
'http://www.miomio.tv/mioplayer/mioplayerconfigfiles/sina.php?{0}'.format(xml_config),
- video_id)
+ headers={'Referer': 'http://www.miomio.tv/mioplayer/mioplayer-v3.0.swf'})
+
+ # the following xml contains the actual configuration information on the video file(s)
+ vid_config = self._download_xml(vid_config_request, video_id)
http_headers = {
'Referer': 'http://www.miomio.tv%s' % mioplayer_path,
From e68dd1921ad7528d225a8571066f99b9934b6a06 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 6 Nov 2015 06:33:05 +0100
Subject: [PATCH 179/415] [miomio] use the formats urls headers for downloading
xml
---
youtube_dl/extractor/miomio.py | 8 +++-----
1 file changed, 3 insertions(+), 5 deletions(-)
diff --git a/youtube_dl/extractor/miomio.py b/youtube_dl/extractor/miomio.py
index 3f812e005..6f40bf1b9 100644
--- a/youtube_dl/extractor/miomio.py
+++ b/youtube_dl/extractor/miomio.py
@@ -52,6 +52,8 @@ class MioMioIE(InfoExtractor):
mioplayer_path = self._search_regex(
r'src="(/mioplayer/[^"]+)"', webpage, 'ref_path')
+ http_headers = {'Referer': 'http://www.miomio.tv%s' % mioplayer_path,}
+
xml_config = self._search_regex(
r'flashvars="type=(?:sina|video)&(.+?)&',
webpage, 'xml config')
@@ -63,15 +65,11 @@ class MioMioIE(InfoExtractor):
vid_config_request = compat_urllib_request.Request(
'http://www.miomio.tv/mioplayer/mioplayerconfigfiles/sina.php?{0}'.format(xml_config),
- headers={'Referer': 'http://www.miomio.tv/mioplayer/mioplayer-v3.0.swf'})
+ headers=http_headers)
# the following xml contains the actual configuration information on the video file(s)
vid_config = self._download_xml(vid_config_request, video_id)
- http_headers = {
- 'Referer': 'http://www.miomio.tv%s' % mioplayer_path,
- }
-
if not int_or_none(xpath_text(vid_config, 'timelength')):
raise ExtractorError('Unable to load videos!', expected=True)
From 5003e4283b35acb82ea9793d91bc3cd1ee679f86 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 6 Nov 2015 21:06:44 +0600
Subject: [PATCH 180/415] [ndr] Relax _VALID_URL (Closes #7383)
---
youtube_dl/extractor/ndr.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/ndr.py b/youtube_dl/extractor/ndr.py
index ba06d8a98..a2b51ccb3 100644
--- a/youtube_dl/extractor/ndr.py
+++ b/youtube_dl/extractor/ndr.py
@@ -23,7 +23,7 @@ class NDRBaseIE(InfoExtractor):
class NDRIE(NDRBaseIE):
IE_NAME = 'ndr'
IE_DESC = 'NDR.de - Norddeutscher Rundfunk'
- _VALID_URL = r'https?://www\.ndr\.de/(?:[^/]+/)+(?P<id>[^/?#]+),[\da-z]+\.html'
+ _VALID_URL = r'https?://www\.ndr\.de/(?:[^/]+/)*(?P<id>[^/?#]+),[\da-z]+\.html'
_TESTS = [{
# httpVideo, same content id
'url': 'http://www.ndr.de/fernsehen/Party-Poette-und-Parade,hafengeburtstag988.html',
From 01003d072c20c2ed095930d87c5ce3d5610e66b1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 6 Nov 2015 21:07:52 +0600
Subject: [PATCH 181/415] [ndr] Add test for #7383
---
youtube_dl/extractor/ndr.py | 3 +++
1 file changed, 3 insertions(+)
diff --git a/youtube_dl/extractor/ndr.py b/youtube_dl/extractor/ndr.py
index a2b51ccb3..0be866681 100644
--- a/youtube_dl/extractor/ndr.py
+++ b/youtube_dl/extractor/ndr.py
@@ -78,6 +78,9 @@ class NDRIE(NDRBaseIE):
'params': {
'skip_download': True,
},
+ }, {
+ 'url': 'https://www.ndr.de/Fettes-Brot-Ferris-MC-und-Thees-Uhlmann-live-on-stage,festivalsommer116.html',
+ 'only_matching': True,
}]
def _extract_embed(self, webpage, display_id):
From 1e2eb4b40d46f39d15c067657ecad16fa3b2121d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 6 Nov 2015 21:08:21 +0600
Subject: [PATCH 182/415] [njoy] Relax _VALID_URL
---
youtube_dl/extractor/ndr.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/ndr.py b/youtube_dl/extractor/ndr.py
index 0be866681..7043c7e0f 100644
--- a/youtube_dl/extractor/ndr.py
+++ b/youtube_dl/extractor/ndr.py
@@ -105,7 +105,7 @@ class NDRIE(NDRBaseIE):
class NJoyIE(NDRBaseIE):
IE_NAME = 'njoy'
IE_DESC = 'N-JOY'
- _VALID_URL = r'https?://www\.n-joy\.de/(?:[^/]+/)+(?:(?P<display_id>[^/?#]+),)?(?P<id>[\da-z]+)\.html'
+ _VALID_URL = r'https?://www\.n-joy\.de/(?:[^/]+/)*(?:(?P<display_id>[^/?#]+),)?(?P<id>[\da-z]+)\.html'
_TESTS = [{
# httpVideo, same content id
'url': 'http://www.n-joy.de/entertainment/comedy/comedy_contest/Benaissa-beim-NDR-Comedy-Contest,comedycontest2480.html',
From 81413c01651eddcc5180af379f2ce3689a376051 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 6 Nov 2015 21:08:52 +0600
Subject: [PATCH 183/415] [ndr:embed] Relax _VALID_URL
---
youtube_dl/extractor/ndr.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/ndr.py b/youtube_dl/extractor/ndr.py
index 7043c7e0f..477ce4e6b 100644
--- a/youtube_dl/extractor/ndr.py
+++ b/youtube_dl/extractor/ndr.py
@@ -238,7 +238,7 @@ class NDREmbedBaseIE(InfoExtractor):
class NDREmbedIE(NDREmbedBaseIE):
IE_NAME = 'ndr:embed'
- _VALID_URL = r'https?://www\.ndr\.de/(?:[^/]+/)+(?P<id>[\da-z]+)-(?:player|externalPlayer)\.html'
+ _VALID_URL = r'https?://www\.ndr\.de/(?:[^/]+/)*(?P<id>[\da-z]+)-(?:player|externalPlayer)\.html'
_TESTS = [{
'url': 'http://www.ndr.de/fernsehen/sendungen/ndr_aktuell/ndraktuell28488-player.html',
'md5': '8b9306142fe65bbdefb5ce24edb6b0a9',
From 92366d189ef280b8ba0057930c54aa14b0ecdd24 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 6 Nov 2015 21:09:17 +0600
Subject: [PATCH 184/415] [njoy:embed] Relax _VALID_URL
---
youtube_dl/extractor/ndr.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/ndr.py b/youtube_dl/extractor/ndr.py
index 477ce4e6b..16213eed9 100644
--- a/youtube_dl/extractor/ndr.py
+++ b/youtube_dl/extractor/ndr.py
@@ -332,7 +332,7 @@ class NDREmbedIE(NDREmbedBaseIE):
class NJoyEmbedIE(NDREmbedBaseIE):
IE_NAME = 'njoy:embed'
- _VALID_URL = r'https?://www\.n-joy\.de/(?:[^/]+/)+(?P<id>[\da-z]+)-(?:player|externalPlayer)_[^/]+\.html'
+ _VALID_URL = r'https?://www\.n-joy\.de/(?:[^/]+/)*(?P<id>[\da-z]+)-(?:player|externalPlayer)_[^/]+\.html'
_TESTS = [{
# httpVideo
'url': 'http://www.n-joy.de/events/reeperbahnfestival/doku948-player_image-bc168e87-5263-4d6d-bd27-bb643005a6de_theme-n-joy.html',
From deb85c32bbd32e8d280e1919432a11c0bdaa26bb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 6 Nov 2015 21:56:31 +0600
Subject: [PATCH 185/415] [postprocessor/ffmpeg] Use ffmpeg as prefix since
it's used all over the places (Closes #7371)
---
youtube_dl/postprocessor/ffmpeg.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 4f320e124..5ed723bc6 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -272,7 +272,7 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor):
return [], information
try:
- self._downloader.to_screen('[' + self.basename + '] Destination: ' + new_path)
+ self._downloader.to_screen('[ffmpeg] Destination: ' + new_path)
self.run_ffmpeg(path, new_path, acodec, more_opts)
except AudioConversionError as e:
raise PostProcessingError(
From 179ffab69c3359ab7d0a7b0a2b63c94d8c70af67 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 6 Nov 2015 23:06:13 +0600
Subject: [PATCH 186/415] [lynda:course] Force log out (Closes #7361)
---
youtube_dl/extractor/lynda.py | 7 +++++++
1 file changed, 7 insertions(+)
diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py
index 5c973e75c..67f2025de 100644
--- a/youtube_dl/extractor/lynda.py
+++ b/youtube_dl/extractor/lynda.py
@@ -82,6 +82,11 @@ class LyndaBaseIE(InfoExtractor):
expected=True)
raise ExtractorError('Unable to log in')
+ def _logout(self):
+ self._download_webpage(
+ 'http://www.lynda.com/ajax/logout.aspx', None,
+ 'Logging out', 'Unable to log out', fatal=False)
+
class LyndaIE(LyndaBaseIE):
IE_NAME = 'lynda'
@@ -210,6 +215,8 @@ class LyndaCourseIE(LyndaBaseIE):
course_id, 'Downloading course JSON')
course_json = json.loads(page)
+ self._logout()
+
if 'Status' in course_json and course_json['Status'] == 'NotFound':
raise ExtractorError(
'Course %s does not exist' % course_id, expected=True)
From 71bb016160744a80fecaadf5b75b0dc2b1e8089b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 6 Nov 2015 23:10:07 +0600
Subject: [PATCH 187/415] [lynda:course] Modernize and make more robust
---
youtube_dl/extractor/lynda.py | 17 +++++++++--------
1 file changed, 9 insertions(+), 8 deletions(-)
diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py
index 67f2025de..98474ded9 100644
--- a/youtube_dl/extractor/lynda.py
+++ b/youtube_dl/extractor/lynda.py
@@ -210,14 +210,13 @@ class LyndaCourseIE(LyndaBaseIE):
course_path = mobj.group('coursepath')
course_id = mobj.group('courseid')
- page = self._download_webpage(
+ course = self._download_json(
'http://www.lynda.com/ajax/player?courseId=%s&type=course' % course_id,
course_id, 'Downloading course JSON')
- course_json = json.loads(page)
self._logout()
- if 'Status' in course_json and course_json['Status'] == 'NotFound':
+ if course.get('Status') == 'NotFound':
raise ExtractorError(
'Course %s does not exist' % course_id, expected=True)
@@ -227,12 +226,14 @@ class LyndaCourseIE(LyndaBaseIE):
# Might want to extract videos right here from video['Formats'] as it seems 'Formats' is not provided
# by single video API anymore
- for chapter in course_json['Chapters']:
- for video in chapter['Videos']:
- if video['HasAccess'] is False:
+ for chapter in course['Chapters']:
+ for video in chapter.get('Videos', []):
+ if video.get('HasAccess') is False:
unaccessible_videos += 1
continue
- videos.append(video['ID'])
+ video_id = video.get('ID')
+ if video_id:
+ videos.append(video_id)
if unaccessible_videos > 0:
self._downloader.report_warning(
@@ -245,6 +246,6 @@ class LyndaCourseIE(LyndaBaseIE):
'Lynda')
for video_id in videos]
- course_title = course_json['Title']
+ course_title = course.get('Title')
return self.playlist_result(entries, course_id, course_title)
From ea8ed40b2fb70fc2f01aba475128821078873d46 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 6 Nov 2015 23:24:39 +0600
Subject: [PATCH 188/415] [lynda] Modernize and make more robust
---
youtube_dl/extractor/lynda.py | 52 ++++++++++++++++-------------------
1 file changed, 24 insertions(+), 28 deletions(-)
diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py
index 98474ded9..c8a16842e 100644
--- a/youtube_dl/extractor/lynda.py
+++ b/youtube_dl/extractor/lynda.py
@@ -113,51 +113,47 @@ class LyndaIE(LyndaBaseIE):
def _real_extract(self, url):
video_id = self._match_id(url)
- page = self._download_webpage(
+ video = self._download_json(
'http://www.lynda.com/ajax/player?videoId=%s&type=video' % video_id,
video_id, 'Downloading video JSON')
- video_json = json.loads(page)
- if 'Status' in video_json:
+ if 'Status' in video:
raise ExtractorError(
- 'lynda returned error: %s' % video_json['Message'], expected=True)
+ 'lynda returned error: %s' % video['Message'], expected=True)
- if video_json['HasAccess'] is False:
+ if video.get('HasAccess') is False:
self.raise_login_required('Video %s is only available for members' % video_id)
- video_id = compat_str(video_json['ID'])
- duration = video_json['DurationInSeconds']
- title = video_json['Title']
+ video_id = compat_str(video.get('ID') or video_id)
+ duration = int_or_none(video.get('DurationInSeconds'))
+ title = video['Title']
formats = []
- fmts = video_json.get('Formats')
+ fmts = video.get('Formats')
if fmts:
- formats.extend([
- {
- 'url': fmt['Url'],
- 'ext': fmt['Extension'],
- 'width': fmt['Width'],
- 'height': fmt['Height'],
- 'filesize': fmt['FileSize'],
- 'format_id': str(fmt['Resolution'])
- } for fmt in fmts])
+ formats.extend([{
+ 'url': f['Url'],
+ 'ext': f.get('Extension'),
+ 'width': int_or_none(f.get('Width')),
+ 'height': int_or_none(f.get('Height')),
+ 'filesize': int_or_none(f.get('FileSize')),
+ 'format_id': compat_str(f.get('Resolution')) if f.get('Resolution') else None,
+ } for f in fmts if f.get('Url')])
- prioritized_streams = video_json.get('PrioritizedStreams')
+ prioritized_streams = video.get('PrioritizedStreams')
if prioritized_streams:
for prioritized_stream_id, prioritized_stream in prioritized_streams.items():
- formats.extend([
- {
- 'url': video_url,
- 'width': int_or_none(format_id),
- 'format_id': '%s-%s' % (prioritized_stream_id, format_id),
- } for format_id, video_url in prioritized_stream.items()
- ])
+ formats.extend([{
+ 'url': video_url,
+ 'width': int_or_none(format_id),
+ 'format_id': '%s-%s' % (prioritized_stream_id, format_id),
+ } for format_id, video_url in prioritized_stream.items()])
self._check_formats(formats, video_id)
self._sort_formats(formats)
- subtitles = self.extract_subtitles(video_id, page)
+ subtitles = self.extract_subtitles(video_id)
return {
'id': video_id,
@@ -188,7 +184,7 @@ class LyndaIE(LyndaBaseIE):
if srt:
return srt
- def _get_subtitles(self, video_id, webpage):
+ def _get_subtitles(self, video_id):
url = 'http://www.lynda.com/ajax/player?videoId=%s&type=transcript' % video_id
subs = self._download_json(url, None, False)
if subs:
From ae4ddf9efae816f4d52fc584c93e4f0e3c79c410 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 6 Nov 2015 23:27:38 +0600
Subject: [PATCH 189/415] [lynda] PEP 8
---
youtube_dl/extractor/lynda.py | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py
index c8a16842e..9a207b2cd 100644
--- a/youtube_dl/extractor/lynda.py
+++ b/youtube_dl/extractor/lynda.py
@@ -227,9 +227,8 @@ class LyndaCourseIE(LyndaBaseIE):
if video.get('HasAccess') is False:
unaccessible_videos += 1
continue
- video_id = video.get('ID')
- if video_id:
- videos.append(video_id)
+ if video.get('ID'):
+ videos.append(video['ID'])
if unaccessible_videos > 0:
self._downloader.report_warning(
From 472404953a22811cc8156da110ea872a924f1f18 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 6 Nov 2015 23:28:14 +0600
Subject: [PATCH 190/415] [miomio] PEP 8
---
youtube_dl/extractor/miomio.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/miomio.py b/youtube_dl/extractor/miomio.py
index 6f40bf1b9..ce391c759 100644
--- a/youtube_dl/extractor/miomio.py
+++ b/youtube_dl/extractor/miomio.py
@@ -52,7 +52,7 @@ class MioMioIE(InfoExtractor):
mioplayer_path = self._search_regex(
r'src="(/mioplayer/[^"]+)"', webpage, 'ref_path')
- http_headers = {'Referer': 'http://www.miomio.tv%s' % mioplayer_path,}
+ http_headers = {'Referer': 'http://www.miomio.tv%s' % mioplayer_path}
xml_config = self._search_regex(
r'flashvars="type=(?:sina|video)&(.+?)&',
From 0fa6b17dccd2347cb0611651fc04e36839d33a4e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 6 Nov 2015 23:45:26 +0600
Subject: [PATCH 191/415] [pbs] Simplify and speed up player URL search
---
youtube_dl/extractor/pbs.py | 10 +++++++---
1 file changed, 7 insertions(+), 3 deletions(-)
diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index 3448736a2..7b868d057 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -191,9 +191,13 @@ class PBSIE(InfoExtractor):
if media_id:
return media_id, presumptive_id, upload_date
- url = self._search_regex(
- r'(?s)<iframe[^>]+?(?:[a-z-]+?=["\'].*?["\'][^>]+?)*?\bsrc=["\']([^\'"]+partnerplayer[^\'"]+)["\']',
- webpage, 'player URL')
+ for iframe in re.findall(r'(?s)<iframe(.+?)></iframe>', webpage):
+ url = self._search_regex(
+ r'src=(["\'])(?P<url>.+?partnerplayer.+?)\1', iframe,
+ 'player URL', default=None, group='url')
+ if url:
+ break
+
mobj = re.match(self._VALID_URL, url)
player_id = mobj.group('player_id')
From 686f98816ecbbcb224d1336682688b05cdb051a6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 7 Nov 2015 00:39:16 +0600
Subject: [PATCH 192/415] [pbs] Add support for flp frontlines (Closes #7369)
---
youtube_dl/extractor/pbs.py | 18 ++++++++++++++++++
1 file changed, 18 insertions(+)
diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index 7b868d057..3169e9c3f 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -8,6 +8,7 @@ from ..utils import (
ExtractorError,
determine_ext,
int_or_none,
+ strip_jsonp,
unified_strdate,
US_RATINGS,
)
@@ -191,6 +192,23 @@ class PBSIE(InfoExtractor):
if media_id:
return media_id, presumptive_id, upload_date
+ # Fronline video embedded via flp
+ video_id = self._search_regex(
+ r'videoid\s*:\s*"([\d+a-z]{7,})"', webpage, 'videoid')
+ if video_id:
+ # pkg_id calculation is reverse engineered from
+ # http://www.pbs.org/wgbh/pages/frontline/js/flp2012.js
+ prg_id = self._search_regex(
+ r'videoid\s*:\s*"([\d+a-z]{7,})"', webpage, 'videoid')[7:]
+ if 'q' in prg_id:
+ prg_id = prg_id.split('q')[1]
+ prg_id = int(prg_id, 16)
+ getdir = self._download_json(
+ 'http://www.pbs.org/wgbh/pages/frontline/.json/getdir/getdir%d.json' % prg_id,
+ presumptive_id, 'Downloading getdir JSON',
+ transform_source=strip_jsonp)
+ return getdir['mid'], presumptive_id, upload_date
+
for iframe in re.findall(r'(?s)<iframe(.+?)></iframe>', webpage):
url = self._search_regex(
r'src=(["\'])(?P<url>.+?partnerplayer.+?)\1', iframe,
From 8b6d9406db1d3361b006016e6aace54b05cb6fea Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 7 Nov 2015 00:42:30 +0600
Subject: [PATCH 193/415] [pbs] Add test for flp frontline embeds
---
youtube_dl/extractor/pbs.py | 16 ++++++++++++++++
1 file changed, 16 insertions(+)
diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index 3169e9c3f..a690f9c29 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -154,6 +154,22 @@ class PBSIE(InfoExtractor):
'params': {
'skip_download': True, # requires ffmpeg
},
+ },
+ {
+ # Frontline video embedded via flp2012.js
+ 'url': 'http://www.pbs.org/wgbh/pages/frontline/the-atomic-artists',
+ 'info_dict': {
+ 'id': '2070868960',
+ 'display_id': 'the-atomic-artists',
+ 'ext': 'mp4',
+ 'title': 'FRONTLINE - The Atomic Artists',
+ 'description': 'md5:f5bfbefadf421e8bb8647602011caf8e',
+ 'duration': 723,
+ 'thumbnail': 're:^https?://.*\.jpg$',
+ },
+ 'params': {
+ 'skip_download': True, # requires ffmpeg
+ },
}
]
_ERRORS = {
From 21d0c33ecde573db961b97f5f0c37ba9d3c02ff3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 7 Nov 2015 01:08:40 +0600
Subject: [PATCH 194/415] [pbs] Make flp embed lookup non fatal
---
youtube_dl/extractor/pbs.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index a690f9c29..8fb9b1849 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -210,7 +210,7 @@ class PBSIE(InfoExtractor):
# Fronline video embedded via flp
video_id = self._search_regex(
- r'videoid\s*:\s*"([\d+a-z]{7,})"', webpage, 'videoid')
+ r'videoid\s*:\s*"([\d+a-z]{7,})"', webpage, 'videoid', default=None)
if video_id:
# pkg_id calculation is reverse engineered from
# http://www.pbs.org/wgbh/pages/frontline/js/flp2012.js
From ee223abb88263bdda2d92c4b2139d1dca60ba3ae Mon Sep 17 00:00:00 2001
From: Mister Hat <misterhat144@gmail.com>
Date: Tue, 3 Nov 2015 19:13:27 -0600
Subject: [PATCH 195/415] [vidzi] fixed. finds url from hash and host in script
Closes #7386.
---
youtube_dl/extractor/vidzi.py | 10 ++++++++--
1 file changed, 8 insertions(+), 2 deletions(-)
diff --git a/youtube_dl/extractor/vidzi.py b/youtube_dl/extractor/vidzi.py
index 08a5a7b8d..2ba9f31df 100644
--- a/youtube_dl/extractor/vidzi.py
+++ b/youtube_dl/extractor/vidzi.py
@@ -20,8 +20,14 @@ class VidziIE(InfoExtractor):
video_id = self._match_id(url)
webpage = self._download_webpage(url, video_id)
- video_url = self._html_search_regex(
- r'{\s*file\s*:\s*"([^"]+)"\s*}', webpage, 'video url')
+ video_host = self._html_search_regex(
+ r'id=\'vplayer\'><img src="http://(.*?)/i', webpage,
+ 'video host')
+ video_hash = self._html_search_regex(
+ r'\|([a-z0-9]+)\|hls\|type', webpage, 'video_hash')
+ ext = self._html_search_regex(
+ r'\|tracks\|([a-z0-9]+)\|', webpage, 'video ext')
+ video_url = 'http://' + video_host + '/' + video_hash + '/v.' + ext
title = self._html_search_regex(
r'(?s)<h2 class="video-title">(.*?)</h2>', webpage, 'title')
From 5d0f84d32cc038dd71673987cb6efaa85e953474 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 7 Nov 2015 06:23:00 +0600
Subject: [PATCH 196/415] [beeg] Skip empty URLs (Closes #7392)
---
youtube_dl/extractor/beeg.py | 2 ++
1 file changed, 2 insertions(+)
diff --git a/youtube_dl/extractor/beeg.py b/youtube_dl/extractor/beeg.py
index e6c928699..61bc2f744 100644
--- a/youtube_dl/extractor/beeg.py
+++ b/youtube_dl/extractor/beeg.py
@@ -33,6 +33,8 @@ class BeegIE(InfoExtractor):
formats = []
for format_id, video_url in video.items():
+ if not video_url:
+ continue
height = self._search_regex(
r'^(\d+)[pP]$', format_id, 'height', default=None)
if not height:
From 5214f1e31d5e5ba692fb1ed4803ff71ef4e480e8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 7 Nov 2015 19:25:59 +0600
Subject: [PATCH 197/415] [crunchyroll] Fix title extraction (Closes #7396)
---
youtube_dl/extractor/crunchyroll.py | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index 0c9b8ca02..4243f3e2e 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -287,7 +287,9 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
if 'To view this, please log in to verify you are 18 or older.' in webpage:
self.raise_login_required()
- video_title = self._html_search_regex(r'<h1[^>]*>(.+?)</h1>', webpage, 'video_title', flags=re.DOTALL)
+ video_title = self._html_search_regex(
+ r'(?s)<h1[^>]*>((?:(?!<h1).)*?<span[^>]+itemprop=["\']title["\'][^>]*>(?:(?!<h1).)+?)</h1>',
+ webpage, 'video_title')
video_title = re.sub(r' {2,}', ' ', video_title)
video_description = self._html_search_regex(r'"description":"([^"]+)', webpage, 'video_description', default='')
if not video_description:
From 2c740cf28d257d2a915195e7cc60f83e6690d2cc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 7 Nov 2015 19:29:09 +0600
Subject: [PATCH 198/415] [crunchyroll] Simplify description extraction
---
youtube_dl/extractor/crunchyroll.py | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index 4243f3e2e..9aa5d58b4 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -291,9 +291,8 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
r'(?s)<h1[^>]*>((?:(?!<h1).)*?<span[^>]+itemprop=["\']title["\'][^>]*>(?:(?!<h1).)+?)</h1>',
webpage, 'video_title')
video_title = re.sub(r' {2,}', ' ', video_title)
- video_description = self._html_search_regex(r'"description":"([^"]+)', webpage, 'video_description', default='')
- if not video_description:
- video_description = None
+ video_description = self._html_search_regex(
+ r'"description":"([^"]+)', webpage, 'video_description', default=None)
video_upload_date = self._html_search_regex(
[r'<div>Availability for free users:(.+?)</div>', r'<div>[^<>]+<span>\s*(.+?\d{4})\s*</span></div>'],
webpage, 'video_upload_date', fatal=False, flags=re.DOTALL)
From 6d02b9a392d39c114d3fb58bf7965f62196ccecd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 7 Nov 2015 20:02:39 +0600
Subject: [PATCH 199/415] [crunchyroll] Fix description extraction
---
youtube_dl/extractor/crunchyroll.py | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index 9aa5d58b4..6e5999c72 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -21,6 +21,7 @@ from ..utils import (
bytes_to_intlist,
intlist_to_bytes,
int_or_none,
+ lowercase_escape,
remove_end,
unified_strdate,
urlencode_postdata,
@@ -104,7 +105,7 @@ class CrunchyrollIE(CrunchyrollBaseIE):
'id': '589804',
'ext': 'flv',
'title': 'Culture Japan Episode 1 – Rebuilding Japan after the 3.11',
- 'description': 'md5:fe2743efedb49d279552926d0bd0cd9e',
+ 'description': 'md5:2fbc01f90b87e8e9137296f37b461c12',
'thumbnail': 're:^https?://.*\.jpg$',
'uploader': 'Danny Choo Network',
'upload_date': '20120213',
@@ -292,7 +293,10 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
webpage, 'video_title')
video_title = re.sub(r' {2,}', ' ', video_title)
video_description = self._html_search_regex(
- r'"description":"([^"]+)', webpage, 'video_description', default=None)
+ r'<script[^>]*>\s*.+?\[media_id=%s\].+?"description"\s*:\s*"([^"]+)' % video_id,
+ webpage, 'description', default=None)
+ if video_description:
+ video_description = lowercase_escape(video_description.replace(r'\r\n', '\n'))
video_upload_date = self._html_search_regex(
[r'<div>Availability for free users:(.+?)</div>', r'<div>[^<>]+<span>\s*(.+?\d{4})\s*</span></div>'],
webpage, 'video_upload_date', fatal=False, flags=re.DOTALL)
From cff551c0b0ed8eb55c1ab63ec669c07a51aa4998 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
<jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 7 Nov 2015 18:43:22 +0100
Subject: [PATCH 200/415] [googleplus] Fix extraction of formats
---
youtube_dl/extractor/googleplus.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/googleplus.py b/youtube_dl/extractor/googleplus.py
index fcefe54cd..731bacd67 100644
--- a/youtube_dl/extractor/googleplus.py
+++ b/youtube_dl/extractor/googleplus.py
@@ -61,7 +61,7 @@ class GooglePlusIE(InfoExtractor):
'width': int(width),
'height': int(height),
} for width, height, video_url in re.findall(
- r'\d+,(\d+),(\d+),"(https?://redirector\.googlevideo\.com.*?)"', webpage)]
+ r'\d+,(\d+),(\d+),"(https?://[^.]+\.googleusercontent.com.*?)"', webpage)]
self._sort_formats(formats)
return {
From ee4337d100f68bbb2ae795101d4c391b522ec753 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 6 Nov 2015 20:16:14 +0100
Subject: [PATCH 201/415] [videolecture] add support for multi part videos
---
youtube_dl/extractor/videolecturesnet.py | 95 +++++++++++++++++-------
1 file changed, 70 insertions(+), 25 deletions(-)
diff --git a/youtube_dl/extractor/videolecturesnet.py b/youtube_dl/extractor/videolecturesnet.py
index 649ac9433..351706362 100644
--- a/youtube_dl/extractor/videolecturesnet.py
+++ b/youtube_dl/extractor/videolecturesnet.py
@@ -10,17 +10,19 @@ from ..compat import (
from ..utils import (
ExtractorError,
parse_duration,
+ js_to_json,
+ parse_iso8601,
)
class VideoLecturesNetIE(InfoExtractor):
- _VALID_URL = r'http://(?:www\.)?videolectures\.net/(?P<id>[^/#?]+)/*(?:[#?].*)?$'
+ _VALID_URL = r'http://(?:www\.)?videolectures\.net/(?P<id>[^/]+)(?:/video/(?P<part>\d+))?'
IE_NAME = 'videolectures.net'
_TESTS = [{
'url': 'http://videolectures.net/promogram_igor_mekjavic_eng/',
'info_dict': {
- 'id': 'promogram_igor_mekjavic_eng',
+ 'id': '20171_part1',
'ext': 'mp4',
'title': 'Automatics, robotics and biocybernetics',
'description': 'md5:815fc1deb6b3a2bff99de2d5325be482',
@@ -32,7 +34,7 @@ class VideoLecturesNetIE(InfoExtractor):
# video with invalid direct format links (HTTP 403)
'url': 'http://videolectures.net/russir2010_filippova_nlp/',
'info_dict': {
- 'id': 'russir2010_filippova_nlp',
+ 'id': '14891_part1',
'ext': 'flv',
'title': 'NLP at Google',
'description': 'md5:fc7a6d9bf0302d7cc0e53f7ca23747b3',
@@ -46,37 +48,80 @@ class VideoLecturesNetIE(InfoExtractor):
}, {
'url': 'http://videolectures.net/deeplearning2015_montreal/',
'info_dict': {
- 'id': 'deeplearning2015_montreal',
+ 'id': '23181',
'title': 'Deep Learning Summer School, Montreal 2015',
- 'description': 'md5:90121a40cc6926df1bf04dcd8563ed3b',
+ 'description': 'md5:0533a85e4bd918df52a01f0e1ebe87b7',
+ 'timestamp': 1438560000,
},
'playlist_count': 30,
+ }, {
+ # multi part lecture
+ 'url': 'http://videolectures.net/mlss09uk_bishop_ibi/',
+ 'info_dict': {
+ 'id': '9737',
+ 'title': 'Introduction To Bayesian Inference',
+ 'timestamp': 1251622800,
+ },
+ 'playlist': [{
+ 'info_dict': {
+ 'id': '9737_part1',
+ 'ext': 'wmv',
+ 'title': 'Introduction To Bayesian Inference',
+ },
+ }, {
+ 'info_dict': {
+ 'id': '9737_part2',
+ 'ext': 'wmv',
+ 'title': 'Introduction To Bayesian Inference',
+ },
+ }],
+ 'playlist_count': 2,
}]
def _real_extract(self, url):
- video_id = self._match_id(url)
+ lecture_slug, part = re.match(self._VALID_URL, url).groups()
- smil_url = 'http://videolectures.net/%s/video/1/smil.xml' % video_id
+ webpage = self._download_webpage(url, lecture_slug)
- try:
- smil = self._download_smil(smil_url, video_id)
- except ExtractorError as e:
- if isinstance(e.cause, compat_HTTPError) and e.cause.code == 404:
- # Probably a playlist
- webpage = self._download_webpage(url, video_id)
- entries = [
- self.url_result(compat_urlparse.urljoin(url, video_url), 'VideoLecturesNet')
- for _, video_url in re.findall(r'<a[^>]+href=(["\'])(.+?)\1[^>]+id=["\']lec=\d+', webpage)]
- playlist_title = self._html_search_meta('title', webpage, 'title', fatal=True)
- playlist_description = self._html_search_meta('description', webpage, 'description')
- return self.playlist_result(entries, video_id, playlist_title, playlist_description)
+ cfg = self._parse_json(self._search_regex(r'cfg\s*:\s*({[^}]+})', webpage, 'cfg'), lecture_slug, js_to_json)
- info = self._parse_smil(smil, smil_url, video_id)
+ lecture_id = str(cfg['obj_id'])
- info['id'] = video_id
+ lecture_data = self._download_json('%s/site/api/lecture/%s?format=json' % (self._proto_relative_url(cfg['livepipe'], 'http:'), lecture_id), lecture_id)['lecture'][0]
- switch = smil.find('.//switch')
- if switch is not None:
- info['duration'] = parse_duration(switch.attrib.get('dur'))
+ lecture_info = {
+ 'id': lecture_id,
+ 'display_id': lecture_slug,
+ 'title': lecture_data['title'],
+ 'timestamp': parse_iso8601(lecture_data.get('time')),
+ 'description': lecture_data.get('description_wiki'),
+ 'thumbnail': lecture_data.get('thumb'),
+ }
- return info
+ entries = []
+ parts = cfg.get('videos')
+ if parts:
+ if len(parts) == 1:
+ part = str(parts[0])
+ if part:
+ smil_url = 'http://videolectures.net/%s/video/%s/smil.xml' % (lecture_slug, part)
+ smil = self._download_smil(smil_url, lecture_id)
+ info = self._parse_smil(smil, smil_url, lecture_id)
+ info['id'] = '%s_part%s' % (lecture_id, part)
+ switch = smil.find('.//switch')
+ if switch is not None:
+ info['duration'] = parse_duration(switch.attrib.get('dur'))
+ return info
+ else:
+ for part in parts:
+ entries.append(self.url_result('http://videolectures.net/%s/video/%s' % (lecture_slug, part), 'VideoLecturesNet'))
+ lecture_info['_type'] = 'multi_video'
+ else:
+ # Probably a playlist
+ entries = [
+ self.url_result(compat_urlparse.urljoin(url, video_url), 'VideoLecturesNet')
+ for _, video_url in re.findall(r'<a[^>]+href=(["\'])(.+?)\1[^>]+id=["\']lec=\d+', webpage)]
+ lecture_info['_type'] = 'playlist'
+
+ lecture_info['entries'] = entries
+ return lecture_info
From a06bf87a2c6009d82ec28afe566f653b3deb11bf Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Fri, 6 Nov 2015 21:23:41 +0100
Subject: [PATCH 202/415] [viidea] add support for sites using viidea service
---
youtube_dl/extractor/__init__.py | 2 +-
.../{videolecturesnet.py => viidea.py} | 33 ++++++++++++++-----
2 files changed, 26 insertions(+), 9 deletions(-)
rename youtube_dl/extractor/{videolecturesnet.py => viidea.py} (77%)
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 94150a28f..0a90da73c 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -724,7 +724,6 @@ from .vh1 import VH1IE
from .vice import ViceIE
from .viddler import ViddlerIE
from .videodetective import VideoDetectiveIE
-from .videolecturesnet import VideoLecturesNetIE
from .videofyme import VideofyMeIE
from .videomega import VideoMegaIE
from .videopremium import VideoPremiumIE
@@ -734,6 +733,7 @@ from .vidme import VidmeIE
from .vidzi import VidziIE
from .vier import VierIE, VierVideosIE
from .viewster import ViewsterIE
+from .viidea import ViideaIE
from .vimeo import (
VimeoIE,
VimeoAlbumIE,
diff --git a/youtube_dl/extractor/videolecturesnet.py b/youtube_dl/extractor/viidea.py
similarity index 77%
rename from youtube_dl/extractor/videolecturesnet.py
rename to youtube_dl/extractor/viidea.py
index 351706362..71fb298e6 100644
--- a/youtube_dl/extractor/videolecturesnet.py
+++ b/youtube_dl/extractor/viidea.py
@@ -15,9 +15,23 @@ from ..utils import (
)
-class VideoLecturesNetIE(InfoExtractor):
- _VALID_URL = r'http://(?:www\.)?videolectures\.net/(?P<id>[^/]+)(?:/video/(?P<part>\d+))?'
- IE_NAME = 'videolectures.net'
+class ViideaIE(InfoExtractor):
+ _VALID_URL = r'''(?x)http://(?:www\.)?(?:
+ videolectures\.net|
+ flexilearn\.viidea\.net|
+ presentations\.ocwconsortium\.org|
+ video\.travel-zoom\.si|
+ video\.pomp-forum\.si|
+ tv\.nil\.si|
+ video\.hekovnik.com|
+ video\.szko\.si|
+ kpk\.viidea\.com|
+ inside\.viidea\.net|
+ video\.kiberpipa\.org|
+ bvvideo\.si|
+ kongres\.viidea\.net|
+ edemokracija\.viidea\.com
+ )(?:/lecture)?/(?P<id>[^/]+)(?:/video/(?P<part>\d+))?'''
_TESTS = [{
'url': 'http://videolectures.net/promogram_igor_mekjavic_eng/',
@@ -87,7 +101,9 @@ class VideoLecturesNetIE(InfoExtractor):
lecture_id = str(cfg['obj_id'])
- lecture_data = self._download_json('%s/site/api/lecture/%s?format=json' % (self._proto_relative_url(cfg['livepipe'], 'http:'), lecture_id), lecture_id)['lecture'][0]
+ base_url = self._proto_relative_url(cfg['livepipe'], 'http:')
+
+ lecture_data = self._download_json('%s/site/api/lecture/%s?format=json' % (base_url, lecture_id), lecture_id)['lecture'][0]
lecture_info = {
'id': lecture_id,
@@ -104,7 +120,7 @@ class VideoLecturesNetIE(InfoExtractor):
if len(parts) == 1:
part = str(parts[0])
if part:
- smil_url = 'http://videolectures.net/%s/video/%s/smil.xml' % (lecture_slug, part)
+ smil_url = '%s/%s/video/%s/smil.xml' % (base_url, lecture_slug, part)
smil = self._download_smil(smil_url, lecture_id)
info = self._parse_smil(smil, smil_url, lecture_id)
info['id'] = '%s_part%s' % (lecture_id, part)
@@ -114,13 +130,14 @@ class VideoLecturesNetIE(InfoExtractor):
return info
else:
for part in parts:
- entries.append(self.url_result('http://videolectures.net/%s/video/%s' % (lecture_slug, part), 'VideoLecturesNet'))
+ entries.append(self.url_result('%s/video/%s' % (base_url, lecture_id, part), 'Viidea'))
lecture_info['_type'] = 'multi_video'
else:
# Probably a playlist
+ playlist_webpage = self._download_webpage('%s/site/ajax/drilldown/?id=%s' % (base_url, lecture_id), lecture_id)
entries = [
- self.url_result(compat_urlparse.urljoin(url, video_url), 'VideoLecturesNet')
- for _, video_url in re.findall(r'<a[^>]+href=(["\'])(.+?)\1[^>]+id=["\']lec=\d+', webpage)]
+ self.url_result(compat_urlparse.urljoin(url, video_url), 'Viidea')
+ for _, video_url in re.findall(r'<a[^>]+href=(["\'])(.+?)\1[^>]+id=["\']lec=\d+', playlist_webpage)]
lecture_info['_type'] = 'playlist'
lecture_info['entries'] = entries
From 8e3a2bd6200660f9fb9d485b1c924fa5462bd566 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 7 Nov 2015 17:43:23 +0100
Subject: [PATCH 203/415] [viidea] fix _VALID_URL regex and tests
---
youtube_dl/extractor/viidea.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/youtube_dl/extractor/viidea.py b/youtube_dl/extractor/viidea.py
index 71fb298e6..ae9a42737 100644
--- a/youtube_dl/extractor/viidea.py
+++ b/youtube_dl/extractor/viidea.py
@@ -31,7 +31,7 @@ class ViideaIE(InfoExtractor):
bvvideo\.si|
kongres\.viidea\.net|
edemokracija\.viidea\.com
- )(?:/lecture)?/(?P<id>[^/]+)(?:/video/(?P<part>\d+))?'''
+ )(?:/lecture)?/(?P<id>[^/]+)(?:/video/(?P<part>\d+))?/*(?:[#?].*)?$'''
_TESTS = [{
'url': 'http://videolectures.net/promogram_igor_mekjavic_eng/',
@@ -130,7 +130,7 @@ class ViideaIE(InfoExtractor):
return info
else:
for part in parts:
- entries.append(self.url_result('%s/video/%s' % (base_url, lecture_id, part), 'Viidea'))
+ entries.append(self.url_result('%s/%s/video/%s' % (base_url, lecture_slug, part), 'Viidea'))
lecture_info['_type'] = 'multi_video'
else:
# Probably a playlist
From 6fdb39ded15c6276b49fa67cb517bf1fed63af35 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 7 Nov 2015 20:38:33 +0100
Subject: [PATCH 204/415] [viidia] Cleaup
[viidea] extract playlist if lecture is an event
[viidia] use compat_str
---
youtube_dl/extractor/viidea.py | 11 +++++------
1 file changed, 5 insertions(+), 6 deletions(-)
diff --git a/youtube_dl/extractor/viidea.py b/youtube_dl/extractor/viidea.py
index ae9a42737..2541a36ed 100644
--- a/youtube_dl/extractor/viidea.py
+++ b/youtube_dl/extractor/viidea.py
@@ -4,11 +4,10 @@ import re
from .common import InfoExtractor
from ..compat import (
- compat_HTTPError,
compat_urlparse,
+ compat_str,
)
from ..utils import (
- ExtractorError,
parse_duration,
js_to_json,
parse_iso8601,
@@ -97,9 +96,9 @@ class ViideaIE(InfoExtractor):
webpage = self._download_webpage(url, lecture_slug)
- cfg = self._parse_json(self._search_regex(r'cfg\s*:\s*({[^}]+})', webpage, 'cfg'), lecture_slug, js_to_json)
+ cfg = self._parse_json(self._search_regex([r'cfg\s*:\s*({.+?}),[\da-zA-Z_]:\(?function', r'cfg\s*:\s*({[^}]+})'], webpage, 'cfg'), lecture_slug, js_to_json)
- lecture_id = str(cfg['obj_id'])
+ lecture_id = compat_str(cfg['obj_id'])
base_url = self._proto_relative_url(cfg['livepipe'], 'http:')
@@ -118,7 +117,7 @@ class ViideaIE(InfoExtractor):
parts = cfg.get('videos')
if parts:
if len(parts) == 1:
- part = str(parts[0])
+ part = compat_str(parts[0])
if part:
smil_url = '%s/%s/video/%s/smil.xml' % (base_url, lecture_slug, part)
smil = self._download_smil(smil_url, lecture_id)
@@ -132,7 +131,7 @@ class ViideaIE(InfoExtractor):
for part in parts:
entries.append(self.url_result('%s/%s/video/%s' % (base_url, lecture_slug, part), 'Viidea'))
lecture_info['_type'] = 'multi_video'
- else:
+ if not parts or lecture_data.get('type') == 'evt':
# Probably a playlist
playlist_webpage = self._download_webpage('%s/site/ajax/drilldown/?id=%s' % (base_url, lecture_id), lecture_id)
entries = [
From e8ce2375e0851e65c4882002297404825fe1045e Mon Sep 17 00:00:00 2001
From: Sergey M? <dstftw@gmail.com>
Date: Sun, 8 Nov 2015 06:54:27 +0600
Subject: [PATCH 205/415] [viidea] Improve and cleanup (Closes #7390)
* Optimize requests for multipart videos
* Fix cfg regex
* Improve titles and identifiers
---
youtube_dl/extractor/viidea.py | 99 ++++++++++++++++++++++++----------
1 file changed, 72 insertions(+), 27 deletions(-)
diff --git a/youtube_dl/extractor/viidea.py b/youtube_dl/extractor/viidea.py
index 2541a36ed..525e303d4 100644
--- a/youtube_dl/extractor/viidea.py
+++ b/youtube_dl/extractor/viidea.py
@@ -35,35 +35,42 @@ class ViideaIE(InfoExtractor):
_TESTS = [{
'url': 'http://videolectures.net/promogram_igor_mekjavic_eng/',
'info_dict': {
- 'id': '20171_part1',
+ 'id': '20171',
+ 'display_id': 'promogram_igor_mekjavic_eng',
'ext': 'mp4',
'title': 'Automatics, robotics and biocybernetics',
'description': 'md5:815fc1deb6b3a2bff99de2d5325be482',
+ 'thumbnail': 're:http://.*\.jpg',
+ 'timestamp': 1372349289,
'upload_date': '20130627',
'duration': 565,
- 'thumbnail': 're:http://.*\.jpg',
},
}, {
# video with invalid direct format links (HTTP 403)
'url': 'http://videolectures.net/russir2010_filippova_nlp/',
'info_dict': {
- 'id': '14891_part1',
+ 'id': '14891',
+ 'display_id': 'russir2010_filippova_nlp',
'ext': 'flv',
'title': 'NLP at Google',
'description': 'md5:fc7a6d9bf0302d7cc0e53f7ca23747b3',
- 'duration': 5352,
'thumbnail': 're:http://.*\.jpg',
+ 'timestamp': 1284375600,
+ 'upload_date': '20100913',
+ 'duration': 5352,
},
'params': {
# rtmp download
'skip_download': True,
},
}, {
+ # event playlist
'url': 'http://videolectures.net/deeplearning2015_montreal/',
'info_dict': {
'id': '23181',
'title': 'Deep Learning Summer School, Montreal 2015',
'description': 'md5:0533a85e4bd918df52a01f0e1ebe87b7',
+ 'thumbnail': 're:http://.*\.jpg',
'timestamp': 1438560000,
},
'playlist_count': 30,
@@ -72,37 +79,54 @@ class ViideaIE(InfoExtractor):
'url': 'http://videolectures.net/mlss09uk_bishop_ibi/',
'info_dict': {
'id': '9737',
+ 'display_id': 'mlss09uk_bishop_ibi',
'title': 'Introduction To Bayesian Inference',
+ 'thumbnail': 're:http://.*\.jpg',
'timestamp': 1251622800,
},
'playlist': [{
'info_dict': {
'id': '9737_part1',
+ 'display_id': 'mlss09uk_bishop_ibi_part1',
'ext': 'wmv',
- 'title': 'Introduction To Bayesian Inference',
+ 'title': 'Introduction To Bayesian Inference (Part 1)',
+ 'thumbnail': 're:http://.*\.jpg',
+ 'duration': 4622,
+ 'timestamp': 1251622800,
+ 'upload_date': '20090830',
},
}, {
'info_dict': {
'id': '9737_part2',
+ 'display_id': 'mlss09uk_bishop_ibi_part2',
'ext': 'wmv',
- 'title': 'Introduction To Bayesian Inference',
+ 'title': 'Introduction To Bayesian Inference (Part 2)',
+ 'thumbnail': 're:http://.*\.jpg',
+ 'duration': 5641,
+ 'timestamp': 1251622800,
+ 'upload_date': '20090830',
},
}],
'playlist_count': 2,
}]
def _real_extract(self, url):
- lecture_slug, part = re.match(self._VALID_URL, url).groups()
+ lecture_slug, explicit_part_id = re.match(self._VALID_URL, url).groups()
webpage = self._download_webpage(url, lecture_slug)
- cfg = self._parse_json(self._search_regex([r'cfg\s*:\s*({.+?}),[\da-zA-Z_]:\(?function', r'cfg\s*:\s*({[^}]+})'], webpage, 'cfg'), lecture_slug, js_to_json)
+ cfg = self._parse_json(self._search_regex(
+ [r'cfg\s*:\s*({.+?})\s*,\s*[\da-zA-Z_]+\s*:\s*\(?\s*function',
+ r'cfg\s*:\s*({[^}]+})'],
+ webpage, 'cfg'), lecture_slug, js_to_json)
lecture_id = compat_str(cfg['obj_id'])
base_url = self._proto_relative_url(cfg['livepipe'], 'http:')
- lecture_data = self._download_json('%s/site/api/lecture/%s?format=json' % (base_url, lecture_id), lecture_id)['lecture'][0]
+ lecture_data = self._download_json(
+ '%s/site/api/lecture/%s?format=json' % (base_url, lecture_id),
+ lecture_id)['lecture'][0]
lecture_info = {
'id': lecture_id,
@@ -113,31 +137,52 @@ class ViideaIE(InfoExtractor):
'thumbnail': lecture_data.get('thumb'),
}
- entries = []
- parts = cfg.get('videos')
+ playlist_entries = []
+ lecture_type = lecture_data.get('type')
+ parts = [compat_str(video) for video in cfg.get('videos', [])]
if parts:
- if len(parts) == 1:
- part = compat_str(parts[0])
- if part:
- smil_url = '%s/%s/video/%s/smil.xml' % (base_url, lecture_slug, part)
+ multipart = len(parts) > 1
+
+ def extract_part(part_id):
+ smil_url = '%s/%s/video/%s/smil.xml' % (base_url, lecture_slug, part_id)
smil = self._download_smil(smil_url, lecture_id)
info = self._parse_smil(smil, smil_url, lecture_id)
- info['id'] = '%s_part%s' % (lecture_id, part)
+ info['id'] = lecture_id if not multipart else '%s_part%s' % (lecture_id, part_id)
+ info['display_id'] = lecture_slug if not multipart else '%s_part%s' % (lecture_slug, part_id)
+ if multipart:
+ info['title'] += ' (Part %s)' % part_id
switch = smil.find('.//switch')
if switch is not None:
info['duration'] = parse_duration(switch.attrib.get('dur'))
- return info
+ item_info = lecture_info.copy()
+ item_info.update(info)
+ return item_info
+
+ if explicit_part_id or not multipart:
+ result = extract_part(explicit_part_id or parts[0])
else:
- for part in parts:
- entries.append(self.url_result('%s/%s/video/%s' % (base_url, lecture_slug, part), 'Viidea'))
- lecture_info['_type'] = 'multi_video'
- if not parts or lecture_data.get('type') == 'evt':
- # Probably a playlist
- playlist_webpage = self._download_webpage('%s/site/ajax/drilldown/?id=%s' % (base_url, lecture_id), lecture_id)
+ result = {
+ '_type': 'multi_video',
+ 'entries': [extract_part(part) for part in parts],
+ }
+ result.update(lecture_info)
+
+ # Immediately return explicitly requested part or non event item
+ if explicit_part_id or lecture_type != 'evt':
+ return result
+
+ playlist_entries.append(result)
+
+ # It's probably a playlist
+ if not parts or lecture_type == 'evt':
+ playlist_webpage = self._download_webpage(
+ '%s/site/ajax/drilldown/?id=%s' % (base_url, lecture_id), lecture_id)
entries = [
self.url_result(compat_urlparse.urljoin(url, video_url), 'Viidea')
- for _, video_url in re.findall(r'<a[^>]+href=(["\'])(.+?)\1[^>]+id=["\']lec=\d+', playlist_webpage)]
- lecture_info['_type'] = 'playlist'
+ for _, video_url in re.findall(
+ r'<a[^>]+href=(["\'])(.+?)\1[^>]+id=["\']lec=\d+', playlist_webpage)]
+ playlist_entries.extend(entries)
- lecture_info['entries'] = entries
- return lecture_info
+ playlist = self.playlist_result(playlist_entries, lecture_id)
+ playlist.update(lecture_info)
+ return playlist
From d5c181a14e08198e400932d591b47683a630c8c1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
<jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 8 Nov 2015 11:49:51 +0100
Subject: [PATCH 206/415] [movieclips] Fix extraction (fixes #7404)
They use theplatform now.
Changed the test, because the old one seems to be georestricted.
---
youtube_dl/extractor/movieclips.py | 77 ++++++++----------------------
1 file changed, 19 insertions(+), 58 deletions(-)
diff --git a/youtube_dl/extractor/movieclips.py b/youtube_dl/extractor/movieclips.py
index 04e17d055..e06828b55 100644
--- a/youtube_dl/extractor/movieclips.py
+++ b/youtube_dl/extractor/movieclips.py
@@ -1,80 +1,41 @@
from __future__ import unicode_literals
-import re
-
from .common import InfoExtractor
from ..compat import (
- compat_str,
-)
-from ..utils import (
- ExtractorError,
- clean_html,
+ compat_urllib_request,
)
class MovieClipsIE(InfoExtractor):
- _VALID_URL = r'https?://movieclips\.com/(?P<id>[\da-zA-Z]+)(?:-(?P<display_id>[\da-z-]+))?'
+ _VALID_URL = r'https?://(?:www.)?movieclips\.com/videos/(?P<id>[^/?#]+)'
_TEST = {
- 'url': 'http://movieclips.com/Wy7ZU-my-week-with-marilyn-movie-do-you-love-me/',
+ 'url': 'http://www.movieclips.com/videos/warcraft-trailer-1-561180739597?autoPlay=true&playlistId=5',
'info_dict': {
- 'id': 'Wy7ZU',
- 'display_id': 'my-week-with-marilyn-movie-do-you-love-me',
+ 'id': 'pKIGmG83AqD9',
+ 'display_id': 'warcraft-trailer-1-561180739597',
'ext': 'mp4',
- 'title': 'My Week with Marilyn - Do You Love Me?',
- 'description': 'md5:e86795bd332fe3cff461e7c8dc542acb',
+ 'title': 'Warcraft Trailer 1',
+ 'description': 'Watch Trailer 1 from Warcraft (2016). Legendary’s WARCRAFT is a 3D epic adventure of world-colliding conflict based.',
'thumbnail': 're:^https?://.*\.jpg$',
},
- 'params': {
- # rtmp download
- 'skip_download': True,
- }
+ 'add_ie': ['ThePlatform'],
}
def _real_extract(self, url):
- mobj = re.match(self._VALID_URL, url)
- video_id = mobj.group('id')
- display_id = mobj.group('display_id')
- show_id = display_id or video_id
+ display_id = self._match_id(url)
- config = self._download_xml(
- 'http://config.movieclips.com/player/config/%s' % video_id,
- show_id, 'Downloading player config')
-
- if config.find('./country-region').text == 'false':
- raise ExtractorError(
- '%s said: %s' % (self.IE_NAME, config.find('./region_alert').text), expected=True)
-
- properties = config.find('./video/properties')
- smil_file = properties.attrib['smil_file']
-
- smil = self._download_xml(smil_file, show_id, 'Downloading SMIL')
- base_url = smil.find('./head/meta').attrib['base']
-
- formats = []
- for video in smil.findall('./body/switch/video'):
- vbr = int(video.attrib['system-bitrate']) / 1000
- src = video.attrib['src']
- formats.append({
- 'url': base_url,
- 'play_path': src,
- 'ext': src.split(':')[0],
- 'vbr': vbr,
- 'format_id': '%dk' % vbr,
- })
-
- self._sort_formats(formats)
-
- title = '%s - %s' % (properties.attrib['clip_movie_title'], properties.attrib['clip_title'])
- description = clean_html(compat_str(properties.attrib['clip_description']))
- thumbnail = properties.attrib['image']
- categories = properties.attrib['clip_categories'].split(',')
+ req = compat_urllib_request.Request(url)
+ # it doesn't work if it thinks the browser it's too old
+ req.add_header('User-Agent', 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20150101 Firefox/43.0 (Chrome)')
+ webpage = self._download_webpage(req, display_id)
+ theplatform_link = self._html_search_regex(r'src="(http://player.theplatform.com/p/.*?)"', webpage, 'theplatform link')
+ title = self._html_search_regex(r'<title[^>]*>([^>]+)-\s*\d+\s*|\s*Movieclips.com</title>', webpage, 'title')
+ description = self._html_search_meta('description', webpage)
return {
- 'id': video_id,
- 'display_id': display_id,
+ '_type': 'url_transparent',
+ 'url': theplatform_link,
'title': title,
+ 'display_id': display_id,
'description': description,
- 'thumbnail': thumbnail,
- 'categories': categories,
- 'formats': formats,
}
From 937511dfc01c3d00c35a00f78c2b6f989b4d46e3 Mon Sep 17 00:00:00 2001
From: Frans de Jonge <fransdejonge@gmail.com>
Date: Sat, 7 Nov 2015 22:55:02 +0100
Subject: [PATCH 207/415] Added support for the RTBF OUFtivi subpage
---
youtube_dl/extractor/rtbf.py | 41 ++++++++++++++++++++++++++----------
1 file changed, 30 insertions(+), 11 deletions(-)
diff --git a/youtube_dl/extractor/rtbf.py b/youtube_dl/extractor/rtbf.py
index 04a66df90..e75b45112 100644
--- a/youtube_dl/extractor/rtbf.py
+++ b/youtube_dl/extractor/rtbf.py
@@ -9,17 +9,36 @@ from ..utils import (
class RTBFIE(InfoExtractor):
- _VALID_URL = r'https?://www.rtbf.be/video/[^\?]+\?id=(?P<id>\d+)'
- _TEST = {
- 'url': 'https://www.rtbf.be/video/detail_les-diables-au-coeur-episode-2?id=1921274',
- 'md5': '799f334ddf2c0a582ba80c44655be570',
- 'info_dict': {
- 'id': '1921274',
- 'ext': 'mp4',
- 'title': 'Les Diables au coeur (épisode 2)',
- 'duration': 3099,
- }
- }
+ _VALID_URL = r'''(?x)
+ https?://www\.rtbf\.be/
+ (?:
+ video/[^\?]+\?id=|
+ ouftivi/heros/[^&]+&videoId=
+ )
+ (?P<id>\d+)
+ '''
+ _TESTS = [
+ {
+ 'url': 'https://www.rtbf.be/video/detail_les-diables-au-coeur-episode-2?id=1921274',
+ 'md5': '799f334ddf2c0a582ba80c44655be570',
+ 'info_dict': {
+ 'id': '1921274',
+ 'ext': 'mp4',
+ 'title': 'Les Diables au coeur (épisode 2)',
+ 'duration': 3099,
+ }
+ },
+ {
+ 'url': 'http://www.rtbf.be/ouftivi/heros/detail_scooby-doo-mysteres-associes?id=1097&videoId=2057442',
+ 'md5': '25aea17e949e1e0c7c41270d60d25f22',
+ 'info_dict': {
+ 'id': '2057442',
+ 'ext': 'mp4',
+ 'title': 'Scooby-Doo, myst\xe8res associ\xe9s',
+ 'duration': 1279,
+ }
+ },
+ ]
_QUALITIES = [
('mobile', 'mobile'),
From fda2717ef9d429358d5816582590d15d18f9109f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 8 Nov 2015 16:56:20 +0600
Subject: [PATCH 208/415] [movieclips] Add coding cookie
---
youtube_dl/extractor/movieclips.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/youtube_dl/extractor/movieclips.py b/youtube_dl/extractor/movieclips.py
index e06828b55..b8c43a163 100644
--- a/youtube_dl/extractor/movieclips.py
+++ b/youtube_dl/extractor/movieclips.py
@@ -1,3 +1,4 @@
+# coding: utf-8
from __future__ import unicode_literals
from .common import InfoExtractor
From 114e6025b09e12bd01b5ce22bd2c43a3ef0ba460 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 8 Nov 2015 17:01:45 +0600
Subject: [PATCH 209/415] [rtbf] Expand _VALID_URL (Closes #7402)
---
youtube_dl/extractor/rtbf.py | 48 ++++++++++++++----------------------
1 file changed, 18 insertions(+), 30 deletions(-)
diff --git a/youtube_dl/extractor/rtbf.py b/youtube_dl/extractor/rtbf.py
index e75b45112..acf10e253 100644
--- a/youtube_dl/extractor/rtbf.py
+++ b/youtube_dl/extractor/rtbf.py
@@ -9,36 +9,24 @@ from ..utils import (
class RTBFIE(InfoExtractor):
- _VALID_URL = r'''(?x)
- https?://www\.rtbf\.be/
- (?:
- video/[^\?]+\?id=|
- ouftivi/heros/[^&]+&videoId=
- )
- (?P<id>\d+)
- '''
- _TESTS = [
- {
- 'url': 'https://www.rtbf.be/video/detail_les-diables-au-coeur-episode-2?id=1921274',
- 'md5': '799f334ddf2c0a582ba80c44655be570',
- 'info_dict': {
- 'id': '1921274',
- 'ext': 'mp4',
- 'title': 'Les Diables au coeur (épisode 2)',
- 'duration': 3099,
- }
- },
- {
- 'url': 'http://www.rtbf.be/ouftivi/heros/detail_scooby-doo-mysteres-associes?id=1097&videoId=2057442',
- 'md5': '25aea17e949e1e0c7c41270d60d25f22',
- 'info_dict': {
- 'id': '2057442',
- 'ext': 'mp4',
- 'title': 'Scooby-Doo, myst\xe8res associ\xe9s',
- 'duration': 1279,
- }
- },
- ]
+ _VALID_URL = r'https?://www\.rtbf\.be/(?:video/[^?]+\?.*\bid=|ouftivi/(?:[^/]+/)*[^?]+\?.*\bvideoId=)(?P<id>\d+)'
+ _TESTS = [{
+ 'url': 'https://www.rtbf.be/video/detail_les-diables-au-coeur-episode-2?id=1921274',
+ 'md5': '799f334ddf2c0a582ba80c44655be570',
+ 'info_dict': {
+ 'id': '1921274',
+ 'ext': 'mp4',
+ 'title': 'Les Diables au coeur (épisode 2)',
+ 'duration': 3099,
+ }
+ }, {
+ # geo restricted
+ 'url': 'http://www.rtbf.be/ouftivi/heros/detail_scooby-doo-mysteres-associes?id=1097&videoId=2057442',
+ 'only_matching': True,
+ }, {
+ 'url': 'http://www.rtbf.be/ouftivi/niouzz?videoId=2055858',
+ 'only_matching': True,
+ }]
_QUALITIES = [
('mobile', 'mobile'),
From aa8d2d5be6a99542b85a85af3310fab1bf641e86 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 8 Nov 2015 17:03:21 +0600
Subject: [PATCH 210/415] [rtbf] Make www optional in _VALID_URL
---
youtube_dl/extractor/rtbf.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/rtbf.py b/youtube_dl/extractor/rtbf.py
index acf10e253..e42b319a3 100644
--- a/youtube_dl/extractor/rtbf.py
+++ b/youtube_dl/extractor/rtbf.py
@@ -9,7 +9,7 @@ from ..utils import (
class RTBFIE(InfoExtractor):
- _VALID_URL = r'https?://www\.rtbf\.be/(?:video/[^?]+\?.*\bid=|ouftivi/(?:[^/]+/)*[^?]+\?.*\bvideoId=)(?P<id>\d+)'
+ _VALID_URL = r'https?://(?:www\.)?rtbf\.be/(?:video/[^?]+\?.*\bid=|ouftivi/(?:[^/]+/)*[^?]+\?.*\bvideoId=)(?P<id>\d+)'
_TESTS = [{
'url': 'https://www.rtbf.be/video/detail_les-diables-au-coeur-episode-2?id=1921274',
'md5': '799f334ddf2c0a582ba80c44655be570',
From 50506cb60798fe4d2ebb9603798b3fb1cb81e55f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 8 Nov 2015 19:01:37 +0600
Subject: [PATCH 211/415] [extremetube] Fix extraction (Closes #7163)
---
youtube_dl/extractor/extremetube.py | 45 +++++++++++++++++++----------
1 file changed, 29 insertions(+), 16 deletions(-)
diff --git a/youtube_dl/extractor/extremetube.py b/youtube_dl/extractor/extremetube.py
index c826a5404..3e11e3299 100644
--- a/youtube_dl/extractor/extremetube.py
+++ b/youtube_dl/extractor/extremetube.py
@@ -3,12 +3,9 @@ from __future__ import unicode_literals
import re
from .common import InfoExtractor
-from ..compat import (
- compat_parse_qs,
- compat_urllib_request,
-)
+from ..compat import compat_urllib_request
from ..utils import (
- qualities,
+ int_or_none,
str_to_int,
)
@@ -49,20 +46,36 @@ class ExtremeTubeIE(InfoExtractor):
r'Views:\s*</strong>\s*<span>([\d,\.]+)</span>',
webpage, 'view count', fatal=False))
- flash_vars = compat_parse_qs(self._search_regex(
- r'<param[^>]+?name="flashvars"[^>]+?value="([^"]+)"', webpage, 'flash vars'))
+ flash_vars = self._parse_json(
+ self._search_regex(
+ r'var\s+flashvars\s*=\s*({.+?});', webpage, 'flash vars'),
+ video_id)
formats = []
- quality = qualities(['180p', '240p', '360p', '480p', '720p', '1080p'])
- for k, vals in flash_vars.items():
- m = re.match(r'quality_(?P<quality>[0-9]+p)$', k)
- if m is not None:
- formats.append({
- 'format_id': m.group('quality'),
- 'quality': quality(m.group('quality')),
- 'url': vals[0],
+ for quality_key, video_url in flash_vars.items():
+ height = int_or_none(self._search_regex(
+ r'quality_(\d+)[pP]$', quality_key, 'height', default=None))
+ if not height:
+ continue
+ f = {
+ 'url': video_url,
+ }
+ mobj = re.search(
+ r'/(?P<height>\d{3,4})[pP]_(?P<bitrate>\d+)[kK]_\d+', video_url)
+ if mobj:
+ height = int(mobj.group('height'))
+ bitrate = int(mobj.group('bitrate'))
+ f.update({
+ 'format_id': '%dp-%dk' % (height, bitrate),
+ 'height': height,
+ 'tbr': bitrate,
})
-
+ else:
+ f.update({
+ 'format_id': '%dp' % height,
+ 'height': height,
+ })
+ formats.append(f)
self._sort_formats(formats)
return {
From cc8034cc4c52fcbfaf9f8edf34d562c481860193 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 8 Nov 2015 19:14:39 +0600
Subject: [PATCH 212/415] [extremetube] Modernize
---
youtube_dl/extractor/extremetube.py | 14 +++++++++-----
1 file changed, 9 insertions(+), 5 deletions(-)
diff --git a/youtube_dl/extractor/extremetube.py b/youtube_dl/extractor/extremetube.py
index 3e11e3299..c5677c82b 100644
--- a/youtube_dl/extractor/extremetube.py
+++ b/youtube_dl/extractor/extremetube.py
@@ -11,12 +11,12 @@ from ..utils import (
class ExtremeTubeIE(InfoExtractor):
- _VALID_URL = r'https?://(?:www\.)?(?P<url>extremetube\.com/.*?video/.+?(?P<id>[0-9]+))(?:[/?&]|$)'
+ _VALID_URL = r'https?://(?:www\.)?extremetube\.com/(?:[^/]+/)?video/(?P<id>[^/#?&]+)'
_TESTS = [{
'url': 'http://www.extremetube.com/video/music-video-14-british-euro-brit-european-cumshots-swallow-652431',
'md5': '344d0c6d50e2f16b06e49ca011d8ac69',
'info_dict': {
- 'id': '652431',
+ 'id': 'music-video-14-british-euro-brit-european-cumshots-swallow-652431',
'ext': 'mp4',
'title': 'Music Video 14 british euro brit european cumshots swallow',
'uploader': 'unknown',
@@ -26,12 +26,16 @@ class ExtremeTubeIE(InfoExtractor):
}, {
'url': 'http://www.extremetube.com/gay/video/abcde-1234',
'only_matching': True,
+ }, {
+ 'url': 'http://www.extremetube.com/video/latina-slut-fucked-by-fat-black-dick',
+ 'only_matching': True,
+ }, {
+ 'url': 'http://www.extremetube.com/video/652431',
+ 'only_matching': True,
}]
def _real_extract(self, url):
- mobj = re.match(self._VALID_URL, url)
- video_id = mobj.group('id')
- url = 'http://www.' + mobj.group('url')
+ video_id = self._match_id(url)
req = compat_urllib_request.Request(url)
req.add_header('Cookie', 'age_verified=1')
From f09a767d3198823e5c0ac187a91284c8d2736eb6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 8 Nov 2015 19:19:13 +0600
Subject: [PATCH 213/415] [mit] Allow external embeds (Closes #7406)
---
youtube_dl/extractor/mit.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/mit.py b/youtube_dl/extractor/mit.py
index f088ab9e2..29ca45778 100644
--- a/youtube_dl/extractor/mit.py
+++ b/youtube_dl/extractor/mit.py
@@ -86,7 +86,7 @@ class MITIE(TechTVMITIE):
webpage = self._download_webpage(url, page_title)
embed_url = self._search_regex(
r'<iframe .*?src="(.+?)"', webpage, 'embed url')
- return self.url_result(embed_url, ie='TechTVMIT')
+ return self.url_result(embed_url)
class OCWMITIE(InfoExtractor):
From 4f5cdf7c9b21ba00822ad0134c5883f6ca366674 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 9 Nov 2015 01:48:46 +0600
Subject: [PATCH 214/415] [cmt] Extend _VALID_URL to support shows (Closes
#7407)
---
youtube_dl/extractor/cmt.py | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/cmt.py b/youtube_dl/extractor/cmt.py
index e96c59f71..f1311b14f 100644
--- a/youtube_dl/extractor/cmt.py
+++ b/youtube_dl/extractor/cmt.py
@@ -4,7 +4,7 @@ from .mtv import MTVIE
class CMTIE(MTVIE):
IE_NAME = 'cmt.com'
- _VALID_URL = r'https?://www\.cmt\.com/videos/.+?/(?P<videoid>[^/]+)\.jhtml'
+ _VALID_URL = r'https?://www\.cmt\.com/(?:videos|shows)/(?:[^/]+/)*(?P<videoid>\d+)'
_FEED_URL = 'http://www.cmt.com/sitewide/apps/player/embed/rss/'
_TESTS = [{
@@ -16,4 +16,7 @@ class CMTIE(MTVIE):
'title': 'Garth Brooks - "The Call (featuring Trisha Yearwood)"',
'description': 'Blame It All On My Roots',
},
+ }, {
+ 'url': 'http://www.cmt.com/shows/party-down-south/party-down-south-ep-407-gone-girl/1738172/playlist/#id=1738172',
+ 'only_matching': True,
}]
From 1c31a5b0e029ae47415f7df7bea30e2b2b5df64f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 9 Nov 2015 20:49:06 +0600
Subject: [PATCH 215/415] [generic] Improve kaltura embed detection (Closes
#7409)
---
youtube_dl/extractor/generic.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index ee5419f51..5f61ab26f 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1673,7 +1673,7 @@ class GenericIE(InfoExtractor):
# Look for Kaltura embeds
mobj = (re.search(r"(?s)kWidget\.(?:thumb)?[Ee]mbed\(\{.*?'wid'\s*:\s*'_?(?P<partner_id>[^']+)',.*?'entry_id'\s*:\s*'(?P<id>[^']+)',", webpage) or
- re.search(r'(?s)(["\'])(?:https?:)?//cdnapisec\.kaltura\.com/.*?(?:p|partner_id)/(?P<partner_id>\d+).*?\1.*?entry_id\s*:\s*(["\'])(?P<id>[^\2]+?)\2', webpage))
+ re.search(r'(?s)(?P<q1>["\'])(?:https?:)?//cdnapi(?:sec)?\.kaltura\.com/.*?(?:p|partner_id)/(?P<partner_id>\d+).*?(?P=q1).*?entry_?[Ii]d\s*:\s*(?P<q2>["\'])(?P<id>.+?)(?P=q2)', webpage))
if mobj is not None:
return self.url_result('kaltura:%(partner_id)s:%(id)s' % mobj.groupdict(), 'Kaltura')
From 6a5d6de1e36f5278b48b7dcc92a8e15adbe541ad Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 9 Nov 2015 20:49:42 +0600
Subject: [PATCH 216/415] [generic] Improve kaltura embed detection (2)
---
youtube_dl/extractor/generic.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 5f61ab26f..d0b486d2a 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1672,7 +1672,7 @@ class GenericIE(InfoExtractor):
return self.url_result(mobj.group('url'), 'Zapiks')
# Look for Kaltura embeds
- mobj = (re.search(r"(?s)kWidget\.(?:thumb)?[Ee]mbed\(\{.*?'wid'\s*:\s*'_?(?P<partner_id>[^']+)',.*?'entry_id'\s*:\s*'(?P<id>[^']+)',", webpage) or
+ mobj = (re.search(r"(?s)kWidget\.(?:thumb)?[Ee]mbed\(\{.*?'wid'\s*:\s*'_?(?P<partner_id>[^']+)',.*?'entry_?[Ii]d'\s*:\s*'(?P<id>[^']+)',", webpage) or
re.search(r'(?s)(?P<q1>["\'])(?:https?:)?//cdnapi(?:sec)?\.kaltura\.com/.*?(?:p|partner_id)/(?P<partner_id>\d+).*?(?P=q1).*?entry_?[Ii]d\s*:\s*(?P<q2>["\'])(?P<id>.+?)(?P=q2)', webpage))
if mobj is not None:
return self.url_result('kaltura:%(partner_id)s:%(id)s' % mobj.groupdict(), 'Kaltura')
From b25f7533975d7463cc772e9d106802ef29252796 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 9 Nov 2015 20:50:43 +0600
Subject: [PATCH 217/415] [kaltura] Relax _VALID_URL
---
youtube_dl/extractor/kaltura.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/kaltura.py b/youtube_dl/extractor/kaltura.py
index 3dca0e566..0dcd6cd05 100644
--- a/youtube_dl/extractor/kaltura.py
+++ b/youtube_dl/extractor/kaltura.py
@@ -16,7 +16,7 @@ class KalturaIE(InfoExtractor):
(?:
kaltura:(?P<partner_id_s>\d+):(?P<id_s>[0-9a-z_]+)|
https?://
- (:?(?:www|cdnapisec)\.)?kaltura\.com/
+ (:?(?:www|cdnapi(?:sec)?)\.)?kaltura\.com/
(?:
(?:
# flash player
From ff29bf81f852941cd7bcef1c051c77eccd4857bc Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 10 Nov 2015 12:54:02 +0800
Subject: [PATCH 218/415] [jsinterp] Support alternative function definition
form
---
test/test_jsinterp.py | 3 +++
youtube_dl/jsinterp.py | 4 ++--
2 files changed, 5 insertions(+), 2 deletions(-)
diff --git a/test/test_jsinterp.py b/test/test_jsinterp.py
index fc73e5dc2..63c350b8f 100644
--- a/test/test_jsinterp.py
+++ b/test/test_jsinterp.py
@@ -19,6 +19,9 @@ class TestJSInterpreter(unittest.TestCase):
jsi = JSInterpreter('function x3(){return 42;}')
self.assertEqual(jsi.call_function('x3'), 42)
+ jsi = JSInterpreter('var x5 = function(){return 42;}')
+ self.assertEqual(jsi.call_function('x5'), 42)
+
def test_calc(self):
jsi = JSInterpreter('function x4(a){return 2*a+1;}')
self.assertEqual(jsi.call_function('x4', 3), 7)
diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index 0e0c7d90d..9bc855144 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -232,10 +232,10 @@ class JSInterpreter(object):
def extract_function(self, funcname):
func_m = re.search(
r'''(?x)
- (?:function\s+%s|[{;]%s\s*=\s*function)\s*
+ (?:function\s+%s|[{;]%s\s*=\s*function|var\s+%s\s*=\s*function)\s*
\((?P<args>[^)]*)\)\s*
\{(?P<code>[^}]+)\}''' % (
- re.escape(funcname), re.escape(funcname)),
+ re.escape(funcname), re.escape(funcname), re.escape(funcname)),
self.code)
if func_m is None:
raise ExtractorError('Could not find JS function %r' % funcname)
From 50f84a9ae171233c08ada41e03f6555c5ed95236 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 10 Nov 2015 12:55:01 +0800
Subject: [PATCH 219/415] [youtube] Support new base.js html5 player
---
youtube_dl/extractor/youtube.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index e2a43299f..687e0b4db 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -703,7 +703,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
def _extract_signature_function(self, video_id, player_url, example_sig):
id_m = re.match(
- r'.*?-(?P<id>[a-zA-Z0-9_-]+)(?:/watch_as3|/html5player(?:-new)?)?\.(?P<ext>[a-z]+)$',
+ r'.*?-(?P<id>[a-zA-Z0-9_-]+)(?:/watch_as3|/html5player(?:-new)?|/base)?\.(?P<ext>[a-z]+)$',
player_url)
if not id_m:
raise ExtractorError('Cannot identify player %r' % player_url)
@@ -1343,7 +1343,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
player_desc = 'flash player %s' % player_version
else:
player_version = self._search_regex(
- r'html5player-([^/]+?)(?:/html5player(?:-new)?)?\.js',
+ [r'html5player-([^/]+?)(?:/html5player(?:-new)?)?\.js', r'(?:www|player)-([^/]+)/base\.js'],
player_url,
'html5 player', fatal=False)
player_desc = 'html5 player %s' % player_version
From 37ca7b22b5521d7797ffefcd1456b4eb31320460 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 10 Nov 2015 11:39:06 +0100
Subject: [PATCH 220/415] release 2015.11.10
---
docs/supportedsites.md | 4 +++-
youtube_dl/version.py | 2 +-
2 files changed, 4 insertions(+), 2 deletions(-)
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 805af14a0..a9820c1f5 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -123,6 +123,7 @@
- **DctpTv**
- **DeezerPlaylist**
- **defense.gouv.fr**
+ - **democracynow**
- **DHM**: Filmarchiv - Deutsches Historisches Museum
- **Discovery**
- **Dotsub**
@@ -195,6 +196,7 @@
- **Giga**
- **Glide**: Glide mobile video messages (glide.me)
- **Globo**
+ - **GloboArticle**
- **GodTube**
- **GoldenMoustache**
- **Golem**
@@ -617,7 +619,6 @@
- **video.mit.edu**
- **VideoDetective**
- **videofy.me**
- - **videolectures.net**
- **VideoMega**
- **VideoPremium**
- **VideoTt**: video.tt - Your True Tube
@@ -627,6 +628,7 @@
- **vier**
- **vier:videos**
- **Viewster**
+ - **Viidea**
- **viki**
- **viki:channel**
- **vimeo**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 6ef482b78..b3d254005 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
from __future__ import unicode_literals
-__version__ = '2015.11.02'
+__version__ = '2015.11.10'
From d3d3e2e3aac13c3c6fbae0fcfeaa41ce5ee9144b Mon Sep 17 00:00:00 2001
From: David Ben Zakai <david.benzakai@gmail.com>
Date: Tue, 10 Nov 2015 16:31:31 +0200
Subject: [PATCH 221/415] Adding proxy to update procedure
---
youtube_dl/YoutubeDL.py | 3 +++
youtube_dl/__init__.py | 2 +-
youtube_dl/update.py | 7 ++-----
3 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 1783ce01b..5a0cc3f9a 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1994,6 +1994,9 @@ class YoutubeDL(object):
encoding = preferredencoding()
return encoding
+ def get_opener(self):
+ return self._opener
+
def _write_thumbnails(self, info_dict, filename):
if self.params.get('writethumbnail', False):
thumbnails = info_dict.get('thumbnails')
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 5e2ed4d4b..760128546 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -377,7 +377,7 @@ def _real_main(argv=None):
with YoutubeDL(ydl_opts) as ydl:
# Update version
if opts.update_self:
- update_self(ydl.to_screen, opts.verbose)
+ update_self(ydl.to_screen, opts.verbose, ydl.get_opener())
# Remove cache dir
if opts.rm_cachedir:
diff --git a/youtube_dl/update.py b/youtube_dl/update.py
index fc7ac8305..04bf0939e 100644
--- a/youtube_dl/update.py
+++ b/youtube_dl/update.py
@@ -13,7 +13,7 @@ from .compat import (
compat_str,
compat_urllib_request,
)
-from .utils import make_HTTPS_handler
+
from .version import __version__
@@ -47,7 +47,7 @@ def rsa_verify(message, signature, key):
return True
-def update_self(to_screen, verbose):
+def update_self(to_screen, verbose, opener):
"""Update the program file with the latest version from the repository"""
UPDATE_URL = "https://rg3.github.io/youtube-dl/update/"
@@ -59,9 +59,6 @@ def update_self(to_screen, verbose):
to_screen('It looks like you installed youtube-dl with a package manager, pip, setup.py or a tarball. Please use that to update.')
return
- https_handler = make_HTTPS_handler({})
- opener = compat_urllib_request.build_opener(https_handler)
-
# Check if there is a new version
try:
newversion = opener.open(VERSION_URL).read().decode('utf-8').strip()
From 90bb5667bf32574caa7c01f66b3ca8ad2ab77e81 Mon Sep 17 00:00:00 2001
From: David Ben Zakai <david.benzakai@gmail.com>
Date: Tue, 10 Nov 2015 17:15:23 +0200
Subject: [PATCH 222/415] Using internal opener
---
youtube_dl/YoutubeDL.py | 3 ---
youtube_dl/__init__.py | 2 +-
2 files changed, 1 insertion(+), 4 deletions(-)
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 5a0cc3f9a..1783ce01b 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1994,9 +1994,6 @@ class YoutubeDL(object):
encoding = preferredencoding()
return encoding
- def get_opener(self):
- return self._opener
-
def _write_thumbnails(self, info_dict, filename):
if self.params.get('writethumbnail', False):
thumbnails = info_dict.get('thumbnails')
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 760128546..9f131f5db 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -377,7 +377,7 @@ def _real_main(argv=None):
with YoutubeDL(ydl_opts) as ydl:
# Update version
if opts.update_self:
- update_self(ydl.to_screen, opts.verbose, ydl.get_opener())
+ update_self(ydl.to_screen, opts.verbose, ydl._opener)
# Remove cache dir
if opts.rm_cachedir:
From 9b738b2caa20e13d0a56d05aa56af44be75476bc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 10 Nov 2015 21:32:54 +0600
Subject: [PATCH 223/415] [funnyordie] Fix extraction and extract m3u8 formats
---
youtube_dl/extractor/funnyordie.py | 15 ++++++++++++---
1 file changed, 12 insertions(+), 3 deletions(-)
diff --git a/youtube_dl/extractor/funnyordie.py b/youtube_dl/extractor/funnyordie.py
index f5f13689c..7f21d7410 100644
--- a/youtube_dl/extractor/funnyordie.py
+++ b/youtube_dl/extractor/funnyordie.py
@@ -45,11 +45,20 @@ class FunnyOrDieIE(InfoExtractor):
links.sort(key=lambda link: 1 if link[1] == 'mp4' else 0)
- bitrates = self._html_search_regex(r'<source src="[^"]+/v,((?:\d+,)+)\.mp4\.csmil', webpage, 'video bitrates')
- bitrates = [int(b) for b in bitrates.rstrip(',').split(',')]
- bitrates.sort()
+ m3u8_url = self._search_regex(
+ r'<source[^>]+src=(["\'])(?P<url>.+?/master\.m3u8)\1',
+ webpage, 'm3u8 url', default=None, group='url')
formats = []
+
+ m3u8_formats = self._extract_m3u8_formats(
+ m3u8_url, video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False)
+ if m3u8_formats:
+ formats.extend(m3u8_formats)
+
+ bitrates = [int(bitrate) for bitrate in re.findall(r'[,/]v(\d+)[,/]', m3u8_url)]
+ bitrates.sort()
+
for bitrate in bitrates:
for link in links:
formats.append({
From a625e5654365a8a246fa5fc5f5d347adc67c5617 Mon Sep 17 00:00:00 2001
From: Marco Ferragina <marco.ferragina@gmail.com>
Date: Wed, 14 Oct 2015 11:11:52 +0200
Subject: [PATCH 224/415] [vidto] Add extractor
---
docs/supportedsites.md | 1 +
youtube_dl/extractor/__init__.py | 1 +
youtube_dl/extractor/vidto.py | 82 ++++++++++++++++++++++++++++++++
3 files changed, 84 insertions(+)
create mode 100644 youtube_dl/extractor/vidto.py
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index a9820c1f5..510d4d627 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -624,6 +624,7 @@
- **VideoTt**: video.tt - Your True Tube
- **videoweed**: VideoWeed
- **Vidme**
+ - **vidto**: VidTo.me
- **Vidzi**
- **vier**
- **vier:videos**
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 0a90da73c..ef6c7f0de 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -730,6 +730,7 @@ from .videopremium import VideoPremiumIE
from .videott import VideoTtIE
from .videoweed import VideoWeedIE
from .vidme import VidmeIE
+from .vidto import VidtoIE
from .vidzi import VidziIE
from .vier import VierIE, VierVideosIE
from .viewster import ViewsterIE
diff --git a/youtube_dl/extractor/vidto.py b/youtube_dl/extractor/vidto.py
new file mode 100644
index 000000000..3cc585471
--- /dev/null
+++ b/youtube_dl/extractor/vidto.py
@@ -0,0 +1,82 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+import sys
+from .common import InfoExtractor
+import time
+
+from ..utils import (
+ encode_dict,
+)
+from ..compat import (
+ compat_chr,
+ compat_parse_qs,
+ compat_urllib_parse,
+ compat_urllib_parse_unquote,
+ compat_urllib_parse_unquote_plus,
+ compat_urllib_parse_urlparse,
+ compat_urllib_request,
+ compat_urlparse,
+ compat_str,
+)
+
+
+class VidtoIE(InfoExtractor):
+ IE_NAME = 'vidto'
+ IE_DESC = 'VidTo.me'
+ _VALID_URL = r'https?://(?:www\.)?vidto\.me/(?P<id>[0-9a-zA-Z]+)\.html'
+ _HOST = 'vidto.me'
+ _TEST = {
+ 'url': 'http://vidto.me/ku5glz52nqe1.html',
+ 'info_dict': {
+ 'id': 'ku5glz52nqe1',
+ 'ext': 'mp4',
+ 'title': 'test.mp4'
+ }
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+
+ page = self._download_webpage(
+ 'http://%s/%s.html' % (self._HOST, video_id), video_id, 'Downloading video page')
+ hash_regex = r'<input type="hidden" name="hash" value="(.*)">'
+ hash_value = self._search_regex(hash_regex, page, 'hash', fatal=True)
+ title_regex = r'<input type="hidden" name="fname" value="(.*)">'
+ title = self._search_regex(title_regex, page, 'title', fatal=False)
+ id_regex = r'<input type="hidden" name="id" value="(.*)">'
+ id_value = self._search_regex(id_regex, page, 'id', fatal=True)
+ cookies = self._get_cookies('http://%s/%s.html' % (self._HOST, video_id))
+
+
+ form_str = {
+ 'op': 'download1',
+ 'imhuman': 'Proceed to video',
+ 'usr_login': '',
+ 'id': id_value,
+ 'fname': title,
+ 'referer': '',
+ 'hash': hash_value,
+ }
+ post_data = compat_urllib_parse.urlencode(encode_dict(form_str)).encode('ascii')
+ req = compat_urllib_request.Request(url, post_data)
+ req.add_header('Content-type', 'application/x-www-form-urlencoded')
+ for key, morsel in cookies.iteritems():
+ req.add_header('Cookie', '%s=%s' % (morsel.key, morsel.value))
+
+ print("Waiting for countdown...")
+ time.sleep(7)
+ post_result = self._download_webpage(
+ req, None,
+ note='Proceed to video...', errnote='unable to proceed', fatal=True)
+
+ file_link_regex = r'file_link ?= ?\'(https?:\/\/[0-9a-zA-z.\/\-_]+)'
+ file_link = self._search_regex(file_link_regex, post_result, 'file_link', fatal=True)
+
+ return {
+ 'id': video_id,
+ 'url': file_link,
+ 'title': title,
+ }
From 42fc93c70928d34d46ceda644eb870e86daa8393 Mon Sep 17 00:00:00 2001
From: Marco Ferragina <marco.ferragina@gmail.com>
Date: Sat, 17 Oct 2015 19:35:41 +0200
Subject: [PATCH 225/415] vidto extractor: code cleanup
---
docs/supportedsites.md | 1 -
youtube_dl/extractor/vidto.py | 32 ++++++++++++--------------------
2 files changed, 12 insertions(+), 21 deletions(-)
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 510d4d627..a9820c1f5 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -624,7 +624,6 @@
- **VideoTt**: video.tt - Your True Tube
- **videoweed**: VideoWeed
- **Vidme**
- - **vidto**: VidTo.me
- **Vidzi**
- **vier**
- **vier:videos**
diff --git a/youtube_dl/extractor/vidto.py b/youtube_dl/extractor/vidto.py
index 3cc585471..391f400fc 100644
--- a/youtube_dl/extractor/vidto.py
+++ b/youtube_dl/extractor/vidto.py
@@ -1,24 +1,14 @@
# coding: utf-8
from __future__ import unicode_literals
-import re
-import sys
from .common import InfoExtractor
+import re
import time
-from ..utils import (
- encode_dict,
-)
+from ..utils import encode_dict
from ..compat import (
- compat_chr,
- compat_parse_qs,
- compat_urllib_parse,
- compat_urllib_parse_unquote,
- compat_urllib_parse_unquote_plus,
- compat_urllib_parse_urlparse,
compat_urllib_request,
- compat_urlparse,
- compat_str,
+ compat_urllib_parse
)
@@ -37,8 +27,7 @@ class VidtoIE(InfoExtractor):
}
def _real_extract(self, url):
- mobj = re.match(self._VALID_URL, url)
- video_id = mobj.group('id')
+ video_id = self._match_id(url)
page = self._download_webpage(
'http://%s/%s.html' % (self._HOST, video_id), video_id, 'Downloading video page')
@@ -63,16 +52,19 @@ class VidtoIE(InfoExtractor):
post_data = compat_urllib_parse.urlencode(encode_dict(form_str)).encode('ascii')
req = compat_urllib_request.Request(url, post_data)
req.add_header('Content-type', 'application/x-www-form-urlencoded')
- for key, morsel in cookies.iteritems():
- req.add_header('Cookie', '%s=%s' % (morsel.key, morsel.value))
+ cookie_string = ""
+ for key in cookies.keys():
+ cookie_string += "%s=%s;" % (key, cookies[key].value)
- print("Waiting for countdown...")
+ req.add_header('Cookie', '%s' % cookie_string)
+
+ self.to_screen("Waiting for countdown...")
time.sleep(7)
post_result = self._download_webpage(
- req, None,
+ req, video_id,
note='Proceed to video...', errnote='unable to proceed', fatal=True)
- file_link_regex = r'file_link ?= ?\'(https?:\/\/[0-9a-zA-z.\/\-_]+)'
+ file_link_regex = r'file_link\s*=\s*\'(https?:\/\/[0-9a-zA-z.\/\-_]+)'
file_link = self._search_regex(file_link_regex, post_result, 'file_link', fatal=True)
return {
From 37120974dc7e3e7febd40b652fc7f9e1748e2ad3 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 11 Nov 2015 02:02:46 +0800
Subject: [PATCH 226/415] [vidto] PEP8
---
youtube_dl/extractor/vidto.py | 2 --
1 file changed, 2 deletions(-)
diff --git a/youtube_dl/extractor/vidto.py b/youtube_dl/extractor/vidto.py
index 391f400fc..d10002534 100644
--- a/youtube_dl/extractor/vidto.py
+++ b/youtube_dl/extractor/vidto.py
@@ -2,7 +2,6 @@
from __future__ import unicode_literals
from .common import InfoExtractor
-import re
import time
from ..utils import encode_dict
@@ -39,7 +38,6 @@ class VidtoIE(InfoExtractor):
id_value = self._search_regex(id_regex, page, 'id', fatal=True)
cookies = self._get_cookies('http://%s/%s.html' % (self._HOST, video_id))
-
form_str = {
'op': 'download1',
'imhuman': 'Proceed to video',
From d8b7e80d29a67835555a6317b2c528b911077d47 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9my=20L=C3=A9one?= <remy.leone@gmail.com>
Date: Wed, 11 Nov 2015 12:00:31 +0100
Subject: [PATCH 227/415] Remove duplicate key
---
youtube_dl/extractor/wsj.py | 1 -
1 file changed, 1 deletion(-)
diff --git a/youtube_dl/extractor/wsj.py b/youtube_dl/extractor/wsj.py
index 2ddf29a69..5a897371d 100644
--- a/youtube_dl/extractor/wsj.py
+++ b/youtube_dl/extractor/wsj.py
@@ -84,6 +84,5 @@ class WSJIE(InfoExtractor):
'duration': duration,
'upload_date': upload_date,
'title': title,
- 'formats': formats,
'categories': categories,
}
From 990e6e8fa346224967f123a50146377088acbd75 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 11 Nov 2015 20:13:03 +0800
Subject: [PATCH 228/415] [vidto] Minor fixes
1. import order
2. fatal is already True in helper functions
---
youtube_dl/extractor/vidto.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/youtube_dl/extractor/vidto.py b/youtube_dl/extractor/vidto.py
index d10002534..45b21e2a9 100644
--- a/youtube_dl/extractor/vidto.py
+++ b/youtube_dl/extractor/vidto.py
@@ -1,9 +1,9 @@
# coding: utf-8
from __future__ import unicode_literals
-from .common import InfoExtractor
import time
+from .common import InfoExtractor
from ..utils import encode_dict
from ..compat import (
compat_urllib_request,
@@ -60,10 +60,10 @@ class VidtoIE(InfoExtractor):
time.sleep(7)
post_result = self._download_webpage(
req, video_id,
- note='Proceed to video...', errnote='unable to proceed', fatal=True)
+ note='Proceed to video...', errnote='unable to proceed')
file_link_regex = r'file_link\s*=\s*\'(https?:\/\/[0-9a-zA-z.\/\-_]+)'
- file_link = self._search_regex(file_link_regex, post_result, 'file_link', fatal=True)
+ file_link = self._search_regex(file_link_regex, post_result, 'file_link')
return {
'id': video_id,
From 6abce58a1226a5c359e4eb0cb228353fa7cb2aaf Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 11 Nov 2015 20:16:18 +0800
Subject: [PATCH 229/415] Credit @sieben for fixing wsj extractor
---
AUTHORS | 1 +
1 file changed, 1 insertion(+)
diff --git a/AUTHORS b/AUTHORS
index cc552bcb2..4cdbf4a5f 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -144,3 +144,4 @@ Lee Jenkins
Anssi Hannula
Lukáš Lalinský
Qijiang Fan
+Rémy Léone
From 2eb99a4b98ecece86a8d0c692066644a855f8524 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 11 Nov 2015 21:00:23 +0600
Subject: [PATCH 230/415] [nowvideo] Replace main host to resolvable one
---
youtube_dl/extractor/nowvideo.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/youtube_dl/extractor/nowvideo.py b/youtube_dl/extractor/nowvideo.py
index 17baa9679..57ee3d366 100644
--- a/youtube_dl/extractor/nowvideo.py
+++ b/youtube_dl/extractor/nowvideo.py
@@ -7,9 +7,9 @@ class NowVideoIE(NovaMovIE):
IE_NAME = 'nowvideo'
IE_DESC = 'NowVideo'
- _VALID_URL = NovaMovIE._VALID_URL_TEMPLATE % {'host': 'nowvideo\.(?:ch|ec|sx|eu|at|ag|co|li)'}
+ _VALID_URL = NovaMovIE._VALID_URL_TEMPLATE % {'host': 'nowvideo\.(?:to|ch|ec|sx|eu|at|ag|co|li)'}
- _HOST = 'www.nowvideo.ch'
+ _HOST = 'www.nowvideo.to'
_FILE_DELETED_REGEX = r'>This file no longer exists on our servers.<'
_FILEKEY_REGEX = r'var fkzd="([^"]+)";'
From 82393e2bb2c499cad285beb07c222f501302d830 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 11 Nov 2015 21:02:05 +0600
Subject: [PATCH 231/415] [novamov] Follow continue-to-the-video button if any
(Closes #7330)
---
youtube_dl/extractor/novamov.py | 38 +++++++++++++++++++++++++++------
1 file changed, 32 insertions(+), 6 deletions(-)
diff --git a/youtube_dl/extractor/novamov.py b/youtube_dl/extractor/novamov.py
index 04d779890..e0bf6d1bc 100644
--- a/youtube_dl/extractor/novamov.py
+++ b/youtube_dl/extractor/novamov.py
@@ -4,10 +4,14 @@ import re
from .common import InfoExtractor
from ..compat import (
+ compat_urllib_request,
compat_urlparse,
)
from ..utils import (
ExtractorError,
+ NO_DEFAULT,
+ encode_dict,
+ urlencode_postdata,
)
@@ -41,16 +45,38 @@ class NovaMovIE(InfoExtractor):
mobj = re.match(self._VALID_URL, url)
video_id = mobj.group('id')
- page = self._download_webpage(
- 'http://%s/video/%s' % (self._HOST, video_id), video_id, 'Downloading video page')
+ url = 'http://%s/video/%s' % (self._HOST, video_id)
- if re.search(self._FILE_DELETED_REGEX, page) is not None:
+ webpage = self._download_webpage(
+ url, video_id, 'Downloading video page')
+
+ if re.search(self._FILE_DELETED_REGEX, webpage) is not None:
raise ExtractorError('Video %s does not exist' % video_id, expected=True)
- filekey = self._search_regex(self._FILEKEY_REGEX, page, 'filekey')
+ def extract_filekey(default=NO_DEFAULT):
+ return self._search_regex(
+ self._FILEKEY_REGEX, webpage, 'filekey', default=default)
- title = self._html_search_regex(self._TITLE_REGEX, page, 'title', fatal=False)
- description = self._html_search_regex(self._DESCRIPTION_REGEX, page, 'description', default='', fatal=False)
+ filekey = extract_filekey(default=None)
+
+ if not filekey:
+ fields = self._hidden_inputs(webpage)
+ post_url = self._search_regex(
+ r'<form[^>]+action=(["\'])(?P<url>.+?)\1', webpage,
+ 'post url', default=url, group='url')
+ if not post_url.startswith('http'):
+ post_url = compat_urlparse.urljoin(url, post_url)
+ request = compat_urllib_request.Request(
+ post_url, urlencode_postdata(encode_dict(fields)))
+ request.add_header('Content-Type', 'application/x-www-form-urlencoded')
+ request.add_header('Referer', post_url)
+ webpage = self._download_webpage(
+ request, video_id, 'Downloading continue to the video page')
+
+ filekey = extract_filekey()
+
+ title = self._html_search_regex(self._TITLE_REGEX, webpage, 'title', fatal=False)
+ description = self._html_search_regex(self._DESCRIPTION_REGEX, webpage, 'description', default='', fatal=False)
api_response = self._download_webpage(
'http://%s/api/player.api.php?key=%s&file=%s' % (self._HOST, filekey, video_id), video_id,
From 8b8a39e279b02926197230024e64a50d7995b95d Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 11 Nov 2015 23:17:59 +0800
Subject: [PATCH 232/415] [vidto] Several simplifications and improvements
1. Use InfoExtractor._hidden_inputs
2. Fetch title from <title> tag
3. Cookies are preserved automatically
4. Use single quotes everywhere
5. Do not declare variables for one-time use only
---
youtube_dl/extractor/vidto.py | 51 +++++++++++++----------------------
1 file changed, 18 insertions(+), 33 deletions(-)
diff --git a/youtube_dl/extractor/vidto.py b/youtube_dl/extractor/vidto.py
index 45b21e2a9..5ebf0f3d3 100644
--- a/youtube_dl/extractor/vidto.py
+++ b/youtube_dl/extractor/vidto.py
@@ -4,24 +4,24 @@ from __future__ import unicode_literals
import time
from .common import InfoExtractor
-from ..utils import encode_dict
-from ..compat import (
- compat_urllib_request,
- compat_urllib_parse
+from ..utils import (
+ encode_dict,
+ remove_end,
+ urlencode_postdata,
)
+from ..compat import compat_urllib_request
class VidtoIE(InfoExtractor):
IE_NAME = 'vidto'
IE_DESC = 'VidTo.me'
_VALID_URL = r'https?://(?:www\.)?vidto\.me/(?P<id>[0-9a-zA-Z]+)\.html'
- _HOST = 'vidto.me'
_TEST = {
'url': 'http://vidto.me/ku5glz52nqe1.html',
'info_dict': {
'id': 'ku5glz52nqe1',
'ext': 'mp4',
- 'title': 'test.mp4'
+ 'title': 'test'
}
}
@@ -29,41 +29,26 @@ class VidtoIE(InfoExtractor):
video_id = self._match_id(url)
page = self._download_webpage(
- 'http://%s/%s.html' % (self._HOST, video_id), video_id, 'Downloading video page')
- hash_regex = r'<input type="hidden" name="hash" value="(.*)">'
- hash_value = self._search_regex(hash_regex, page, 'hash', fatal=True)
- title_regex = r'<input type="hidden" name="fname" value="(.*)">'
- title = self._search_regex(title_regex, page, 'title', fatal=False)
- id_regex = r'<input type="hidden" name="id" value="(.*)">'
- id_value = self._search_regex(id_regex, page, 'id', fatal=True)
- cookies = self._get_cookies('http://%s/%s.html' % (self._HOST, video_id))
+ 'http://vidto.me/%s.html' % video_id, video_id, 'Downloading video page')
- form_str = {
- 'op': 'download1',
- 'imhuman': 'Proceed to video',
- 'usr_login': '',
- 'id': id_value,
- 'fname': title,
- 'referer': '',
- 'hash': hash_value,
- }
- post_data = compat_urllib_parse.urlencode(encode_dict(form_str)).encode('ascii')
- req = compat_urllib_request.Request(url, post_data)
- req.add_header('Content-type', 'application/x-www-form-urlencoded')
- cookie_string = ""
- for key in cookies.keys():
- cookie_string += "%s=%s;" % (key, cookies[key].value)
+ title = remove_end(self._html_search_regex(
+ r'<Title>\s*([^<]+)\s*</Title>', page, 'title'), ' - Vidto')
- req.add_header('Cookie', '%s' % cookie_string)
+ hidden_fields = self._hidden_inputs(page)
- self.to_screen("Waiting for countdown...")
+ self.to_screen('Waiting for countdown...')
time.sleep(7)
+
+ req = compat_urllib_request.Request(
+ url, urlencode_postdata(encode_dict(hidden_fields)))
+ req.add_header('Content-type', 'application/x-www-form-urlencoded')
+
post_result = self._download_webpage(
req, video_id,
note='Proceed to video...', errnote='unable to proceed')
- file_link_regex = r'file_link\s*=\s*\'(https?:\/\/[0-9a-zA-z.\/\-_]+)'
- file_link = self._search_regex(file_link_regex, post_result, 'file_link')
+ file_link = self._search_regex(
+ r'file_link\s*=\s*\'(https?:\/\/[0-9a-zA-z.\/\-_]+)', post_result, 'file_link')
return {
'id': video_id,
From 3d9c4bf09a57871f0030d6b6022b863d66616b8b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 11 Nov 2015 21:21:21 +0600
Subject: [PATCH 233/415] [vimeo] Fix password protected videos (Closes #7451)
---
youtube_dl/extractor/vimeo.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index ca716c8f5..9dc6247e8 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -217,7 +217,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
url = url.replace('http://', 'https://')
password_request = compat_urllib_request.Request(url + '/password', data)
password_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
- password_request.add_header('Cookie', 'clip_test2=1; vuid=%s' % vuid)
+ password_request.add_header('Cookie', 'clip_test_v2=0; vuid=%s' % vuid)
password_request.add_header('Referer', url)
return self._download_webpage(
password_request, video_id,
From 9a8a12b7d8c09c6034a3d44ee64eef402c9bc076 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 11 Nov 2015 22:23:23 +0600
Subject: [PATCH 234/415] [vimeo] Append cookies instead of overriding
---
youtube_dl/extractor/vimeo.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 9dc6247e8..abfd9b6c4 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -49,8 +49,8 @@ class VimeoBaseInfoExtractor(InfoExtractor):
}))
login_request = compat_urllib_request.Request(self._LOGIN_URL, data)
login_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
- login_request.add_header('Cookie', 'vuid=%s' % vuid)
login_request.add_header('Referer', self._LOGIN_URL)
+ self._set_cookie('vimeo.com', 'vuid', vuid)
self._download_webpage(login_request, None, False, 'Wrong login info')
def _extract_xsrft_and_vuid(self, webpage):
@@ -217,8 +217,8 @@ class VimeoIE(VimeoBaseInfoExtractor):
url = url.replace('http://', 'https://')
password_request = compat_urllib_request.Request(url + '/password', data)
password_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
- password_request.add_header('Cookie', 'clip_test_v2=0; vuid=%s' % vuid)
password_request.add_header('Referer', url)
+ self._set_cookie('vimeo.com', 'vuid', vuid)
return self._download_webpage(
password_request, video_id,
'Verifying the password', 'Wrong password')
@@ -494,7 +494,7 @@ class VimeoChannelIE(VimeoBaseInfoExtractor):
password_url = compat_urlparse.urljoin(page_url, password_path)
password_request = compat_urllib_request.Request(password_url, post)
password_request.add_header('Content-type', 'application/x-www-form-urlencoded')
- password_request.add_header('Cookie', 'vuid=%s' % vuid)
+ self._set_cookie('vimeo.com', 'vuid', vuid)
self._set_cookie('vimeo.com', 'xsrft', token)
return self._download_webpage(
From 9eab37dca0e767a1c9a9d659dd4125ce68d049f9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 11 Nov 2015 22:32:13 +0600
Subject: [PATCH 235/415] [vimeo] Simplify set cookie
---
youtube_dl/extractor/vimeo.py | 11 +++++++----
1 file changed, 7 insertions(+), 4 deletions(-)
diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index abfd9b6c4..b056ba720 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -50,7 +50,7 @@ class VimeoBaseInfoExtractor(InfoExtractor):
login_request = compat_urllib_request.Request(self._LOGIN_URL, data)
login_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
login_request.add_header('Referer', self._LOGIN_URL)
- self._set_cookie('vimeo.com', 'vuid', vuid)
+ self._set_vimeo_cookie('vuid', vuid)
self._download_webpage(login_request, None, False, 'Wrong login info')
def _extract_xsrft_and_vuid(self, webpage):
@@ -62,6 +62,9 @@ class VimeoBaseInfoExtractor(InfoExtractor):
webpage, 'vuid', group='vuid')
return xsrft, vuid
+ def _set_vimeo_cookie(self, name, value):
+ self._set_cookie('vimeo.com', name, value)
+
class VimeoIE(VimeoBaseInfoExtractor):
"""Information extractor for vimeo.com."""
@@ -218,7 +221,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
password_request = compat_urllib_request.Request(url + '/password', data)
password_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
password_request.add_header('Referer', url)
- self._set_cookie('vimeo.com', 'vuid', vuid)
+ self._set_vimeo_cookie('vuid', vuid)
return self._download_webpage(
password_request, video_id,
'Verifying the password', 'Wrong password')
@@ -494,8 +497,8 @@ class VimeoChannelIE(VimeoBaseInfoExtractor):
password_url = compat_urlparse.urljoin(page_url, password_path)
password_request = compat_urllib_request.Request(password_url, post)
password_request.add_header('Content-type', 'application/x-www-form-urlencoded')
- self._set_cookie('vimeo.com', 'vuid', vuid)
- self._set_cookie('vimeo.com', 'xsrft', token)
+ self._set_vimeo_cookie('vuid', vuid)
+ self._set_vimeo_cookie('xsrft', token)
return self._download_webpage(
password_request, list_id,
From 699ed30ceed8a0499d17c54742b63469e5ff08ee Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 11 Nov 2015 22:34:49 +0600
Subject: [PATCH 236/415] [novamov] Modernize
---
youtube_dl/extractor/novamov.py | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/youtube_dl/extractor/novamov.py b/youtube_dl/extractor/novamov.py
index e0bf6d1bc..6b15fc2e5 100644
--- a/youtube_dl/extractor/novamov.py
+++ b/youtube_dl/extractor/novamov.py
@@ -42,8 +42,7 @@ class NovaMovIE(InfoExtractor):
}
def _real_extract(self, url):
- mobj = re.match(self._VALID_URL, url)
- video_id = mobj.group('id')
+ video_id = self._match_id(url)
url = 'http://%s/video/%s' % (self._HOST, video_id)
From 435911029f1dfbe0caaf460a09742e864e656478 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 11 Nov 2015 22:43:17 +0600
Subject: [PATCH 237/415] [vidto] Remove extractor
---
youtube_dl/extractor/__init__.py | 1 -
youtube_dl/extractor/vidto.py | 57 --------------------------------
2 files changed, 58 deletions(-)
delete mode 100644 youtube_dl/extractor/vidto.py
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index ef6c7f0de..0a90da73c 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -730,7 +730,6 @@ from .videopremium import VideoPremiumIE
from .videott import VideoTtIE
from .videoweed import VideoWeedIE
from .vidme import VidmeIE
-from .vidto import VidtoIE
from .vidzi import VidziIE
from .vier import VierIE, VierVideosIE
from .viewster import ViewsterIE
diff --git a/youtube_dl/extractor/vidto.py b/youtube_dl/extractor/vidto.py
deleted file mode 100644
index 5ebf0f3d3..000000000
--- a/youtube_dl/extractor/vidto.py
+++ /dev/null
@@ -1,57 +0,0 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
-import time
-
-from .common import InfoExtractor
-from ..utils import (
- encode_dict,
- remove_end,
- urlencode_postdata,
-)
-from ..compat import compat_urllib_request
-
-
-class VidtoIE(InfoExtractor):
- IE_NAME = 'vidto'
- IE_DESC = 'VidTo.me'
- _VALID_URL = r'https?://(?:www\.)?vidto\.me/(?P<id>[0-9a-zA-Z]+)\.html'
- _TEST = {
- 'url': 'http://vidto.me/ku5glz52nqe1.html',
- 'info_dict': {
- 'id': 'ku5glz52nqe1',
- 'ext': 'mp4',
- 'title': 'test'
- }
- }
-
- def _real_extract(self, url):
- video_id = self._match_id(url)
-
- page = self._download_webpage(
- 'http://vidto.me/%s.html' % video_id, video_id, 'Downloading video page')
-
- title = remove_end(self._html_search_regex(
- r'<Title>\s*([^<]+)\s*</Title>', page, 'title'), ' - Vidto')
-
- hidden_fields = self._hidden_inputs(page)
-
- self.to_screen('Waiting for countdown...')
- time.sleep(7)
-
- req = compat_urllib_request.Request(
- url, urlencode_postdata(encode_dict(hidden_fields)))
- req.add_header('Content-type', 'application/x-www-form-urlencoded')
-
- post_result = self._download_webpage(
- req, video_id,
- note='Proceed to video...', errnote='unable to proceed')
-
- file_link = self._search_regex(
- r'file_link\s*=\s*\'(https?:\/\/[0-9a-zA-z.\/\-_]+)', post_result, 'file_link')
-
- return {
- 'id': video_id,
- 'url': file_link,
- 'title': title,
- }
From b9ad101926b928445acf4a1ab48c24e17836b22c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 11 Nov 2015 22:44:03 +0600
Subject: [PATCH 238/415] [gorillavid] Add support for vidto.me
---
youtube_dl/extractor/gorillavid.py | 15 ++++++++++-----
1 file changed, 10 insertions(+), 5 deletions(-)
diff --git a/youtube_dl/extractor/gorillavid.py b/youtube_dl/extractor/gorillavid.py
index d23e3eac1..d4fd0449c 100644
--- a/youtube_dl/extractor/gorillavid.py
+++ b/youtube_dl/extractor/gorillavid.py
@@ -16,10 +16,10 @@ from ..utils import (
class GorillaVidIE(InfoExtractor):
- IE_DESC = 'GorillaVid.in, daclips.in, movpod.in, fastvideo.in, realvid.net and filehoot.com'
+ IE_DESC = 'GorillaVid.in, daclips.in, movpod.in, fastvideo.in, realvid.net, filehoot.com and vidto.me'
_VALID_URL = r'''(?x)
https?://(?P<host>(?:www\.)?
- (?:daclips\.in|gorillavid\.in|movpod\.in|fastvideo\.in|realvid\.net|filehoot\.com))/
+ (?:daclips\.in|gorillavid\.in|movpod\.in|fastvideo\.in|realvid\.net|filehoot\.com|vidto.\me))/
(?:embed-)?(?P<id>[0-9a-zA-Z]+)(?:-[0-9]+x[0-9]+\.html)?
'''
@@ -105,12 +105,17 @@ class GorillaVidIE(InfoExtractor):
webpage = self._download_webpage(req, video_id, 'Downloading video page')
title = self._search_regex(
- [r'style="z-index: [0-9]+;">([^<]+)</span>', r'<td nowrap>([^<]+)</td>', r'>Watch (.+) '],
+ [r'style="z-index: [0-9]+;">([^<]+)</span>',
+ r'<td nowrap>([^<]+)</td>',
+ r'>Watch (.+) ',
+ r'<h2 class="video-page-head">([^<]+)</h2>'],
webpage, 'title', default=None) or self._og_search_title(webpage)
video_url = self._search_regex(
- r'file\s*:\s*["\'](http[^"\']+)["\'],', webpage, 'file url')
+ [r'file\s*:\s*["\'](http[^"\']+)["\'],',
+ r'file_link\s*=\s*\'(https?:\/\/[0-9a-zA-z.\/\-_]+)'],
+ webpage, 'file url')
thumbnail = self._search_regex(
- r'image\s*:\s*["\'](http[^"\']+)["\'],', webpage, 'thumbnail', fatal=False)
+ r'image\s*:\s*["\'](http[^"\']+)["\'],', webpage, 'thumbnail', default=None)
formats = [{
'format_id': 'sd',
From 668db403f97eb1b50a3f54266db4bc274f1fa874 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 11 Nov 2015 22:47:28 +0600
Subject: [PATCH 239/415] [gorillavid] Add test for vidto.me and strip title
---
youtube_dl/extractor/gorillavid.py | 11 +++++++++--
1 file changed, 9 insertions(+), 2 deletions(-)
diff --git a/youtube_dl/extractor/gorillavid.py b/youtube_dl/extractor/gorillavid.py
index d4fd0449c..8c982d8e9 100644
--- a/youtube_dl/extractor/gorillavid.py
+++ b/youtube_dl/extractor/gorillavid.py
@@ -76,6 +76,13 @@ class GorillaVidIE(InfoExtractor):
'title': 'youtube-dl test video \'äBaW_jenozKc.mp4.mp4',
'thumbnail': 're:http://.*\.jpg',
}
+ }, {
+ 'url': 'http://vidto.me/ku5glz52nqe1.html',
+ 'info_dict': {
+ 'id': 'ku5glz52nqe1',
+ 'ext': 'mp4',
+ 'title': 'test'
+ }
}]
def _real_extract(self, url):
@@ -104,12 +111,12 @@ class GorillaVidIE(InfoExtractor):
webpage = self._download_webpage(req, video_id, 'Downloading video page')
- title = self._search_regex(
+ title = (self._search_regex(
[r'style="z-index: [0-9]+;">([^<]+)</span>',
r'<td nowrap>([^<]+)</td>',
r'>Watch (.+) ',
r'<h2 class="video-page-head">([^<]+)</h2>'],
- webpage, 'title', default=None) or self._og_search_title(webpage)
+ webpage, 'title', default=None) or self._og_search_title(webpage)).strip()
video_url = self._search_regex(
[r'file\s*:\s*["\'](http[^"\']+)["\'],',
r'file_link\s*=\s*\'(https?:\/\/[0-9a-zA-z.\/\-_]+)'],
From 031ec536f00af5dc4ec034d3adb7d5fe3cc02453 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 11 Nov 2015 22:58:39 +0600
Subject: [PATCH 240/415] [gorillavid] Rename to xfileshare
---
youtube_dl/extractor/__init__.py | 2 +-
youtube_dl/extractor/{gorillavid.py => xfileshare.py} | 6 +++---
2 files changed, 4 insertions(+), 4 deletions(-)
rename youtube_dl/extractor/{gorillavid.py => xfileshare.py} (96%)
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 0a90da73c..06d25ef40 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -221,7 +221,6 @@ from .goldenmoustache import GoldenMoustacheIE
from .golem import GolemIE
from .googleplus import GooglePlusIE
from .googlesearch import GoogleSearchIE
-from .gorillavid import GorillaVidIE
from .goshgay import GoshgayIE
from .groupon import GrouponIE
from .hark import HarkIE
@@ -786,6 +785,7 @@ from .wrzuta import WrzutaIE
from .wsj import WSJIE
from .xbef import XBefIE
from .xboxclips import XboxClipsIE
+from .xfileshare import XFileShareIE
from .xhamster import (
XHamsterIE,
XHamsterEmbedIE,
diff --git a/youtube_dl/extractor/gorillavid.py b/youtube_dl/extractor/xfileshare.py
similarity index 96%
rename from youtube_dl/extractor/gorillavid.py
rename to youtube_dl/extractor/xfileshare.py
index 8c982d8e9..7610dc627 100644
--- a/youtube_dl/extractor/gorillavid.py
+++ b/youtube_dl/extractor/xfileshare.py
@@ -1,4 +1,4 @@
-# -*- coding: utf-8 -*-
+# coding: utf-8
from __future__ import unicode_literals
import re
@@ -15,8 +15,8 @@ from ..utils import (
)
-class GorillaVidIE(InfoExtractor):
- IE_DESC = 'GorillaVid.in, daclips.in, movpod.in, fastvideo.in, realvid.net, filehoot.com and vidto.me'
+class XFileShareIE(InfoExtractor):
+ IE_DESC = 'XFileShare based sites: GorillaVid.in, daclips.in, movpod.in, fastvideo.in, realvid.net, filehoot.com and vidto.me'
_VALID_URL = r'''(?x)
https?://(?P<host>(?:www\.)?
(?:daclips\.in|gorillavid\.in|movpod\.in|fastvideo\.in|realvid\.net|filehoot\.com|vidto.\me))/
From fcd817a32636268156dfab20ffc7896fb6082f50 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 12 Nov 2015 03:56:11 +0600
Subject: [PATCH 241/415] [vimeo] Fix extraction (Closes #7460)
---
youtube_dl/extractor/vimeo.py | 52 ++++++++++++-----------------------
1 file changed, 17 insertions(+), 35 deletions(-)
diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index b056ba720..b72341a2b 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -387,47 +387,29 @@ class VimeoIE(VimeoBaseInfoExtractor):
like_count = None
comment_count = None
- # Vimeo specific: extract request signature and timestamp
- sig = config['request']['signature']
- timestamp = config['request']['timestamp']
-
- # Vimeo specific: extract video codec and quality information
- # First consider quality, then codecs, then take everything
- codecs = [('vp6', 'flv'), ('vp8', 'flv'), ('h264', 'mp4')]
- files = {'hd': [], 'sd': [], 'other': []}
- config_files = config["video"].get("files") or config["request"].get("files")
- for codec_name, codec_extension in codecs:
- for quality in config_files.get(codec_name, []):
- format_id = '-'.join((codec_name, quality)).lower()
- key = quality if quality in files else 'other'
- video_url = None
- if isinstance(config_files[codec_name], dict):
- file_info = config_files[codec_name][quality]
- video_url = file_info.get('url')
- else:
- file_info = {}
- if video_url is None:
- video_url = "http://player.vimeo.com/play_redirect?clip_id=%s&sig=%s&time=%s&quality=%s&codecs=%s&type=moogaloop_local&embed_location=" \
- % (video_id, sig, timestamp, quality, codec_name.upper())
-
- files[key].append({
- 'ext': codec_extension,
- 'url': video_url,
- 'format_id': format_id,
- 'width': int_or_none(file_info.get('width')),
- 'height': int_or_none(file_info.get('height')),
- 'tbr': int_or_none(file_info.get('bitrate')),
- })
formats = []
- m3u8_url = config_files.get('hls', {}).get('all')
+ config_files = config['video'].get('files') or config['request'].get('files', {})
+ for f in config_files.get('progressive', []):
+ video_url = f.get('url')
+ if not video_url:
+ continue
+ formats.append({
+ 'url': video_url,
+ 'format_id': 'http-%s' % f.get('quality'),
+ 'width': int_or_none(f.get('width')),
+ 'height': int_or_none(f.get('height')),
+ 'fps': int_or_none(f.get('fps')),
+ 'tbr': int_or_none(f.get('bitrate')),
+ })
+ m3u8_url = config_files.get('hls', {}).get('url')
if m3u8_url:
m3u8_formats = self._extract_m3u8_formats(
m3u8_url, video_id, 'mp4', 'm3u8_native', 0, 'hls', fatal=False)
if m3u8_formats:
formats.extend(m3u8_formats)
- for key in ('other', 'sd', 'hd'):
- formats += files[key]
- self._sort_formats(formats)
+ # Bitrates are completely broken. Single m3u8 may contain entries in kbps and bps
+ # at the same time without actual units specified. This lead to wrong sorting.
+ self._sort_formats(formats, field_preference=('height', 'width', 'fps', 'format_id'))
subtitles = {}
text_tracks = config['request'].get('text_tracks')
From cf5881fc4defe3902367fc02b8eb39d912572d5a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 12 Nov 2015 02:13:42 +0800
Subject: [PATCH 242/415] Credit @ferama
For providing idea for vidto.me (#7167) and extending nowvideo support (#6760)
---
AUTHORS | 1 +
youtube_dl/extractor/twitter.py | 58 +++++++++++++++++++++++++++------
2 files changed, 49 insertions(+), 10 deletions(-)
diff --git a/AUTHORS b/AUTHORS
index 4cdbf4a5f..f465d20ed 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -145,3 +145,4 @@ Anssi Hannula
Lukáš Lalinský
Qijiang Fan
Rémy Léone
+Marco Ferragina
diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index 9d3e46b94..ae1212d19 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -9,6 +9,8 @@ from ..utils import (
float_or_none,
xpath_text,
remove_end,
+ int_or_none,
+ ExtractorError,
)
@@ -120,7 +122,7 @@ class TwitterIE(InfoExtractor):
_VALID_URL = r'https?://(?:www\.|m\.|mobile\.)?twitter\.com/(?P<user_id>[^/]+)/status/(?P<id>\d+)'
_TEMPLATE_URL = 'https://twitter.com/%s/status/%s'
- _TEST = {
+ _TESTS = [{
'url': 'https://twitter.com/freethenipple/status/643211948184596480',
'md5': '31cd83a116fc41f99ae3d909d4caf6a0',
'info_dict': {
@@ -133,7 +135,19 @@ class TwitterIE(InfoExtractor):
'uploader': 'FREE THE NIPPLE',
'uploader_id': 'freethenipple',
},
- }
+ }, {
+ 'url': 'https://twitter.com/giphz/status/657991469417025536/photo/1',
+ 'md5': 'f36dcd5fb92bf7057f155e7d927eeb42',
+ 'info_dict': {
+ 'id': '657991469417025536',
+ 'ext': 'mp4',
+ 'title': 'Gifs - tu vai cai tu vai cai tu nao eh capaz disso tu vai cai',
+ 'description': 'Gifs on Twitter: "tu vai cai tu vai cai tu nao eh capaz disso tu vai cai https://t.co/tM46VHFlO5"',
+ 'thumbnail': 're:^https?://.*\.png',
+ 'uploader': 'Gifs',
+ 'uploader_id': 'giphz',
+ },
+ }]
def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url)
@@ -150,17 +164,41 @@ class TwitterIE(InfoExtractor):
mobj = re.match(r'“(.*)\s+(https?://[^ ]+)”', title)
title, short_url = mobj.groups()
- card_id = self._search_regex(
- r'["\']/i/cards/tfw/v1/(\d+)', webpage, 'twitter card url')
- card_url = 'https://twitter.com/i/cards/tfw/v1/' + card_id
-
- return {
- '_type': 'url_transparent',
- 'ie_key': 'TwitterCard',
+ info = {
'uploader_id': user_id,
'uploader': username,
- 'url': card_url,
'webpage_url': url,
'description': '%s on Twitter: "%s %s"' % (username, title, short_url),
'title': username + ' - ' + title,
}
+
+ card_id = self._search_regex(
+ r'["\']/i/cards/tfw/v1/(\d+)', webpage, 'twitter card url', default=None)
+ if card_id:
+ card_url = 'https://twitter.com/i/cards/tfw/v1/' + card_id
+ info.update({
+ '_type': 'url_transparent',
+ 'ie_key': 'TwitterCard',
+ 'url': card_url,
+ })
+ return info
+
+ mobj = re.search(r'''(?x)
+ <video[^>]+class="animated-gif"[^>]+
+ (?:data-height="(?P<height>\d+)")?[^>]+
+ (?:data-width="(?P<width>\d+)")?[^>]+
+ (?:poster="(?P<poster>[^"]+)")?[^>]*>\s*
+ <source[^>]+video-src="(?P<url>[^"]+)"
+ ''', webpage)
+
+ if mobj:
+ info.update({
+ 'id': twid,
+ 'url': mobj.group('url'),
+ 'height': int_or_none(mobj.group('height')),
+ 'width': int_or_none(mobj.group('width')),
+ 'thumbnail': mobj.group('poster'),
+ })
+ return info
+
+ raise ExtractorError('There\'s not video in this tweet.')
From 1ebb4717dfc67890c66abd4274e201cafda48e79 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
<jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 12 Nov 2015 21:02:56 +0100
Subject: [PATCH 243/415] [cbsnews] Fix construction of 'play_path' in some
videos (fixes #7394)
---
youtube_dl/extractor/cbsnews.py | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/cbsnews.py b/youtube_dl/extractor/cbsnews.py
index 52e61d85b..f9a64a0a2 100644
--- a/youtube_dl/extractor/cbsnews.py
+++ b/youtube_dl/extractor/cbsnews.py
@@ -67,9 +67,12 @@ class CBSNewsIE(InfoExtractor):
'format_id': format_id,
}
if uri.startswith('rtmp'):
+ play_path = re.sub(
+ r'{slistFilePath}', '',
+ uri.split('<break>')[-1].split('{break}')[-1])
fmt.update({
'app': 'ondemand?auth=cbs',
- 'play_path': 'mp4:' + uri.split('<break>')[-1],
+ 'play_path': 'mp4:' + play_path,
'player_url': 'http://www.cbsnews.com/[[IMPORT]]/vidtech.cbsinteractive.com/player/3_3_0/CBSI_PLAYER_HD.swf',
'page_url': 'http://www.cbsnews.com',
'ext': 'flv',
From 5d6c3d6a665ed5de89bb3b056bb051b043400897 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 13 Nov 2015 02:41:38 +0600
Subject: [PATCH 244/415] [ruutu] Skip NOT-USED URLs(Closes #7478)
---
youtube_dl/extractor/ruutu.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/ruutu.py b/youtube_dl/extractor/ruutu.py
index a16b73ff4..c19107cb0 100644
--- a/youtube_dl/extractor/ruutu.py
+++ b/youtube_dl/extractor/ruutu.py
@@ -57,7 +57,8 @@ class RuutuIE(InfoExtractor):
extract_formats(child)
elif child.tag.endswith('File'):
video_url = child.text
- if not video_url or video_url in processed_urls or 'NOT_USED' in video_url:
+ if (not video_url or video_url in processed_urls or
+ any(p in video_url for p in ('NOT_USED', 'NOT-USED'))):
return
processed_urls.append(video_url)
ext = determine_ext(video_url)
From 91d644b5ba7e520363d146442682b7df5a175d31 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 13 Nov 2015 02:43:27 +0600
Subject: [PATCH 245/415] [ruutu] Relax formats extraction
---
youtube_dl/extractor/ruutu.py | 12 ++++++++----
1 file changed, 8 insertions(+), 4 deletions(-)
diff --git a/youtube_dl/extractor/ruutu.py b/youtube_dl/extractor/ruutu.py
index c19107cb0..e417bf661 100644
--- a/youtube_dl/extractor/ruutu.py
+++ b/youtube_dl/extractor/ruutu.py
@@ -63,11 +63,15 @@ class RuutuIE(InfoExtractor):
processed_urls.append(video_url)
ext = determine_ext(video_url)
if ext == 'm3u8':
- formats.extend(self._extract_m3u8_formats(
- video_url, video_id, 'mp4', m3u8_id='hls'))
+ m3u8_formats = self._extract_m3u8_formats(
+ video_url, video_id, 'mp4', m3u8_id='hls', fatal=False)
+ if m3u8_formats:
+ formats.extend(m3u8_formats)
elif ext == 'f4m':
- formats.extend(self._extract_f4m_formats(
- video_url, video_id, f4m_id='hds'))
+ f4m_formats = self._extract_f4m_formats(
+ video_url, video_id, f4m_id='hds', fatal=False)
+ if f4m_formats:
+ formats.extend(f4m_formats)
else:
proto = compat_urllib_parse_urlparse(video_url).scheme
if not child.tag.startswith('HTTP') and proto != 'rtmp':
From a1ec9a7553b6bb09b0f88ce5e987c04e2f16ba4d Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 13 Nov 2015 11:07:30 +0100
Subject: [PATCH 246/415] release 2015.11.13
---
docs/supportedsites.md | 2 +-
youtube_dl/version.py | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index a9820c1f5..5016ba4bc 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -200,7 +200,6 @@
- **GodTube**
- **GoldenMoustache**
- **Golem**
- - **GorillaVid**: GorillaVid.in, daclips.in, movpod.in, fastvideo.in, realvid.net and filehoot.com
- **Goshgay**
- **Groupon**
- **Hark**
@@ -671,6 +670,7 @@
- **WSJ**: Wall Street Journal
- **XBef**
- **XboxClips**
+ - **XFileShare**: XFileShare based sites: GorillaVid.in, daclips.in, movpod.in, fastvideo.in, realvid.net, filehoot.com and vidto.me
- **XHamster**
- **XHamsterEmbed**
- **XMinus**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index b3d254005..6585d60d5 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
from __future__ import unicode_literals
-__version__ = '2015.11.10'
+__version__ = '2015.11.13'
From b84a5f03375d086e9912cfa7c690ff824b1092a9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
<jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 13 Nov 2015 18:55:07 +0100
Subject: [PATCH 247/415] [twitter] Update tests checksums
---
youtube_dl/extractor/twitter.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index ae1212d19..0db7a4440 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -20,7 +20,7 @@ class TwitterCardIE(InfoExtractor):
_TESTS = [
{
'url': 'https://twitter.com/i/cards/tfw/v1/560070183650213889',
- 'md5': '7d2f6b4d2eb841a7ccc893d479bfceb4',
+ 'md5': '4fa26a35f9d1bf4b646590ba8e84be19',
'info_dict': {
'id': '560070183650213889',
'ext': 'mp4',
@@ -124,7 +124,7 @@ class TwitterIE(InfoExtractor):
_TESTS = [{
'url': 'https://twitter.com/freethenipple/status/643211948184596480',
- 'md5': '31cd83a116fc41f99ae3d909d4caf6a0',
+ 'md5': 'db6612ec5d03355953c3ca9250c97e5e',
'info_dict': {
'id': '643211948184596480',
'ext': 'mp4',
From b703ebeeafed3e890270633788c7340d174cde86 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
<jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 13 Nov 2015 19:09:42 +0100
Subject: [PATCH 248/415] [twitter] Don't fail if the description doesn't
contain an URL (fixes #7489)
---
youtube_dl/extractor/twitter.py | 18 ++++++++++++++----
1 file changed, 14 insertions(+), 4 deletions(-)
diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index 0db7a4440..2bd5946ac 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -147,6 +147,17 @@ class TwitterIE(InfoExtractor):
'uploader': 'Gifs',
'uploader_id': 'giphz',
},
+ }, {
+ 'url': 'https://twitter.com/starwars/status/665052190608723968',
+ 'md5': '39b7199856dee6cd4432e72c74bc69d4',
+ 'info_dict': {
+ 'id': '665052190608723968',
+ 'ext': 'mp4',
+ 'title': 'Star Wars - A new beginning is coming December 18. Watch the official 60 second #TV spot for #StarWars: #TheForceAwakens.',
+ 'description': 'Star Wars on Twitter: "A new beginning is coming December 18. Watch the official 60 second #TV spot for #StarWars: #TheForceAwakens."',
+ 'uploader_id': 'starwars',
+ 'uploader': 'Star Wars',
+ },
}]
def _real_extract(self, url):
@@ -158,17 +169,16 @@ class TwitterIE(InfoExtractor):
username = remove_end(self._og_search_title(webpage), ' on Twitter')
- title = self._og_search_description(webpage).strip('').replace('\n', ' ')
+ title = description = self._og_search_description(webpage).strip('').replace('\n', ' ').strip('“”')
# strip 'https -_t.co_BJYgOjSeGA' junk from filenames
- mobj = re.match(r'“(.*)\s+(https?://[^ ]+)”', title)
- title, short_url = mobj.groups()
+ title = re.sub(r'\s+(https?://[^ ]+)', '', title)
info = {
'uploader_id': user_id,
'uploader': username,
'webpage_url': url,
- 'description': '%s on Twitter: "%s %s"' % (username, title, short_url),
+ 'description': '%s on Twitter: "%s"' % (username, description),
'title': username + ' - ' + title,
}
From 4e21b3a94f1ce7ba3757d59d1ac4baf9efaeed84 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Nov 2015 00:22:32 +0600
Subject: [PATCH 249/415] [cbs] Use android UA for higher quality streams
(Closes #7490)
---
youtube_dl/extractor/cbs.py | 12 ++++++++++--
1 file changed, 10 insertions(+), 2 deletions(-)
diff --git a/youtube_dl/extractor/cbs.py b/youtube_dl/extractor/cbs.py
index 75fffb156..43f05d278 100644
--- a/youtube_dl/extractor/cbs.py
+++ b/youtube_dl/extractor/cbs.py
@@ -1,6 +1,8 @@
from __future__ import unicode_literals
from .common import InfoExtractor
+from ..compat import compat_urllib_request
+from ..utils import smuggle_url
class CBSIE(InfoExtractor):
@@ -46,13 +48,19 @@ class CBSIE(InfoExtractor):
def _real_extract(self, url):
display_id = self._match_id(url)
- webpage = self._download_webpage(url, display_id)
+ request = compat_urllib_request.Request(url)
+ # Android UA is served with higher quality (720p) streams (see
+ # https://github.com/rg3/youtube-dl/issues/7490)
+ request.add_header('User-Agent', 'Mozilla/5.0 (Linux; Android 4.4; Nexus 5)')
+ webpage = self._download_webpage(request, display_id)
real_id = self._search_regex(
[r"video\.settings\.pid\s*=\s*'([^']+)';", r"cbsplayer\.pid\s*=\s*'([^']+)';"],
webpage, 'real video ID')
return {
'_type': 'url_transparent',
'ie_key': 'ThePlatform',
- 'url': 'theplatform:%s' % real_id,
+ 'url': smuggle_url(
+ 'http://link.theplatform.com/s/dJ5BDC/%s?mbr=true&manifest=m3u' % real_id,
+ {'force_smil_url': True}),
'display_id': display_id,
}
From a662489877aa8d88b898a4984c2e580b9edbe7de Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Nov 2015 05:09:50 +0600
Subject: [PATCH 250/415] [brightcove:embedinpage] Make more robust and extract
rtmp streams
---
youtube_dl/extractor/brightcove.py | 117 +++++++++++++++++++----------
1 file changed, 79 insertions(+), 38 deletions(-)
diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 2c7d968a8..2ad35ec90 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -23,6 +23,7 @@ from ..utils import (
unescapeHTML,
unsmuggle_url,
js_to_json,
+ float_or_none,
int_or_none,
parse_iso8601,
extract_attributes,
@@ -353,7 +354,7 @@ class BrightcoveIE(InfoExtractor):
class BrightcoveInPageEmbedIE(InfoExtractor):
- _VALID_URL = r'https?://players\.brightcove\.net/(?P<account_id>\d+)/([a-z0-9-]+)_([a-z]+)/index.html?.*videoId=(?P<video_id>\d+)'
+ _VALID_URL = r'https?://players\.brightcove\.net/(?P<account_id>\d+)/(?P<player_id>[\da-f-]+)_(?P<embed>[a-z]+)/index\.html\?.*videoId=(?P<video_id>\d+)'
_TEST = {
'url': 'http://players.brightcove.net/929656772001/e41d32dc-ec74-459e-a845-6c69f7b724ea_default/index.html?videoId=4463358922001',
'md5': 'c8100925723840d4b0d243f7025703be',
@@ -385,59 +386,99 @@ class BrightcoveInPageEmbedIE(InfoExtractor):
def _real_extract(self, url):
account_id, player_id, embed, video_id = re.match(self._VALID_URL, url).groups()
- webpage = self._download_webpage('http://players.brightcove.net/%s/%s_%s/index.min.js' % (account_id, player_id, embed), video_id)
+ webpage = self._download_webpage(
+ 'http://players.brightcove.net/%s/%s_%s/index.min.js'
+ % (account_id, player_id, embed), video_id)
- catalog = self._parse_json(
- js_to_json(
- self._search_regex(
- r'catalog\(({[^}]+})\);',
- webpage,
- 'catalog'
- )
- ),
- video_id
- )
- policy_key = catalog['policyKey']
+ policy_key = None
+
+ catalog = self._search_regex(
+ r'catalog\(({.+?})\);', webpage, 'catalog', default=None)
+ if catalog:
+ catalog = self._parse_json(
+ js_to_json(catalog), video_id, fatal=False)
+ if catalog:
+ policy_key = catalog.get('policyKey')
+
+ if not policy_key:
+ policy_key = self._search_regex(
+ r'policyKey\s*:\s*(["\'])(?P<pk>.+?)\1',
+ webpage, 'policy key', group='pk')
req = compat_urllib_request.Request(
- 'https://edge.api.brightcove.com/playback/v1/accounts/%s/videos/%s' % (account_id, video_id),
+ 'https://edge.api.brightcove.com/playback/v1/accounts/%s/videos/%s'
+ % (account_id, video_id),
headers={'Accept': 'application/json;pk=%s' % policy_key})
json_data = self._download_json(req, video_id)
title = json_data['name']
+
+ formats = []
+ for source in json_data.get('sources', []):
+ source_type = source.get('type')
+ src = source.get('src')
+ if source_type == 'application/x-mpegURL':
+ if not src:
+ continue
+ m3u8_formats = self._extract_m3u8_formats(
+ src, video_id, 'mp4', entry_protocol='m3u8_native',
+ m3u8_id='hls', fatal=False)
+ if m3u8_formats:
+ formats.extend(m3u8_formats)
+ else:
+ streaming_src = source.get('streaming_src')
+ stream_name, app_name = source.get('stream_name'), source.get('app_name')
+ if not src and not streaming_src and (not stream_name or not app_name):
+ continue
+ tbr = float_or_none(source.get('avg_bitrate'), 1000)
+ height = int_or_none(source.get('height'))
+ f = {
+ 'tbr': tbr,
+ 'width': int_or_none(source.get('width')),
+ 'height': height,
+ 'filesize': int_or_none(source.get('size')),
+ 'container': source.get('container'),
+ 'vcodec': source.get('codec'),
+ 'ext': source.get('container').lower(),
+ }
+
+ def build_format_id(kind):
+ format_id = kind
+ if tbr:
+ format_id += '-%dk' % int(tbr)
+ if height:
+ format_id += '-%dp' % height
+ return format_id
+
+ if src or streaming_src:
+ f.update({
+ 'url': src or streaming_src,
+ 'format_id': build_format_id('http' if src else 'http-streaming'),
+ 'preference': 2 if src else 1,
+ })
+ else:
+ f.update({
+ 'url': app_name,
+ 'play_path': stream_name,
+ 'format_id': build_format_id('rtmp'),
+ })
+ formats.append(f)
+ self._sort_formats(formats)
+
description = json_data.get('description')
thumbnail = json_data.get('thumbnail')
timestamp = parse_iso8601(json_data.get('published_at'))
- duration = int_or_none(json_data.get('duration'))
-
- formats = []
- for source in json_data.get('sources'):
- source_type = source.get('type')
- if source_type == 'application/x-mpegURL':
- formats.extend(self._extract_m3u8_formats(source.get('src'), video_id))
- else:
- src = source.get('src') or source.get('streaming_src')
- if src:
- formats.append({
- 'url': src,
- 'tbr': source.get('avg_bitrate'),
- 'width': int_or_none(source.get('width')),
- 'height': int_or_none(source.get('height')),
- 'filesize': source.get('size'),
- 'container': source.get('container'),
- 'vcodec': source.get('codec'),
- 'ext': source.get('container').lower(),
- })
-
- self._sort_formats(formats)
+ duration = float_or_none(json_data.get('duration'), 1000)
+ tags = json_data.get('tags', [])
return {
'id': video_id,
'title': title,
'description': description,
'thumbnail': thumbnail,
- 'timestamp': timestamp,
'duration': duration,
- 'formats': formats,
+ 'timestamp': timestamp,
'uploader_id': account_id,
+ 'formats': formats,
+ 'tags': tags,
}
From 536f819eda975224666374a9ce83cc3472f5aa5a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Nov 2015 05:51:05 +0600
Subject: [PATCH 251/415] [brightcove] Imrove extraction of new embeds
---
youtube_dl/extractor/brightcove.py | 42 +++++++++++++++++++++---------
1 file changed, 29 insertions(+), 13 deletions(-)
diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 2ad35ec90..d494b8b67 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -354,7 +354,7 @@ class BrightcoveIE(InfoExtractor):
class BrightcoveInPageEmbedIE(InfoExtractor):
- _VALID_URL = r'https?://players\.brightcove\.net/(?P<account_id>\d+)/(?P<player_id>[\da-f-]+)_(?P<embed>[a-z]+)/index\.html\?.*videoId=(?P<video_id>\d+)'
+ _VALID_URL = r'https?://players\.brightcove\.net/(?P<account_id>\d+)/(?P<player_id>[^/]+)_(?P<embed>[^/]+)/index\.html\?.*videoId=(?P<video_id>\d+)'
_TEST = {
'url': 'http://players.brightcove.net/929656772001/e41d32dc-ec74-459e-a845-6c69f7b724ea_default/index.html?videoId=4463358922001',
'md5': 'c8100925723840d4b0d243f7025703be',
@@ -370,18 +370,34 @@ class BrightcoveInPageEmbedIE(InfoExtractor):
}
}
- @staticmethod
- def _extract_url(webpage):
- video_attributes = re.search(r'(?s)<video([^>]*)>.*?</(?:video|audio)>', webpage)
- if video_attributes:
- video_attributes = extract_attributes(video_attributes.group(), r'(?s)\s*data-(account|video-id|playlist-id|policy-key|player|embed)\s*=\s*["\']([^"\']+)["\']')
- account_id = video_attributes.get('account')
- player_id = video_attributes.get('player')
- embed = video_attributes.get('embed')
- video_id = video_attributes.get('video-id')
- if account_id and player_id and embed and video_id:
- return 'http://players.brightcove.net/%s/%s_%s/index.html?videoId=%s' % (account_id, player_id, embed, video_id)
- return None
+ def _extract_urls(self, webpage):
+ # Reference:
+ # 1. http://docs.brightcove.com/en/video-cloud/brightcove-player/guides/publish-video.html#setvideoiniframe
+ # 2. http://docs.brightcove.com/en/video-cloud/brightcove-player/guides/publish-video.html#setvideousingjavascript)
+ # 3. http://docs.brightcove.com/en/video-cloud/brightcove-player/guides/embed-in-page.html
+
+ entries = []
+
+ # Look for iframe embeds [1]
+ for _, url in re.findall(
+ r'<iframe[^>]+src=(["\'])((?:https?:)//players\.brightcove\.net/\d+/[^/]+/index\.html.+?)\1', webpage):
+ entries.append(self.url_result(self._proto_relative_url(url)))
+ # Look for embed_in_page embeds [2]
+ # According to examples from [3] it's unclear whether video id may be optional
+ # and what to do when it is
+ for video_id, account_id, player_id, embed in re.findall(
+ r'''(?sx)
+ <video[^>]+
+ data-video-id=["\'](\d+)["\'][^>]*>.*?
+ </video>.*?
+ <script[^>]+
+ src=["\'](?:https?:)?//players\.brightcove\.net/
+ (\d+)/([\da-f-]+)_([^/]+)/index\.min\.js
+ ''', webpage):
+ entries.append(self.url_result(
+ 'http://players.brightcove.net/%s/%s_%s/index.html?videoId=%s'
+ % (account_id, player_id, embed, video_id)))
+ return entries
def _real_extract(self, url):
account_id, player_id, embed, video_id = re.match(self._VALID_URL, url).groups()
From 4fcaa4f4a5ef328009bef53ebc491ebe76452550 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Nov 2015 05:54:16 +0600
Subject: [PATCH 252/415] [brightcove] Rename extractor to brightcove legacy
Old embedding approaches are now "Legacy Studio"
---
youtube_dl/extractor/__init__.py | 2 +-
youtube_dl/extractor/brightcove.py | 2 +-
youtube_dl/extractor/generic.py | 4 ++--
youtube_dl/extractor/nowness.py | 4 ++--
youtube_dl/extractor/safari.py | 4 ++--
youtube_dl/extractor/space.py | 6 +++---
youtube_dl/extractor/tlc.py | 6 +++---
7 files changed, 14 insertions(+), 14 deletions(-)
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 08cb93d76..8a0f76d7e 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -61,7 +61,7 @@ from .bpb import BpbIE
from .br import BRIE
from .breakcom import BreakIE
from .brightcove import (
- BrightcoveIE,
+ BrightcoveLegacyIE,
BrightcoveInPageEmbedIE,
)
from .buzzfeed import BuzzFeedIE
diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index d494b8b67..4dbc2e975 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -30,7 +30,7 @@ from ..utils import (
)
-class BrightcoveIE(InfoExtractor):
+class BrightcoveLegacyIE(InfoExtractor):
_VALID_URL = r'(?:https?://.*brightcove\.com/(services|viewer).*?\?|brightcove:)(?P<query>.*)'
_FEDERATED_URL_TEMPLATE = 'http://c.brightcove.com/services/viewer/htmlFederated?%s'
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 34d930a2d..8f99dd9b1 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -31,7 +31,7 @@ from ..utils import (
xpath_text,
)
from .brightcove import (
- BrightcoveIE,
+ BrightcoveLegacyIE,
BrightcoveInPageEmbedIE,
)
from .nbc import NBCSportsVPlayerIE
@@ -1305,7 +1305,7 @@ class GenericIE(InfoExtractor):
urlrs, playlist_id=video_id, playlist_title=video_title)
# Look for BrightCove:
- bc_urls = BrightcoveIE._extract_brightcove_urls(webpage)
+ bc_urls = BrightcoveLegacyIE._extract_brightcove_urls(webpage)
if bc_urls:
self.to_screen('Brightcove video detected.')
entries = [{
diff --git a/youtube_dl/extractor/nowness.py b/youtube_dl/extractor/nowness.py
index b97f62fdb..dab487ea4 100644
--- a/youtube_dl/extractor/nowness.py
+++ b/youtube_dl/extractor/nowness.py
@@ -1,7 +1,7 @@
# encoding: utf-8
from __future__ import unicode_literals
-from .brightcove import BrightcoveIE
+from .brightcove import BrightcoveLegacyIE
from .common import InfoExtractor
from ..utils import ExtractorError
from ..compat import (
@@ -22,7 +22,7 @@ class NownessBaseIE(InfoExtractor):
'http://www.nowness.com/iframe?id=%s' % video_id, video_id,
note='Downloading player JavaScript',
errnote='Unable to download player JavaScript')
- bc_url = BrightcoveIE._extract_brightcove_url(player_code)
+ bc_url = BrightcoveLegacyIE._extract_brightcove_url(player_code)
if bc_url is None:
raise ExtractorError('Could not find player definition')
return self.url_result(bc_url, 'Brightcove')
diff --git a/youtube_dl/extractor/safari.py b/youtube_dl/extractor/safari.py
index a602af692..4f1f05c6a 100644
--- a/youtube_dl/extractor/safari.py
+++ b/youtube_dl/extractor/safari.py
@@ -4,7 +4,7 @@ from __future__ import unicode_literals
import re
from .common import InfoExtractor
-from .brightcove import BrightcoveIE
+from .brightcove import BrightcoveLegacyIE
from ..compat import (
compat_urllib_parse,
@@ -112,7 +112,7 @@ class SafariIE(SafariBaseIE):
'%s/%s/chapter-content/%s.html' % (self._API_BASE, course_id, part),
part)
- bc_url = BrightcoveIE._extract_brightcove_url(webpage)
+ bc_url = BrightcoveLegacyIE._extract_brightcove_url(webpage)
if not bc_url:
raise ExtractorError('Could not extract Brightcove URL from %s' % url, expected=True)
diff --git a/youtube_dl/extractor/space.py b/youtube_dl/extractor/space.py
index c2d0d36a6..2f190f764 100644
--- a/youtube_dl/extractor/space.py
+++ b/youtube_dl/extractor/space.py
@@ -3,7 +3,7 @@ from __future__ import unicode_literals
import re
from .common import InfoExtractor
-from .brightcove import BrightcoveIE
+from .brightcove import BrightcoveLegacyIE
from ..utils import RegexNotFoundError, ExtractorError
@@ -31,8 +31,8 @@ class SpaceIE(InfoExtractor):
brightcove_url = self._og_search_video_url(webpage)
except RegexNotFoundError:
# Other videos works fine with the info from the object
- brightcove_url = BrightcoveIE._extract_brightcove_url(webpage)
+ brightcove_url = BrightcoveLegacyIE._extract_brightcove_url(webpage)
if brightcove_url is None:
raise ExtractorError(
'The webpage does not contain a video', expected=True)
- return self.url_result(brightcove_url, BrightcoveIE.ie_key())
+ return self.url_result(brightcove_url, BrightcoveLegacyIE.ie_key())
diff --git a/youtube_dl/extractor/tlc.py b/youtube_dl/extractor/tlc.py
index 13263614c..d6d038a8d 100644
--- a/youtube_dl/extractor/tlc.py
+++ b/youtube_dl/extractor/tlc.py
@@ -3,7 +3,7 @@ from __future__ import unicode_literals
import re
from .common import InfoExtractor
-from .brightcove import BrightcoveIE
+from .brightcove import BrightcoveLegacyIE
from .discovery import DiscoveryIE
from ..compat import compat_urlparse
@@ -66,6 +66,6 @@ class TlcDeIE(InfoExtractor):
return {
'_type': 'url',
- 'url': BrightcoveIE._extract_brightcove_url(iframe),
- 'ie': BrightcoveIE.ie_key(),
+ 'url': BrightcoveLegacyIE._extract_brightcove_url(iframe),
+ 'ie': BrightcoveLegacyIE.ie_key(),
}
From 5c17f0a67a3a0518f448825eee54d16045acd63c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Nov 2015 05:55:59 +0600
Subject: [PATCH 253/415] [brightcove:embedinpage] Rename extractor to
brightcove new
It's not actually embed_in_page but "New Studio" and allows both iframe and embed_in_page embeds
---
youtube_dl/extractor/__init__.py | 2 +-
youtube_dl/extractor/brightcove.py | 2 +-
youtube_dl/extractor/generic.py | 4 ++--
3 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 8a0f76d7e..64ce3210b 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -62,7 +62,7 @@ from .br import BRIE
from .breakcom import BreakIE
from .brightcove import (
BrightcoveLegacyIE,
- BrightcoveInPageEmbedIE,
+ BrightcoveNewIE,
)
from .buzzfeed import BuzzFeedIE
from .byutv import BYUtvIE
diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 4dbc2e975..5aca0b378 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -353,7 +353,7 @@ class BrightcoveLegacyIE(InfoExtractor):
return info
-class BrightcoveInPageEmbedIE(InfoExtractor):
+class BrightcoveNewIE(InfoExtractor):
_VALID_URL = r'https?://players\.brightcove\.net/(?P<account_id>\d+)/(?P<player_id>[^/]+)_(?P<embed>[^/]+)/index\.html\?.*videoId=(?P<video_id>\d+)'
_TEST = {
'url': 'http://players.brightcove.net/929656772001/e41d32dc-ec74-459e-a845-6c69f7b724ea_default/index.html?videoId=4463358922001',
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 8f99dd9b1..0797e1a90 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -32,7 +32,7 @@ from ..utils import (
)
from .brightcove import (
BrightcoveLegacyIE,
- BrightcoveInPageEmbedIE,
+ BrightcoveNewIE,
)
from .nbc import NBCSportsVPlayerIE
from .ooyala import OoyalaIE
@@ -1322,7 +1322,7 @@ class GenericIE(InfoExtractor):
}
# Look for Brightcove In Page Embed:
- brightcove_in_page_embed_url = BrightcoveInPageEmbedIE._extract_url(webpage)
+ brightcove_in_page_embed_url = BrightcoveNewIE._extract_url(webpage)
if brightcove_in_page_embed_url:
return self.url_result(brightcove_in_page_embed_url, 'BrightcoveInPageEmbed')
From e721d857c2b24c10c09626a4a79172d85e0dc5fb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Nov 2015 05:56:51 +0600
Subject: [PATCH 254/415] [brightcove] Clarify IE_NAMEs
---
youtube_dl/extractor/brightcove.py | 2 ++
1 file changed, 2 insertions(+)
diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 5aca0b378..8ee5486fe 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -31,6 +31,7 @@ from ..utils import (
class BrightcoveLegacyIE(InfoExtractor):
+ IE_NAME = 'brightcove:legacy'
_VALID_URL = r'(?:https?://.*brightcove\.com/(services|viewer).*?\?|brightcove:)(?P<query>.*)'
_FEDERATED_URL_TEMPLATE = 'http://c.brightcove.com/services/viewer/htmlFederated?%s'
@@ -354,6 +355,7 @@ class BrightcoveLegacyIE(InfoExtractor):
class BrightcoveNewIE(InfoExtractor):
+ IE_NAME = 'brightcove:new'
_VALID_URL = r'https?://players\.brightcove\.net/(?P<account_id>\d+)/(?P<player_id>[^/]+)_(?P<embed>[^/]+)/index\.html\?.*videoId=(?P<video_id>\d+)'
_TEST = {
'url': 'http://players.brightcove.net/929656772001/e41d32dc-ec74-459e-a845-6c69f7b724ea_default/index.html?videoId=4463358922001',
From 24af85298ed1862ac809677e70ff59f3e9ee3234 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Nov 2015 06:01:56 +0600
Subject: [PATCH 255/415] [brightcove] Fix _extract_urls
---
youtube_dl/extractor/brightcove.py | 9 +++++----
1 file changed, 5 insertions(+), 4 deletions(-)
diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 8ee5486fe..1c7783dcb 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -372,7 +372,8 @@ class BrightcoveNewIE(InfoExtractor):
}
}
- def _extract_urls(self, webpage):
+ @staticmethod
+ def _extract_urls(webpage):
# Reference:
# 1. http://docs.brightcove.com/en/video-cloud/brightcove-player/guides/publish-video.html#setvideoiniframe
# 2. http://docs.brightcove.com/en/video-cloud/brightcove-player/guides/publish-video.html#setvideousingjavascript)
@@ -383,7 +384,7 @@ class BrightcoveNewIE(InfoExtractor):
# Look for iframe embeds [1]
for _, url in re.findall(
r'<iframe[^>]+src=(["\'])((?:https?:)//players\.brightcove\.net/\d+/[^/]+/index\.html.+?)\1', webpage):
- entries.append(self.url_result(self._proto_relative_url(url)))
+ entries.append(url)
# Look for embed_in_page embeds [2]
# According to examples from [3] it's unclear whether video id may be optional
# and what to do when it is
@@ -396,9 +397,9 @@ class BrightcoveNewIE(InfoExtractor):
src=["\'](?:https?:)?//players\.brightcove\.net/
(\d+)/([\da-f-]+)_([^/]+)/index\.min\.js
''', webpage):
- entries.append(self.url_result(
+ entries.append(
'http://players.brightcove.net/%s/%s_%s/index.html?videoId=%s'
- % (account_id, player_id, embed, video_id)))
+ % (account_id, player_id, embed, video_id))
return entries
def _real_extract(self, url):
From f6519f89b09be788549f68ba12f0cc31c55d9751 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Nov 2015 06:03:07 +0600
Subject: [PATCH 256/415] [generic] Extract Brightcove New Studio embeds
---
youtube_dl/extractor/generic.py | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 0797e1a90..334864db3 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1321,10 +1321,10 @@ class GenericIE(InfoExtractor):
'entries': entries,
}
- # Look for Brightcove In Page Embed:
- brightcove_in_page_embed_url = BrightcoveNewIE._extract_url(webpage)
- if brightcove_in_page_embed_url:
- return self.url_result(brightcove_in_page_embed_url, 'BrightcoveInPageEmbed')
+ # Look for Brightcove New Studio embeds
+ bc_urls = BrightcoveNewIE._extract_urls(webpage)
+ if bc_urls:
+ return _playlist_from_matches(bc_urls, ie='BrightcoveNew')
# Look for embedded rtl.nl player
matches = re.findall(
From 1f4b722b00fd5c24468cd4d072e8b5c5428ca515 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Nov 2015 06:03:32 +0600
Subject: [PATCH 257/415] [generic] Clarify Brightcove Legacy Studio comment
---
youtube_dl/extractor/generic.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 334864db3..8ba0a9913 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1304,7 +1304,7 @@ class GenericIE(InfoExtractor):
return self.playlist_result(
urlrs, playlist_id=video_id, playlist_title=video_title)
- # Look for BrightCove:
+ # Look for Brightcove Legacy Studio embeds
bc_urls = BrightcoveLegacyIE._extract_brightcove_urls(webpage)
if bc_urls:
self.to_screen('Brightcove video detected.')
From 3b7d9aa487399e06bba5dc03c90b6576c2b067b9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Nov 2015 06:05:46 +0600
Subject: [PATCH 258/415] Rename all references to legacy studio Brightcove
extractor
---
youtube_dl/extractor/aljazeera.py | 4 ++--
youtube_dl/extractor/generic.py | 8 ++++----
youtube_dl/extractor/nowness.py | 2 +-
youtube_dl/extractor/safari.py | 2 +-
youtube_dl/extractor/space.py | 2 +-
5 files changed, 9 insertions(+), 9 deletions(-)
diff --git a/youtube_dl/extractor/aljazeera.py b/youtube_dl/extractor/aljazeera.py
index 184a14a4f..5b2c0dc9a 100644
--- a/youtube_dl/extractor/aljazeera.py
+++ b/youtube_dl/extractor/aljazeera.py
@@ -15,7 +15,7 @@ class AlJazeeraIE(InfoExtractor):
'description': 'As a birth attendant advocating for family planning, Remy is on the frontline of Tondo\'s battle with overcrowding.',
'uploader': 'Al Jazeera English',
},
- 'add_ie': ['Brightcove'],
+ 'add_ie': ['BrightcoveLegacy'],
'skip': 'Not accessible from Travis CI server',
}
@@ -32,5 +32,5 @@ class AlJazeeraIE(InfoExtractor):
'playerKey=AQ~~%2CAAAAmtVJIFk~%2CTVGOQ5ZTwJbeMWnq5d_H4MOM57xfzApc'
'&%40videoPlayer={0}'.format(brightcove_id)
),
- 'ie_key': 'Brightcove',
+ 'ie_key': 'BrightcoveLegacy',
}
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 8ba0a9913..51516a38a 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -278,7 +278,7 @@ class GenericIE(InfoExtractor):
# it also tests brightcove videos that need to set the 'Referer' in the
# http requests
{
- 'add_ie': ['Brightcove'],
+ 'add_ie': ['BrightcoveLegacy'],
'url': 'http://www.bfmtv.com/video/bfmbusiness/cours-bourse/cours-bourse-l-analyse-technique-154522/',
'info_dict': {
'id': '2765128793001',
@@ -302,7 +302,7 @@ class GenericIE(InfoExtractor):
'uploader': 'thestar.com',
'description': 'Mississauga resident David Farmer is still out of power as a result of the ice storm a month ago. To keep the house warm, Farmer cuts wood from his property for a wood burning stove downstairs.',
},
- 'add_ie': ['Brightcove'],
+ 'add_ie': ['BrightcoveLegacy'],
},
{
'url': 'http://www.championat.com/video/football/v/87/87499.html',
@@ -317,7 +317,7 @@ class GenericIE(InfoExtractor):
},
{
# https://github.com/rg3/youtube-dl/issues/3541
- 'add_ie': ['Brightcove'],
+ 'add_ie': ['BrightcoveLegacy'],
'url': 'http://www.kijk.nl/sbs6/leermijvrouwenkennen/videos/jqMiXKAYan2S/aflevering-1',
'info_dict': {
'id': '3866516442001',
@@ -1311,7 +1311,7 @@ class GenericIE(InfoExtractor):
entries = [{
'_type': 'url',
'url': smuggle_url(bc_url, {'Referer': url}),
- 'ie_key': 'Brightcove'
+ 'ie_key': 'BrightcoveLegacy'
} for bc_url in bc_urls]
return {
diff --git a/youtube_dl/extractor/nowness.py b/youtube_dl/extractor/nowness.py
index dab487ea4..0fba55833 100644
--- a/youtube_dl/extractor/nowness.py
+++ b/youtube_dl/extractor/nowness.py
@@ -25,7 +25,7 @@ class NownessBaseIE(InfoExtractor):
bc_url = BrightcoveLegacyIE._extract_brightcove_url(player_code)
if bc_url is None:
raise ExtractorError('Could not find player definition')
- return self.url_result(bc_url, 'Brightcove')
+ return self.url_result(bc_url, 'BrightcoveLegacy')
elif source == 'vimeo':
return self.url_result('http://vimeo.com/%s' % video_id, 'Vimeo')
elif source == 'youtube':
diff --git a/youtube_dl/extractor/safari.py b/youtube_dl/extractor/safari.py
index 4f1f05c6a..e9e33d0a3 100644
--- a/youtube_dl/extractor/safari.py
+++ b/youtube_dl/extractor/safari.py
@@ -116,7 +116,7 @@ class SafariIE(SafariBaseIE):
if not bc_url:
raise ExtractorError('Could not extract Brightcove URL from %s' % url, expected=True)
- return self.url_result(smuggle_url(bc_url, {'Referer': url}), 'Brightcove')
+ return self.url_result(smuggle_url(bc_url, {'Referer': url}), 'BrightcoveLegacy')
class SafariCourseIE(SafariBaseIE):
diff --git a/youtube_dl/extractor/space.py b/youtube_dl/extractor/space.py
index 2f190f764..ebb5d6ec0 100644
--- a/youtube_dl/extractor/space.py
+++ b/youtube_dl/extractor/space.py
@@ -10,7 +10,7 @@ from ..utils import RegexNotFoundError, ExtractorError
class SpaceIE(InfoExtractor):
_VALID_URL = r'https?://(?:(?:www|m)\.)?space\.com/\d+-(?P<title>[^/\.\?]*?)-video\.html'
_TEST = {
- 'add_ie': ['Brightcove'],
+ 'add_ie': ['BrightcoveLegacy'],
'url': 'http://www.space.com/23373-huge-martian-landforms-detail-revealed-by-european-probe-video.html',
'info_dict': {
'id': '2780937028001',
From 75eac8961ee2ff004891ec57d5a2fec4f0b5574d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Nov 2015 06:07:24 +0600
Subject: [PATCH 259/415] [brightcove] Remove unused import
---
youtube_dl/extractor/brightcove.py | 1 -
1 file changed, 1 deletion(-)
diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 1c7783dcb..ef34ae48f 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -26,7 +26,6 @@ from ..utils import (
float_or_none,
int_or_none,
parse_iso8601,
- extract_attributes,
)
From c7b959ce383040f1d507eef0e43041029583b307 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Nov 2015 06:07:44 +0600
Subject: [PATCH 260/415] [utils] Remove unused function
---
youtube_dl/utils.py | 9 ---------
1 file changed, 9 deletions(-)
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 65556d056..d39f313a4 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -259,15 +259,6 @@ def get_element_by_attribute(attribute, value, html):
return unescapeHTML(res)
-def extract_attributes(attributes_str, attributes_regex=r'(?s)\s*([^\s=]+)\s*=\s*["\']([^"\']+)["\']'):
- attributes = re.findall(attributes_regex, attributes_str)
- attributes_dict = {}
- if attributes:
- for (attribute_name, attribute_value) in attributes:
- attributes_dict[attribute_name] = attribute_value
- return attributes_dict
-
-
def clean_html(html):
"""Clean an HTML snippet into a readable string"""
From fd91257c4019a1956cc59eac1232f2c413b9747d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Nov 2015 06:08:36 +0600
Subject: [PATCH 261/415] [brightcove] Order imports alphabetically
---
youtube_dl/extractor/brightcove.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index ef34ae48f..f137ba8c6 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -20,12 +20,12 @@ from ..utils import (
ExtractorError,
find_xpath_attr,
fix_xml_ampersands,
- unescapeHTML,
- unsmuggle_url,
- js_to_json,
float_or_none,
+ js_to_json,
int_or_none,
parse_iso8601,
+ unescapeHTML,
+ unsmuggle_url,
)
From e01b432ad38b36b1ba6cb1b6dccecec51f9fc1e7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Nov 2015 06:11:17 +0600
Subject: [PATCH 262/415] [brightcove:new] Fix test
---
youtube_dl/extractor/brightcove.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index f137ba8c6..6b184157c 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -366,7 +366,7 @@ class BrightcoveNewIE(InfoExtractor):
'description': 'md5:eac376a4fe366edc70279bfb681aea16',
'timestamp': 1441391203,
'upload_date': '20150904',
- 'duration': 165768,
+ 'duration': 165.768,
'uploader_id': '929656772001',
}
}
From cb33d389ed452583638f73360599642b86617dfb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Nov 2015 06:20:09 +0600
Subject: [PATCH 263/415] [brightcove:new] Add test with rtmp streams
---
youtube_dl/extractor/brightcove.py | 24 +++++++++++++++++++++---
1 file changed, 21 insertions(+), 3 deletions(-)
diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 6b184157c..9ae724c2b 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -356,7 +356,7 @@ class BrightcoveLegacyIE(InfoExtractor):
class BrightcoveNewIE(InfoExtractor):
IE_NAME = 'brightcove:new'
_VALID_URL = r'https?://players\.brightcove\.net/(?P<account_id>\d+)/(?P<player_id>[^/]+)_(?P<embed>[^/]+)/index\.html\?.*videoId=(?P<video_id>\d+)'
- _TEST = {
+ _TESTS = [{
'url': 'http://players.brightcove.net/929656772001/e41d32dc-ec74-459e-a845-6c69f7b724ea_default/index.html?videoId=4463358922001',
'md5': 'c8100925723840d4b0d243f7025703be',
'info_dict': {
@@ -364,12 +364,30 @@ class BrightcoveNewIE(InfoExtractor):
'ext': 'mp4',
'title': 'Meet the man behind Popcorn Time',
'description': 'md5:eac376a4fe366edc70279bfb681aea16',
+ 'duration': 165.768,
'timestamp': 1441391203,
'upload_date': '20150904',
- 'duration': 165.768,
'uploader_id': '929656772001',
+ 'formats': 'mincount:22',
+ },
+ }, {
+ # with rtmp streams
+ 'url': 'http://players.brightcove.net/4036320279001/5d112ed9-283f-485f-a7f9-33f42e8bc042_default/index.html?videoId=4279049078001',
+ 'info_dict': {
+ 'id': '4279049078001',
+ 'ext': 'mp4',
+ 'title': 'Titansgrave: Chapter 0',
+ 'description': 'Titansgrave: Chapter 0',
+ 'duration': 1242.058,
+ 'timestamp': 1433556729,
+ 'upload_date': '20150606',
+ 'uploader_id': '4036320279001',
+ 'formats': 'mincount:41',
+ },
+ 'params': {
+ 'skip_download': True,
}
- }
+ }]
@staticmethod
def _extract_urls(webpage):
From d8a1caf04f1733d7c94f213af25c587addcea35b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Nov 2015 06:22:12 +0600
Subject: [PATCH 264/415] [brightcove:new] Style
---
youtube_dl/extractor/brightcove.py | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 9ae724c2b..14ee05f21 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -402,10 +402,11 @@ class BrightcoveNewIE(InfoExtractor):
for _, url in re.findall(
r'<iframe[^>]+src=(["\'])((?:https?:)//players\.brightcove\.net/\d+/[^/]+/index\.html.+?)\1', webpage):
entries.append(url)
+
# Look for embed_in_page embeds [2]
- # According to examples from [3] it's unclear whether video id may be optional
- # and what to do when it is
for video_id, account_id, player_id, embed in re.findall(
+ # According to examples from [3] it's unclear whether video id
+ # may be optional and what to do when it is
r'''(?sx)
<video[^>]+
data-video-id=["\'](\d+)["\'][^>]*>.*?
@@ -417,6 +418,7 @@ class BrightcoveNewIE(InfoExtractor):
entries.append(
'http://players.brightcove.net/%s/%s_%s/index.html?videoId=%s'
% (account_id, player_id, embed, video_id))
+
return entries
def _real_extract(self, url):
From a90189c3ad9bab0571121ecb70baea89ccac154e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Nov 2015 07:20:33 +0600
Subject: [PATCH 265/415] [instagram] Relax _VALID_URL (Closes #7497)
---
youtube_dl/extractor/instagram.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/instagram.py b/youtube_dl/extractor/instagram.py
index 3d78f78c4..ddcaf76d1 100644
--- a/youtube_dl/extractor/instagram.py
+++ b/youtube_dl/extractor/instagram.py
@@ -10,7 +10,7 @@ from ..utils import (
class InstagramIE(InfoExtractor):
- _VALID_URL = r'https://instagram\.com/p/(?P<id>[\da-zA-Z]+)'
+ _VALID_URL = r'https://instagram\.com/p/(?P<id>[^/?#&]+)'
_TEST = {
'url': 'https://instagram.com/p/aye83DjauH/?foo=bar#abc',
'md5': '0d2da106a9d2631273e192b372806516',
From 4479600d57d1a48de5a910764e2d18c30d7a4bd9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Nov 2015 07:21:20 +0600
Subject: [PATCH 266/415] [instagram] Add test for #7497
---
youtube_dl/extractor/instagram.py | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git a/youtube_dl/extractor/instagram.py b/youtube_dl/extractor/instagram.py
index ddcaf76d1..fce179000 100644
--- a/youtube_dl/extractor/instagram.py
+++ b/youtube_dl/extractor/instagram.py
@@ -11,7 +11,7 @@ from ..utils import (
class InstagramIE(InfoExtractor):
_VALID_URL = r'https://instagram\.com/p/(?P<id>[^/?#&]+)'
- _TEST = {
+ _TESTS = [{
'url': 'https://instagram.com/p/aye83DjauH/?foo=bar#abc',
'md5': '0d2da106a9d2631273e192b372806516',
'info_dict': {
@@ -21,7 +21,10 @@ class InstagramIE(InfoExtractor):
'title': 'Video by naomipq',
'description': 'md5:1f17f0ab29bd6fe2bfad705f58de3cb8',
}
- }
+ }, {
+ 'url': 'https://instagram.com/p/-Cmh1cukG2/',
+ 'only_matching': True,
+ }]
def _real_extract(self, url):
video_id = self._match_id(url)
From 5f1b2aea8051a34dd543ba0fd17e351a8b7bf858 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 14 Nov 2015 17:02:07 +0800
Subject: [PATCH 267/415] [twitter:card] Support vine.co embeds (closes #7496)
---
youtube_dl/extractor/twitter.py | 23 ++++++++++++++++++-----
1 file changed, 18 insertions(+), 5 deletions(-)
diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index 2bd5946ac..afbe8a651 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -52,6 +52,19 @@ class TwitterCardIE(InfoExtractor):
'uploader': 'OMG! Ubuntu!',
'uploader_id': 'omgubuntu',
},
+ },
+ {
+ 'url': 'https://twitter.com/i/cards/tfw/v1/665289828897005568',
+ 'md5': 'ab2745d0b0ce53319a534fccaa986439',
+ 'info_dict': {
+ 'id': 'iBb2x00UVlv',
+ 'ext': 'mp4',
+ 'upload_date': '20151113',
+ 'uploader_id': '1189339351084113920',
+ 'uploader': '@ArsenalTerje',
+ 'title': 'Vine by @ArsenalTerje',
+ },
+ 'add_ie': ['Vine'],
}
]
@@ -71,11 +84,11 @@ class TwitterCardIE(InfoExtractor):
request.add_header('User-Agent', user_agent)
webpage = self._download_webpage(request, video_id)
- youtube_url = self._html_search_regex(
- r'<iframe[^>]+src="((?:https?:)?//www.youtube.com/embed/[^"]+)"',
- webpage, 'youtube iframe', default=None)
- if youtube_url:
- return self.url_result(youtube_url, 'Youtube')
+ iframe_url = self._html_search_regex(
+ r'<iframe[^>]+src="((?:https?:)?//(?:www.youtube.com/embed/[^"]+|(?:www\.)?vine\.co/v/\w+/card))"',
+ webpage, 'video iframe', default=None)
+ if iframe_url:
+ return self.url_result(iframe_url)
config = self._parse_json(self._html_search_regex(
r'data-player-config="([^"]+)"', webpage, 'data player config'),
From 31752f76f76c6ca6f9e515b69a7488103575763e Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 14 Nov 2015 17:03:26 +0800
Subject: [PATCH 268/415] [twitter:card] Add add_ie for the external test
---
youtube_dl/extractor/twitter.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index afbe8a651..055047340 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -52,6 +52,7 @@ class TwitterCardIE(InfoExtractor):
'uploader': 'OMG! Ubuntu!',
'uploader_id': 'omgubuntu',
},
+ 'add_ie': ['Youtube'],
},
{
'url': 'https://twitter.com/i/cards/tfw/v1/665289828897005568',
From 9d584da7d0fd207d46e1592b25244cfcad4354ed Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 14 Nov 2015 17:27:00 +0800
Subject: [PATCH 269/415] [xfileshare] Correct _VALID_URL
---
youtube_dl/extractor/xfileshare.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/xfileshare.py b/youtube_dl/extractor/xfileshare.py
index 7610dc627..952515c98 100644
--- a/youtube_dl/extractor/xfileshare.py
+++ b/youtube_dl/extractor/xfileshare.py
@@ -19,7 +19,7 @@ class XFileShareIE(InfoExtractor):
IE_DESC = 'XFileShare based sites: GorillaVid.in, daclips.in, movpod.in, fastvideo.in, realvid.net, filehoot.com and vidto.me'
_VALID_URL = r'''(?x)
https?://(?P<host>(?:www\.)?
- (?:daclips\.in|gorillavid\.in|movpod\.in|fastvideo\.in|realvid\.net|filehoot\.com|vidto.\me))/
+ (?:daclips\.in|gorillavid\.in|movpod\.in|fastvideo\.in|realvid\.net|filehoot\.com|vidto\.me))/
(?:embed-)?(?P<id>[0-9a-zA-Z]+)(?:-[0-9]+x[0-9]+\.html)?
'''
From 903d1369428f72397dbc698654dbad445ecaf2ac Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Nov 2015 16:43:58 +0600
Subject: [PATCH 270/415] [lynda] Logout only when login info present (Closes
#7500)
---
youtube_dl/extractor/lynda.py | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py
index 9a207b2cd..7d78a8805 100644
--- a/youtube_dl/extractor/lynda.py
+++ b/youtube_dl/extractor/lynda.py
@@ -83,6 +83,10 @@ class LyndaBaseIE(InfoExtractor):
raise ExtractorError('Unable to log in')
def _logout(self):
+ username, _ = self._get_login_info()
+ if username is None:
+ return
+
self._download_webpage(
'http://www.lynda.com/ajax/logout.aspx', None,
'Logging out', 'Unable to log out', fatal=False)
From 0d85c3a7327e75173897d0e212254e496a46ea2d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Nov 2015 16:44:24 +0600
Subject: [PATCH 271/415] [lynda] Style
---
youtube_dl/extractor/lynda.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py
index 7d78a8805..3d7e7e003 100644
--- a/youtube_dl/extractor/lynda.py
+++ b/youtube_dl/extractor/lynda.py
@@ -25,7 +25,7 @@ class LyndaBaseIE(InfoExtractor):
self._login()
def _login(self):
- (username, password) = self._get_login_info()
+ username, password = self._get_login_info()
if username is None:
return
From dcdfeb33d2666ca07d2fb73ebf9284a77e714f69 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Nov 2015 22:32:54 +0600
Subject: [PATCH 272/415] [quickscope] Remove extractor
---
youtube_dl/extractor/periscope.py | 21 ---------------------
1 file changed, 21 deletions(-)
diff --git a/youtube_dl/extractor/periscope.py b/youtube_dl/extractor/periscope.py
index 887c8020d..f4c1b622a 100644
--- a/youtube_dl/extractor/periscope.py
+++ b/youtube_dl/extractor/periscope.py
@@ -81,24 +81,3 @@ class PeriscopeIE(InfoExtractor):
'thumbnails': thumbnails,
'formats': formats,
}
-
-
-class QuickscopeIE(InfoExtractor):
- IE_DESC = 'Quick Scope'
- _VALID_URL = r'https?://watchonperiscope\.com/broadcast/(?P<id>\d+)'
- _TEST = {
- 'url': 'https://watchonperiscope.com/broadcast/56180087',
- 'only_matching': True,
- }
-
- def _real_extract(self, url):
- broadcast_id = self._match_id(url)
- request = compat_urllib_request.Request(
- 'https://watchonperiscope.com/api/accessChannel', compat_urllib_parse.urlencode({
- 'broadcast_id': broadcast_id,
- 'entry_ticket': '',
- 'from_push': 'false',
- 'uses_sessions': 'true',
- }).encode('utf-8'))
- return self.url_result(
- self._download_json(request, broadcast_id)['share_url'], 'Periscope')
From 3b3e8ed332e44003f410d9f97967f0773e41ed1d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Nov 2015 22:34:30 +0600
Subject: [PATCH 273/415] [quickscope] Remove extractor (2)
---
youtube_dl/extractor/__init__.py | 5 +----
1 file changed, 1 insertion(+), 4 deletions(-)
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 64ce3210b..eca4b00f5 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -458,10 +458,7 @@ from .orf import (
from .parliamentliveuk import ParliamentLiveUKIE
from .patreon import PatreonIE
from .pbs import PBSIE
-from .periscope import (
- PeriscopeIE,
- QuickscopeIE,
-)
+from .periscope import PeriscopeIE
from .philharmoniedeparis import PhilharmonieDeParisIE
from .phoenix import PhoenixIE
from .photobucket import PhotobucketIE
From d781e293168cbdf93ee8d76e33b65387b13a2755 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Nov 2015 23:08:13 +0600
Subject: [PATCH 274/415] [bbc] Allow selectionunavailable errors (Closes
#7502)
---
youtube_dl/extractor/bbc.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index a55a6dbc9..33b296eaf 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -27,7 +27,7 @@ class BBCCoUkIE(InfoExtractor):
_MEDIASELECTOR_URLS = [
# Provides HQ HLS streams with even better quality that pc mediaset but fails
# with geolocation in some cases when it's even not geo restricted at all (e.g.
- # http://www.bbc.co.uk/programmes/b06bp7lf)
+ # http://www.bbc.co.uk/programmes/b06bp7lf). Also may fail with selectionunavailable.
'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/iptv-all/vpid/%s',
'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/pc/vpid/%s',
]
@@ -334,7 +334,7 @@ class BBCCoUkIE(InfoExtractor):
return self._download_media_selector_url(
mediaselector_url % programme_id, programme_id)
except BBCCoUkIE.MediaSelectionError as e:
- if e.id in ('notukerror', 'geolocation'):
+ if e.id in ('notukerror', 'geolocation', 'selectionunavailable'):
last_exception = e
continue
self._raise_extractor_error(e)
@@ -345,7 +345,7 @@ class BBCCoUkIE(InfoExtractor):
media_selection = self._download_xml(
url, programme_id, 'Downloading media selection XML')
except ExtractorError as ee:
- if isinstance(ee.cause, compat_HTTPError) and ee.cause.code == 403:
+ if isinstance(ee.cause, compat_HTTPError) and ee.cause.code in (403, 404):
media_selection = compat_etree_fromstring(ee.cause.read().decode('utf-8'))
else:
raise
From 0f72beb5153fbd4780a1c6b52e24acf5f0515874 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
<jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 14 Nov 2015 18:31:33 +0100
Subject: [PATCH 275/415] [periscope] Remove unused imports
---
youtube_dl/extractor/periscope.py | 4 ----
1 file changed, 4 deletions(-)
diff --git a/youtube_dl/extractor/periscope.py b/youtube_dl/extractor/periscope.py
index f4c1b622a..459fa2c43 100644
--- a/youtube_dl/extractor/periscope.py
+++ b/youtube_dl/extractor/periscope.py
@@ -2,10 +2,6 @@
from __future__ import unicode_literals
from .common import InfoExtractor
-from ..compat import (
- compat_urllib_parse,
- compat_urllib_request,
-)
from ..utils import parse_iso8601
From 0c59d02bdc86056957fcbebfec591ad53b7fcda3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 15 Nov 2015 00:20:17 +0600
Subject: [PATCH 276/415] [periscope] Relax _VALID_URL (Closes #7503)
---
youtube_dl/extractor/periscope.py | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/periscope.py b/youtube_dl/extractor/periscope.py
index 459fa2c43..63cc764bb 100644
--- a/youtube_dl/extractor/periscope.py
+++ b/youtube_dl/extractor/periscope.py
@@ -7,7 +7,7 @@ from ..utils import parse_iso8601
class PeriscopeIE(InfoExtractor):
IE_DESC = 'Periscope'
- _VALID_URL = r'https?://(?:www\.)?periscope\.tv/w/(?P<id>[^/?#]+)'
+ _VALID_URL = r'https?://(?:www\.)?periscope\.tv/[^/]+/(?P<id>[^/?#]+)'
# Alive example URLs can be found here http://onperiscope.com/
_TESTS = [{
'url': 'https://www.periscope.tv/w/aJUQnjY3MjA3ODF8NTYxMDIyMDl2zCg2pECBgwTqRpQuQD352EMPTKQjT4uqlM3cgWFA-g==',
@@ -25,6 +25,9 @@ class PeriscopeIE(InfoExtractor):
}, {
'url': 'https://www.periscope.tv/w/1ZkKzPbMVggJv',
'only_matching': True,
+ }, {
+ 'url': 'https://www.periscope.tv/bastaakanoggano/1OdKrlkZZjOJX',
+ 'only_matching': True,
}]
def _call_api(self, method, value):
From dc0279532a218f5f43ca461aacd01058b8bc9e6d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 15 Nov 2015 02:21:24 +0600
Subject: [PATCH 277/415] [dumpert] Disable SSL (Closes #7504)
---
youtube_dl/extractor/dumpert.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/dumpert.py b/youtube_dl/extractor/dumpert.py
index 1f00386fe..8811be985 100644
--- a/youtube_dl/extractor/dumpert.py
+++ b/youtube_dl/extractor/dumpert.py
@@ -28,7 +28,7 @@ class DumpertIE(InfoExtractor):
def _real_extract(self, url):
video_id = self._match_id(url)
- url = 'https://www.dumpert.nl/mediabase/' + video_id
+ url = 'http://www.dumpert.nl/mediabase/' + video_id
req = compat_urllib_request.Request(url)
req.add_header('Cookie', 'nsfw=1; cpc=10')
webpage = self._download_webpage(req, video_id)
From b2f77388305975570d57a4db3ced46aa3856f7a4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 15 Nov 2015 02:25:00 +0600
Subject: [PATCH 278/415] [dumpert] Use original protocol
---
youtube_dl/extractor/dumpert.py | 9 ++++++---
1 file changed, 6 insertions(+), 3 deletions(-)
diff --git a/youtube_dl/extractor/dumpert.py b/youtube_dl/extractor/dumpert.py
index 8811be985..f5a31058d 100644
--- a/youtube_dl/extractor/dumpert.py
+++ b/youtube_dl/extractor/dumpert.py
@@ -2,6 +2,7 @@
from __future__ import unicode_literals
import base64
+import re
from .common import InfoExtractor
from ..compat import compat_urllib_request
@@ -9,7 +10,7 @@ from ..utils import qualities
class DumpertIE(InfoExtractor):
- _VALID_URL = r'https?://(?:www\.)?dumpert\.nl/(?:mediabase|embed)/(?P<id>[0-9]+/[0-9a-zA-Z]+)'
+ _VALID_URL = r'(?P<protocol>https?)://(?:www\.)?dumpert\.nl/(?:mediabase|embed)/(?P<id>[0-9]+/[0-9a-zA-Z]+)'
_TESTS = [{
'url': 'http://www.dumpert.nl/mediabase/6646981/951bc60f/',
'md5': '1b9318d7d5054e7dcb9dc7654f21d643',
@@ -26,9 +27,11 @@ class DumpertIE(InfoExtractor):
}]
def _real_extract(self, url):
- video_id = self._match_id(url)
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+ protocol = mobj.group('protocol')
- url = 'http://www.dumpert.nl/mediabase/' + video_id
+ url = '%s://www.dumpert.nl/mediabase/%s' % (protocol, video_id)
req = compat_urllib_request.Request(url)
req.add_header('Cookie', 'nsfw=1; cpc=10')
webpage = self._download_webpage(req, video_id)
From 2ff7cbeaaabbc8d67caa224a5cef1fbd32761918 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 15 Nov 2015 08:30:13 +0600
Subject: [PATCH 279/415] [nowtv:list] Add extrator (Closes #7147)
---
youtube_dl/extractor/__init__.py | 5 +-
youtube_dl/extractor/nowtv.py | 164 +++++++++++++++++++++----------
2 files changed, 118 insertions(+), 51 deletions(-)
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index eca4b00f5..59c82f65d 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -420,7 +420,10 @@ from .nowness import (
NownessPlaylistIE,
NownessSeriesIE,
)
-from .nowtv import NowTVIE
+from .nowtv import (
+ NowTVIE,
+ NowTVListIE,
+)
from .nowvideo import NowVideoIE
from .npo import (
NPOIE,
diff --git a/youtube_dl/extractor/nowtv.py b/youtube_dl/extractor/nowtv.py
index b0bdffc4e..67e34b294 100644
--- a/youtube_dl/extractor/nowtv.py
+++ b/youtube_dl/extractor/nowtv.py
@@ -1,6 +1,8 @@
# coding: utf-8
from __future__ import unicode_literals
+import re
+
from .common import InfoExtractor
from ..compat import compat_str
from ..utils import (
@@ -13,8 +15,63 @@ from ..utils import (
)
-class NowTVIE(InfoExtractor):
- _VALID_URL = r'https?://(?:www\.)?nowtv\.(?:de|at|ch)/(?:rtl|rtl2|rtlnitro|superrtl|ntv|vox)/(?P<id>.+?)/(?:player|preview)'
+class NowTVBaseIE(InfoExtractor):
+ _VIDEO_FIELDS = (
+ 'id', 'title', 'free', 'geoblocked', 'articleLong', 'articleShort',
+ 'broadcastStartDate', 'seoUrl', 'duration', 'files',
+ 'format.defaultImage169Format', 'format.defaultImage169Logo')
+
+ def _extract_video(self, info, display_id=None):
+ video_id = compat_str(info['id'])
+
+ files = info['files']
+ if not files:
+ if info.get('geoblocked', False):
+ raise ExtractorError(
+ 'Video %s is not available from your location due to geo restriction' % video_id,
+ expected=True)
+ if not info.get('free', True):
+ raise ExtractorError(
+ 'Video %s is not available for free' % video_id, expected=True)
+
+ formats = []
+ for item in files['items']:
+ if determine_ext(item['path']) != 'f4v':
+ continue
+ app, play_path = remove_start(item['path'], '/').split('/', 1)
+ formats.append({
+ 'url': 'rtmpe://fms.rtl.de',
+ 'app': app,
+ 'play_path': 'mp4:%s' % play_path,
+ 'ext': 'flv',
+ 'page_url': 'http://rtlnow.rtl.de',
+ 'player_url': 'http://cdn.static-fra.de/now/vodplayer.swf',
+ 'tbr': int_or_none(item.get('bitrate')),
+ })
+ self._sort_formats(formats)
+
+ title = info['title']
+ description = info.get('articleLong') or info.get('articleShort')
+ timestamp = parse_iso8601(info.get('broadcastStartDate'), ' ')
+ duration = parse_duration(info.get('duration'))
+
+ f = info.get('format', {})
+ thumbnail = f.get('defaultImage169Format') or f.get('defaultImage169Logo')
+
+ return {
+ 'id': video_id,
+ 'display_id': display_id or info.get('seoUrl'),
+ 'title': title,
+ 'description': description,
+ 'thumbnail': thumbnail,
+ 'timestamp': timestamp,
+ 'duration': duration,
+ 'formats': formats,
+ }
+
+
+class NowTVIE(NowTVBaseIE):
+ _VALID_URL = r'https?://(?:www\.)?nowtv\.(?:de|at|ch)/(?:rtl|rtl2|rtlnitro|superrtl|ntv|vox)/(?P<show_id>[^/]+)/(?:list/[^/]+/)?(?P<id>[^/]+)/(?:player|preview)'
_TESTS = [{
# rtl
@@ -23,7 +80,7 @@ class NowTVIE(InfoExtractor):
'id': '203519',
'display_id': 'bauer-sucht-frau/die-neuen-bauern-und-eine-hochzeit',
'ext': 'flv',
- 'title': 'Die neuen Bauern und eine Hochzeit',
+ 'title': 'Inka Bause stellt die neuen Bauern vor',
'description': 'md5:e234e1ed6d63cf06be5c070442612e7e',
'thumbnail': 're:^https?://.*\.jpg$',
'timestamp': 1432580700,
@@ -136,58 +193,65 @@ class NowTVIE(InfoExtractor):
}]
def _real_extract(self, url):
- display_id = self._match_id(url)
- display_id_split = display_id.split('/')
- if len(display_id) > 2:
- display_id = '/'.join((display_id_split[0], display_id_split[-1]))
+ mobj = re.match(self._VALID_URL, url)
+ display_id = '%s/%s' % (mobj.group('show_id'), mobj.group('id'))
info = self._download_json(
- 'https://api.nowtv.de/v3/movies/%s?fields=id,title,free,geoblocked,articleLong,articleShort,broadcastStartDate,seoUrl,duration,format,files' % display_id,
- display_id)
+ 'https://api.nowtv.de/v3/movies/%s?fields=%s'
+ % (display_id, ','.join(self._VIDEO_FIELDS)), display_id)
- video_id = compat_str(info['id'])
+ return self._extract_video(info, display_id)
- files = info['files']
- if not files:
- if info.get('geoblocked', False):
- raise ExtractorError(
- 'Video %s is not available from your location due to geo restriction' % video_id,
- expected=True)
- if not info.get('free', True):
- raise ExtractorError(
- 'Video %s is not available for free' % video_id, expected=True)
- formats = []
- for item in files['items']:
- if determine_ext(item['path']) != 'f4v':
- continue
- app, play_path = remove_start(item['path'], '/').split('/', 1)
- formats.append({
- 'url': 'rtmpe://fms.rtl.de',
- 'app': app,
- 'play_path': 'mp4:%s' % play_path,
- 'ext': 'flv',
- 'page_url': 'http://rtlnow.rtl.de',
- 'player_url': 'http://cdn.static-fra.de/now/vodplayer.swf',
- 'tbr': int_or_none(item.get('bitrate')),
- })
- self._sort_formats(formats)
+class NowTVListIE(NowTVBaseIE):
+ _VALID_URL = r'https?://(?:www\.)?nowtv\.(?:de|at|ch)/(?:rtl|rtl2|rtlnitro|superrtl|ntv|vox)/(?P<show_id>[^/]+)/list/(?P<id>[^?/#&]+)$'
- title = info['title']
- description = info.get('articleLong') or info.get('articleShort')
- timestamp = parse_iso8601(info.get('broadcastStartDate'), ' ')
- duration = parse_duration(info.get('duration'))
+ _SHOW_FIELDS = ('title', )
+ _SEASON_FIELDS = ('id', 'headline', 'seoheadline', )
- f = info.get('format', {})
- thumbnail = f.get('defaultImage169Format') or f.get('defaultImage169Logo')
+ _TESTS = [{
+ 'url': 'http://www.nowtv.at/rtl/stern-tv/list/aktuell',
+ 'info_dict': {
+ 'id': '17006',
+ 'title': 'stern TV - Aktuell',
+ },
+ 'playlist_count': 1,
+ }, {
+ 'url': 'http://www.nowtv.at/rtl/das-supertalent/list/free-staffel-8',
+ 'info_dict': {
+ 'id': '20716',
+ 'title': 'Das Supertalent - FREE Staffel 8',
+ },
+ 'playlist_count': 14,
+ }]
- return {
- 'id': video_id,
- 'display_id': display_id,
- 'title': title,
- 'description': description,
- 'thumbnail': thumbnail,
- 'timestamp': timestamp,
- 'duration': duration,
- 'formats': formats,
- }
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ show_id = mobj.group('show_id')
+ season_id = mobj.group('id')
+
+ fields = []
+ fields.extend(self._SHOW_FIELDS)
+ fields.extend('formatTabs.%s' % field for field in self._SEASON_FIELDS)
+ fields.extend(
+ 'formatTabs.formatTabPages.container.movies.%s' % field
+ for field in self._VIDEO_FIELDS)
+
+ list_info = self._download_json(
+ 'https://api.nowtv.de/v3/formats/seo?fields=%s&name=%s.php'
+ % (','.join(fields), show_id),
+ season_id)
+
+ season = next(
+ season for season in list_info['formatTabs']['items']
+ if season.get('seoheadline') == season_id)
+
+ title = '%s - %s' % (list_info['title'], season['headline'])
+
+ entries = []
+ for container in season['formatTabPages']['items']:
+ for info in ((container.get('container') or {}).get('movies') or {}).get('items') or []:
+ entries.append(self._extract_video(info))
+
+ return self.playlist_result(
+ entries, compat_str(season.get('id') or season_id), title)
From 828b2a5cd971ae04469aeb7b11dfeaab7962c4d9 Mon Sep 17 00:00:00 2001
From: David Ben Zakai <david.benzakai@gmail.com>
Date: Sun, 15 Nov 2015 09:40:32 +0200
Subject: [PATCH 280/415] Removing an unnecessary import
---
youtube_dl/update.py | 5 +----
1 file changed, 1 insertion(+), 4 deletions(-)
diff --git a/youtube_dl/update.py b/youtube_dl/update.py
index 04bf0939e..074eb64a7 100644
--- a/youtube_dl/update.py
+++ b/youtube_dl/update.py
@@ -9,10 +9,7 @@ import subprocess
import sys
from zipimport import zipimporter
-from .compat import (
- compat_str,
- compat_urllib_request,
-)
+from .compat import compat_str
from .version import __version__
From ad1f4e7902464532ba11f3a09ac6ff2312b01661 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 15 Nov 2015 23:43:23 +0600
Subject: [PATCH 281/415] [theplatform] Handle explicitly specified SMIL
(#7385)
---
youtube_dl/extractor/theplatform.py | 9 +++++++++
1 file changed, 9 insertions(+)
diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index 25edc3100..114c70b7c 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -193,6 +193,15 @@ class ThePlatformIE(ThePlatformBaseIE):
if smuggled_data.get('force_smil_url', False):
smil_url = url
+ # Explicitly specified SMIL (see https://github.com/rg3/youtube-dl/issues/7385)
+ elif '/guid/' in url:
+ webpage = self._download_webpage(url, video_id)
+ smil_url = self._search_regex(
+ r'<link[^>]+href=(["\'])(?P<url>.+?)\1[^>]+type=["\']application/smil\+xml',
+ webpage, 'smil url', group='url')
+ path = self._search_regex(
+ r'link\.theplatform\.com/s/((?:[^/?#&]+/)+[^/?#&]+)', smil_url, 'path')
+ smil_url += '?' if '?' not in smil_url else '&' + 'formats=m3u,mpeg4&format=SMIL'
elif mobj.group('config'):
config_url = url + '&form=json'
config_url = config_url.replace('swf/', 'config/')
From 9a4acbfaf58f7334bdd8a3b66dadf331f1ce1d7b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 16 Nov 2015 00:28:04 +0600
Subject: [PATCH 282/415] [theplatform] Add test for #7385
---
youtube_dl/extractor/theplatform.py | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index 114c70b7c..43315e75d 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -139,6 +139,11 @@ class ThePlatformIE(ThePlatformBaseIE):
'upload_date': '20150701',
'categories': ['Today/Shows/Orange Room', 'Today/Sections/Money', 'Today/Topics/Tech', "Today/Topics/Editor's picks"],
},
+ }, {
+ # From http://www.nbc.com/the-blacklist/video/sir-crispin-crandall/2928790?onid=137781#vc137781=1
+ # geo-restricted (US), HLS encrypted with AES-128
+ 'url': 'http://player.theplatform.com/p/NnzsPC/onsite_universal/select/media/guid/2410887629/2928790?fwsitesection=nbc_the_blacklist_video_library&autoPlay=true&carouselID=137781',
+ 'only_matching': True,
}]
@staticmethod
From bd1512d19649c280197729814766d590ea6c023b Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 15 Nov 2015 22:16:08 +0100
Subject: [PATCH 283/415] release 2015.11.15
---
docs/supportedsites.md | 5 +++--
youtube_dl/version.py | 2 +-
2 files changed, 4 insertions(+), 3 deletions(-)
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 5016ba4bc..2e5283747 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -67,7 +67,8 @@
- **Bpb**: Bundeszentrale für politische Bildung
- **BR**: Bayerischer Rundfunk Mediathek
- **Break**
- - **Brightcove**
+ - **brightcove:legacy**
+ - **brightcove:new**
- **bt:article**: Bergens Tidende Articles
- **bt:vestlendingen**: Bergens Tidende - Vestlendingen
- **BuzzFeed**
@@ -366,6 +367,7 @@
- **nowness:playlist**
- **nowness:series**
- **NowTV**
+ - **NowTVList**
- **nowvideo**: NowVideo
- **npo**: npo.nl and ntr.nl
- **npo.nl:live**
@@ -425,7 +427,6 @@
- **qqmusic:playlist**: QQ音乐 - 歌单
- **qqmusic:singer**: QQ音乐 - 歌手
- **qqmusic:toplist**: QQ音乐 - 排行榜
- - **Quickscope**: Quick Scope
- **QuickVid**
- **R7**
- **radio.de**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 6585d60d5..6f601cbb1 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
from __future__ import unicode_literals
-__version__ = '2015.11.13'
+__version__ = '2015.11.15'
From 76adc820682b78b67ffc9cb1544807f7e021e73b Mon Sep 17 00:00:00 2001
From: ping <lipng.ong@gmail.com>
Date: Mon, 16 Nov 2015 11:39:18 +0800
Subject: [PATCH 284/415] [neteasemusic] Fixes #7301
---
youtube_dl/extractor/neteasemusic.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/neteasemusic.py b/youtube_dl/extractor/neteasemusic.py
index a8e0a64ed..bb3362069 100644
--- a/youtube_dl/extractor/neteasemusic.py
+++ b/youtube_dl/extractor/neteasemusic.py
@@ -40,7 +40,7 @@ class NetEaseMusicBaseIE(InfoExtractor):
if not details:
continue
formats.append({
- 'url': 'http://m1.music.126.net/%s/%s.%s' %
+ 'url': 'http://m5.music.126.net/%s/%s.%s' %
(cls._encrypt(details['dfsId']), details['dfsId'],
details['extension']),
'ext': details.get('extension'),
From 741dd8ea65b276997f3eadae43b2879e9a229a80 Mon Sep 17 00:00:00 2001
From: Rastislav Barlik <barlik@zoho.com>
Date: Mon, 16 Nov 2015 14:15:25 +0000
Subject: [PATCH 285/415] Clarify that automatic subtitles are generated.
It wasn't clear what automatic word mean.
---
README.md | 4 ++--
youtube_dl/YoutubeDL.py | 2 +-
youtube_dl/options.py | 2 +-
3 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/README.md b/README.md
index 38db97c59..b286651cd 100644
--- a/README.md
+++ b/README.md
@@ -329,8 +329,8 @@ which means you can modify it, redistribute it or use it however you like.
## Subtitle Options:
--write-sub Write subtitle file
- --write-auto-sub Write automatic subtitle file (YouTube
- only)
+ --write-auto-sub Write automatically generated subtitle file
+ (YouTube only)
--all-subs Download all the available subtitles of the
video
--list-subs List all available subtitles for the video
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 1783ce01b..422f3ffca 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -156,7 +156,7 @@ class YoutubeDL(object):
writethumbnail: Write the thumbnail image to a file
write_all_thumbnails: Write all thumbnail formats to files
writesubtitles: Write the video subtitles to a file
- writeautomaticsub: Write the automatic subtitles to a file
+ writeautomaticsub: Write the automatically generated subtitles to a file
allsubtitles: Downloads all the subtitles of the video
(requires writesubtitles or writeautomaticsub)
listsubtitles: Lists all available subtitles for the video
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 3dd6d290b..079fe7e8a 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -363,7 +363,7 @@ def parseOpts(overrideArguments=None):
subtitles.add_option(
'--write-auto-sub', '--write-automatic-sub',
action='store_true', dest='writeautomaticsub', default=False,
- help='Write automatic subtitle file (YouTube only)')
+ help='Write automatically generated subtitle file (YouTube only)')
subtitles.add_option(
'--all-subs',
action='store_true', dest='allsubtitles', default=False,
From 7aefc49c4013efb5056b2c1237e22c52cb5d3c49 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 16 Nov 2015 20:20:16 +0600
Subject: [PATCH 286/415] [utils] Skip invalid/non HTML entities (Closes #7518)
---
test/test_utils.py | 4 ++--
youtube_dl/utils.py | 6 +++++-
2 files changed, 7 insertions(+), 3 deletions(-)
diff --git a/test/test_utils.py b/test/test_utils.py
index 01829f71e..ea1ff0547 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -210,8 +210,8 @@ class TestUtil(unittest.TestCase):
self.assertEqual(unescapeHTML('%20;'), '%20;')
self.assertEqual(unescapeHTML('/'), '/')
self.assertEqual(unescapeHTML('/'), '/')
- self.assertEqual(
- unescapeHTML('é'), 'é')
+ self.assertEqual(unescapeHTML('é'), 'é')
+ self.assertEqual(unescapeHTML('�'), '�')
def test_daterange(self):
_20century = DateRange("19000101", "20000101")
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index d39f313a4..b7013a6aa 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -396,7 +396,11 @@ def _htmlentity_transform(entity):
numstr = '0%s' % numstr
else:
base = 10
- return compat_chr(int(numstr, base))
+ # See https://github.com/rg3/youtube-dl/issues/7518
+ try:
+ return compat_chr(int(numstr, base))
+ except ValueError:
+ pass
# Unknown entity in name, return its literal representation
return ('&%s;' % entity)
From 7a3f0c00ad138eef396ae8fd1583fe29b4c4c684 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 16 Nov 2015 20:24:09 +0600
Subject: [PATCH 287/415] [utils] Style
---
youtube_dl/utils.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index b7013a6aa..d00b14b86 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -403,7 +403,7 @@ def _htmlentity_transform(entity):
pass
# Unknown entity in name, return its literal representation
- return ('&%s;' % entity)
+ return '&%s;' % entity
def unescapeHTML(s):
From 9b464929fe62d405daccf866cdfdfa6257208839 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 17 Nov 2015 21:11:42 +0600
Subject: [PATCH 288/415] [rtve.es:alacarta] Fix extraction
---
youtube_dl/extractor/rtve.py | 8 +-------
1 file changed, 1 insertion(+), 7 deletions(-)
diff --git a/youtube_dl/extractor/rtve.py b/youtube_dl/extractor/rtve.py
index 5b97d33ca..81fcfedc7 100644
--- a/youtube_dl/extractor/rtve.py
+++ b/youtube_dl/extractor/rtve.py
@@ -107,15 +107,9 @@ class RTVEALaCartaIE(InfoExtractor):
png = self._download_webpage(png_request, video_id, 'Downloading url information')
video_url = _decrypt_url(png)
if not video_url.endswith('.f4m'):
- auth_url = video_url.replace(
+ video_url = video_url.replace(
'resources/', 'auth/resources/'
).replace('.net.rtve', '.multimedia.cdn.rtve')
- video_path = self._download_webpage(
- auth_url, video_id, 'Getting video url')
- # Use mvod1.akcdn instead of flash.akamaihd.multimedia.cdn to get
- # the right Content-Length header and the mp4 format
- video_url = compat_urlparse.urljoin(
- 'http://mvod1.akcdn.rtve.es/', video_path)
subtitles = None
if info.get('sbtFile') is not None:
From e156e70281332b14c3aa47a71fd4a7a16be1a225 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
<jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 17 Nov 2015 16:23:29 +0100
Subject: [PATCH 289/415] [rtve] Remove unused import
---
youtube_dl/extractor/rtve.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/rtve.py b/youtube_dl/extractor/rtve.py
index 81fcfedc7..0fe6356db 100644
--- a/youtube_dl/extractor/rtve.py
+++ b/youtube_dl/extractor/rtve.py
@@ -6,7 +6,7 @@ import re
import time
from .common import InfoExtractor
-from ..compat import compat_urllib_request, compat_urlparse
+from ..compat import compat_urllib_request
from ..utils import (
ExtractorError,
float_or_none,
From 4cd759f73d734882b6d74e764abb068e64a99635 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
<jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 17 Nov 2015 17:52:29 +0100
Subject: [PATCH 290/415] [dplay] Add extractor (closes #7515)
Since I haven't figured out how to download the hds stream, we use the hls one instead.
---
youtube_dl/extractor/__init__.py | 1 +
youtube_dl/extractor/dplay.py | 50 ++++++++++++++++++++++++++++++++
2 files changed, 51 insertions(+)
create mode 100644 youtube_dl/extractor/dplay.py
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 59c82f65d..26e5745d6 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -132,6 +132,7 @@ from .dfb import DFBIE
from .dhm import DHMIE
from .dotsub import DotsubIE
from .douyutv import DouyuTVIE
+from .dplay import DPlayIE
from .dramafever import (
DramaFeverIE,
DramaFeverSeriesIE,
diff --git a/youtube_dl/extractor/dplay.py b/youtube_dl/extractor/dplay.py
new file mode 100644
index 000000000..93768c294
--- /dev/null
+++ b/youtube_dl/extractor/dplay.py
@@ -0,0 +1,50 @@
+from __future__ import unicode_literals
+
+import time
+
+from .common import InfoExtractor
+from ..utils import int_or_none
+
+
+class DPlayIE(InfoExtractor):
+ _VALID_URL = r'http://www\.dplay\.se/[^/]+/(?P<id>[^/?#]+)'
+
+ _TEST = {
+ 'url': 'http://www.dplay.se/nugammalt-77-handelser-som-format-sverige/season-1-svensken-lar-sig-njuta-av-livet/',
+ 'info_dict': {
+ 'id': '3172',
+ 'ext': 'mp4',
+ 'display_id': 'season-1-svensken-lar-sig-njuta-av-livet',
+ 'title': 'Svensken lär sig njuta av livet',
+ 'duration': 2650,
+ },
+ }
+
+ def _real_extract(self, url):
+ display_id = self._match_id(url)
+ webpage = self._download_webpage(url, display_id)
+ video_id = self._search_regex(
+ r'data-video-id="(\d+)"', webpage, 'video id')
+
+ info = self._download_json(
+ 'http://www.dplay.se/api/v2/ajax/videos?video_id=' + video_id,
+ video_id)['data'][0]
+
+ self._set_cookie(
+ 'secure.dplay.se', 'dsc-geo',
+ '{"countryCode":"NL","expiry":%d}' % ((time.time() + 20 * 60) * 1000))
+ # TODO: consider adding support for 'stream_type=hds', it seems to
+ # require setting some cookies
+ manifest_url = self._download_json(
+ 'https://secure.dplay.se/secure/api/v2/user/authorization/stream/%s?stream_type=hls' % video_id,
+ video_id, 'Getting manifest url for hls stream')['hls']
+ formats = self._extract_m3u8_formats(
+ manifest_url, video_id, ext='mp4', entry_protocol='m3u8_native')
+
+ return {
+ 'id': video_id,
+ 'display_id': display_id,
+ 'title': info['title'],
+ 'formats': formats,
+ 'duration': int_or_none(info.get('video_metadata_length'), scale=1000),
+ }
From 609af1ae1ce578b0fdd1dd7ec6e56f1272e107f6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
<jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 17 Nov 2015 17:57:40 +0100
Subject: [PATCH 291/415] [dplay] Add 'encoding: utf-8' line
---
youtube_dl/extractor/dplay.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/youtube_dl/extractor/dplay.py b/youtube_dl/extractor/dplay.py
index 93768c294..6cda56a7f 100644
--- a/youtube_dl/extractor/dplay.py
+++ b/youtube_dl/extractor/dplay.py
@@ -1,3 +1,4 @@
+# encoding: utf-8
from __future__ import unicode_literals
import time
From 312a3f389b4c788556cfcceec1556580c9a9520f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 18 Nov 2015 00:46:41 +0600
Subject: [PATCH 292/415] [pbs] Extend _VALID_URL
---
youtube_dl/extractor/pbs.py | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index 8fb9b1849..33d5c1cf7 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -22,7 +22,7 @@ class PBSIE(InfoExtractor):
# Article with embedded player (or direct video)
(?:www\.)?pbs\.org/(?:[^/]+/){2,5}(?P<presumptive_id>[^/]+?)(?:\.html)?/?(?:$|[?\#]) |
# Player
- video\.pbs\.org/(?:widget/)?partnerplayer/(?P<player_id>[^/]+)/
+ (?:video|player)\.pbs\.org/(?:widget/)?partnerplayer/(?P<player_id>[^/]+)/
)
'''
@@ -170,6 +170,10 @@ class PBSIE(InfoExtractor):
'params': {
'skip_download': True, # requires ffmpeg
},
+ },
+ {
+ 'url': 'http://player.pbs.org/widget/partnerplayer/2365297708/?start=0&end=0&chapterbar=false&endscreen=false&topbar=true',
+ 'only_matching': True,
}
]
_ERRORS = {
From 63b4295d20a5b98a9fca4dc3ce132b26408110d6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
<jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 18 Nov 2015 18:28:05 +0100
Subject: [PATCH 293/415] [youtube:playlist] fix title extraction (fixes #7544
and #7545)
---
youtube_dl/extractor/youtube.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 687e0b4db..364ca102a 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1615,7 +1615,7 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor, YoutubePlaylistBaseInfoExtract
self.report_warning('Youtube gives an alert message: ' + match)
playlist_title = self._html_search_regex(
- r'(?s)<h1 class="pl-header-title[^"]*">\s*(.*?)\s*</h1>',
+ r'(?s)<h1 class="pl-header-title[^"]*"[^>]*>\s*(.*?)\s*</h1>',
page, 'title')
return self.playlist_result(self._entries(page, playlist_id), playlist_id, playlist_title)
From 82beaabb41f015b4aaa1b7460a881a9653e398e9 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 18 Nov 2015 19:23:04 +0100
Subject: [PATCH 294/415] release 2015.11.18
---
docs/supportedsites.md | 1 +
youtube_dl/version.py | 2 +-
2 files changed, 2 insertions(+), 1 deletion(-)
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 2e5283747..f56a6986b 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -129,6 +129,7 @@
- **Discovery**
- **Dotsub**
- **DouyuTV**: 斗鱼
+ - **DPlay**
- **dramafever**
- **dramafever:series**
- **DRBonanza**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 6f601cbb1..b394cb880 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
from __future__ import unicode_literals
-__version__ = '2015.11.15'
+__version__ = '2015.11.18'
From a9c09a7c62df3024b0774de7204e9262bb2d7d4a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 19 Nov 2015 20:25:28 +0600
Subject: [PATCH 295/415] [pbs] Update API URL (Closes #7565)
---
youtube_dl/extractor/pbs.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index 33d5c1cf7..b787e2a73 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -263,7 +263,7 @@ class PBSIE(InfoExtractor):
return self.playlist_result(entries, display_id)
info = self._download_json(
- 'http://video.pbs.org/videoInfo/%s?format=json&type=partner' % video_id,
+ 'http://player.pbs.org/videoInfo/%s?format=json&type=partner' % video_id,
display_id)
formats = []
From c39fd7b1ca3e81901aa28d36523fe698295e58d5 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 19 Nov 2015 22:22:57 +0800
Subject: [PATCH 296/415] [UDNEmbed] Fix generic UDN pages
Closes #7547
---
youtube_dl/extractor/generic.py | 2 +-
youtube_dl/extractor/udn.py | 3 ++-
2 files changed, 3 insertions(+), 2 deletions(-)
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 51516a38a..b483eba65 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1739,7 +1739,7 @@ class GenericIE(InfoExtractor):
# Look for UDN embeds
mobj = re.search(
- r'<iframe[^>]+src="(?P<url>%s)"' % UDNEmbedIE._VALID_URL, webpage)
+ r'<iframe[^>]+src="(?P<url>%s)"' % UDNEmbedIE._PROTOCOL_RELATIVE_VALID_URL, webpage)
if mobj is not None:
return self.url_result(
compat_urlparse.urljoin(url, mobj.group('url')), 'UDNEmbed')
diff --git a/youtube_dl/extractor/udn.py b/youtube_dl/extractor/udn.py
index 2151f8338..ee35b7227 100644
--- a/youtube_dl/extractor/udn.py
+++ b/youtube_dl/extractor/udn.py
@@ -12,7 +12,8 @@ from ..compat import compat_urlparse
class UDNEmbedIE(InfoExtractor):
IE_DESC = '聯合影音'
- _VALID_URL = r'https?://video\.udn\.com/(?:embed|play)/news/(?P<id>\d+)'
+ _PROTOCOL_RELATIVE_VALID_URL = r'//video\.udn\.com/(?:embed|play)/news/(?P<id>\d+)'
+ _VALID_URL = r'https?:' + _PROTOCOL_RELATIVE_VALID_URL
_TESTS = [{
'url': 'http://video.udn.com/embed/news/300040',
'md5': 'de06b4c90b042c128395a88f0384817e',
From e8110b81254075462fd39590dd05d7cd0214cf22 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 19 Nov 2015 15:35:13 +0100
Subject: [PATCH 297/415] release 2015.11.19
---
youtube_dl/version.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index b394cb880..b3b1b90ba 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
from __future__ import unicode_literals
-__version__ = '2015.11.18'
+__version__ = '2015.11.19'
From 2c94198eb62a9ce46a09badfdb232f8b5243a893 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 19 Nov 2015 21:29:32 +0600
Subject: [PATCH 298/415] [vimeo] Improve playlists extraction
---
youtube_dl/extractor/vimeo.py | 19 +++++++++----------
1 file changed, 9 insertions(+), 10 deletions(-)
diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index b72341a2b..cb8c0d484 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -486,8 +486,7 @@ class VimeoChannelIE(VimeoBaseInfoExtractor):
password_request, list_id,
'Verifying the password', 'Wrong password')
- def _extract_videos(self, list_id, base_url):
- video_ids = []
+ def _title_and_entries(self, list_id, base_url):
for pagenum in itertools.count(1):
page_url = self._page_url(base_url, pagenum)
webpage = self._download_webpage(
@@ -496,18 +495,18 @@ class VimeoChannelIE(VimeoBaseInfoExtractor):
if pagenum == 1:
webpage = self._login_list_password(page_url, list_id, webpage)
+ yield self._extract_list_title(webpage)
+
+ for video_id in re.findall(r'id="clip_(\d+?)"', webpage):
+ yield self.url_result('https://vimeo.com/%s' % video_id, 'Vimeo')
- video_ids.extend(re.findall(r'id="clip_(\d+?)"', webpage))
if re.search(self._MORE_PAGES_INDICATOR, webpage, re.DOTALL) is None:
break
- entries = [self.url_result('https://vimeo.com/%s' % video_id, 'Vimeo')
- for video_id in video_ids]
- return {'_type': 'playlist',
- 'id': list_id,
- 'title': self._extract_list_title(webpage),
- 'entries': entries,
- }
+ def _extract_videos(self, list_id, base_url):
+ title_and_entries = self._title_and_entries(list_id, base_url)
+ list_title = next(title_and_entries)
+ return self.playlist_result(title_and_entries, list_id, list_title)
def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url)
From fdb20a27a335dddeebd2e6535a1b9c12c08aa42f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 19 Nov 2015 22:30:58 +0600
Subject: [PATCH 299/415] [vimeo:group] Improve _VALID_URL (Closes #7552)
---
youtube_dl/extractor/vimeo.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index cb8c0d484..836a7312b 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -567,7 +567,7 @@ class VimeoAlbumIE(VimeoChannelIE):
class VimeoGroupsIE(VimeoAlbumIE):
IE_NAME = 'vimeo:group'
- _VALID_URL = r'https://vimeo\.com/groups/(?P<name>[^/]+)'
+ _VALID_URL = r'https://vimeo\.com/groups/(?P<name>[^/]+)(?:/(?!videos?/\d+)|$)'
_TESTS = [{
'url': 'https://vimeo.com/groups/rolexawards',
'info_dict': {
From 6b7ceee1b9124521d2f583fb0d0717999747fdcf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 19 Nov 2015 22:31:16 +0600
Subject: [PATCH 300/415] [vimeo] Add test for #7552
---
youtube_dl/extractor/vimeo.py | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 836a7312b..057c72f39 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -189,6 +189,10 @@ class VimeoIE(VimeoBaseInfoExtractor):
'note': 'Video not completely processed, "failed" seed status',
'only_matching': True,
},
+ {
+ 'url': 'https://vimeo.com/groups/travelhd/videos/22439234',
+ 'only_matching': True,
+ },
]
@staticmethod
From 371c3b796cb4168dbb2cb1ad48e3cf12745a8601 Mon Sep 17 00:00:00 2001
From: hedii <hedi.chaibs@gmail.com>
Date: Tue, 17 Nov 2015 16:41:59 +0100
Subject: [PATCH 301/415] [YoutubeDL] Add playlist finished downloading message
(Closes #7517)
Conflicts:
youtube_dl/YoutubeDL.py
---
youtube_dl/YoutubeDL.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 1783ce01b..9d626049a 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -833,6 +833,7 @@ class YoutubeDL(object):
extra_info=extra)
playlist_results.append(entry_result)
ie_result['entries'] = playlist_results
+ self.to_screen('[download] Finished downloading playlist: %s' % playlist)
return ie_result
elif result_type == 'compat_list':
self.report_warning(
From 342609a1b452c8b8a02e86c3de2ec2a55b6e809b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 19 Nov 2015 22:55:06 +0600
Subject: [PATCH 302/415] [bloomberg] Reax _VALID_URL (Closes #7546)
---
youtube_dl/extractor/bloomberg.py | 9 ++++++---
1 file changed, 6 insertions(+), 3 deletions(-)
diff --git a/youtube_dl/extractor/bloomberg.py b/youtube_dl/extractor/bloomberg.py
index 0dca29b71..11ace91dd 100644
--- a/youtube_dl/extractor/bloomberg.py
+++ b/youtube_dl/extractor/bloomberg.py
@@ -6,9 +6,9 @@ from .common import InfoExtractor
class BloombergIE(InfoExtractor):
- _VALID_URL = r'https?://www\.bloomberg\.com/news/videos/[^/]+/(?P<id>[^/?#]+)'
+ _VALID_URL = r'https?://www\.bloomberg\.com/news/[^/]+/[^/]+/(?P<id>[^/?#]+)'
- _TEST = {
+ _TESTS = [{
'url': 'http://www.bloomberg.com/news/videos/b/aaeae121-5949-481e-a1ce-4562db6f5df2',
# The md5 checksum changes
'info_dict': {
@@ -17,7 +17,10 @@ class BloombergIE(InfoExtractor):
'title': 'Shah\'s Presentation on Foreign-Exchange Strategies',
'description': 'md5:a8ba0302912d03d246979735c17d2761',
},
- }
+ }, {
+ 'url': 'http://www.bloomberg.com/news/articles/2015-11-12/five-strange-things-that-have-been-happening-in-financial-markets',
+ 'only_matching': True,
+ }]
def _real_extract(self, url):
name = self._match_id(url)
From ee5cd8418e7d6503bfa33ef44e2eb863a55a8c8c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 19 Nov 2015 22:58:29 +0600
Subject: [PATCH 303/415] [theplatform] Handle protocolless feed URLs (Closes
#7532)
---
youtube_dl/extractor/theplatform.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index 43315e75d..d52f9ae4b 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -187,7 +187,8 @@ class ThePlatformIE(ThePlatformBaseIE):
# Seems there's no pattern for the interested script filename, so
# I try one by one
for script in reversed(scripts):
- feed_script = self._download_webpage(script, video_id, 'Downloading feed script')
+ feed_script = self._download_webpage(
+ self._proto_relative_url(script, 'http:'), video_id, 'Downloading feed script')
feed_id = self._search_regex(r'defaultFeedId\s*:\s*"([^"]+)"', feed_script, 'default feed id', default=None)
if feed_id is not None:
break
From 325bb615a79928541aabfbab366ef813b4f72309 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 19 Nov 2015 22:58:43 +0600
Subject: [PATCH 304/415] [theplatform] Style
---
youtube_dl/extractor/theplatform.py | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index d52f9ae4b..1555aa77c 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -188,8 +188,11 @@ class ThePlatformIE(ThePlatformBaseIE):
# I try one by one
for script in reversed(scripts):
feed_script = self._download_webpage(
- self._proto_relative_url(script, 'http:'), video_id, 'Downloading feed script')
- feed_id = self._search_regex(r'defaultFeedId\s*:\s*"([^"]+)"', feed_script, 'default feed id', default=None)
+ self._proto_relative_url(script, 'http:'),
+ video_id, 'Downloading feed script')
+ feed_id = self._search_regex(
+ r'defaultFeedId\s*:\s*"([^"]+)"', feed_script,
+ 'default feed id', default=None)
if feed_id is not None:
break
if feed_id is None:
From 67446fd49b447531720d62aa339fc364a1699e2b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 20 Nov 2015 04:07:39 +0600
Subject: [PATCH 305/415] [instagram] Improve _VALID_URL (Closes #7568)
---
youtube_dl/extractor/instagram.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/instagram.py b/youtube_dl/extractor/instagram.py
index fce179000..c158f2064 100644
--- a/youtube_dl/extractor/instagram.py
+++ b/youtube_dl/extractor/instagram.py
@@ -10,7 +10,7 @@ from ..utils import (
class InstagramIE(InfoExtractor):
- _VALID_URL = r'https://instagram\.com/p/(?P<id>[^/?#&]+)'
+ _VALID_URL = r'https?://(?:www\.)?instagram\.com/p/(?P<id>[^/?#&]+)'
_TESTS = [{
'url': 'https://instagram.com/p/aye83DjauH/?foo=bar#abc',
'md5': '0d2da106a9d2631273e192b372806516',
From 17cc1534359aad2c180cfd8a4f66d9d0eccb9e01 Mon Sep 17 00:00:00 2001
From: ashutosh-mishra <ashutosh.mishra@oneconvergence.com>
Date: Fri, 20 Nov 2015 22:51:46 +0530
Subject: [PATCH 306/415] Typo fix, found while going through the code.
---
youtube_dl/YoutubeDL.py | 2 +-
youtube_dl/downloader/common.py | 2 +-
youtube_dl/downloader/rtmp.py | 2 +-
3 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 2e824117a..fba99af8d 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -938,7 +938,7 @@ class YoutubeDL(object):
filter_parts.append(string)
def _remove_unused_ops(tokens):
- # Remove operators that we don't use and join them with the sourrounding strings
+ # Remove operators that we don't use and join them with the surrounding strings
# for example: 'mp4' '-' 'baseline' '-' '16x9' is converted to 'mp4-baseline-16x9'
ALLOWED_OPS = ('/', '+', ',', '(', ')')
last_string, last_start, last_end, last_line = None, None, None, None
diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py
index 29a4500d3..b8bf8daf8 100644
--- a/youtube_dl/downloader/common.py
+++ b/youtube_dl/downloader/common.py
@@ -42,7 +42,7 @@ class FileDownloader(object):
min_filesize: Skip files smaller than this size
max_filesize: Skip files larger than this size
xattr_set_filesize: Set ytdl.filesize user xattribute with expected size.
- (experimenatal)
+ (experimental)
external_downloader_args: A list of additional command-line arguments for the
external downloader.
diff --git a/youtube_dl/downloader/rtmp.py b/youtube_dl/downloader/rtmp.py
index f1d219ba9..14d56db47 100644
--- a/youtube_dl/downloader/rtmp.py
+++ b/youtube_dl/downloader/rtmp.py
@@ -117,7 +117,7 @@ class RtmpFD(FileDownloader):
return False
# Download using rtmpdump. rtmpdump returns exit code 2 when
- # the connection was interrumpted and resuming appears to be
+ # the connection was interrupted and resuming appears to be
# possible. This is part of rtmpdump's normal usage, AFAIK.
basic_args = [
'rtmpdump', '--verbose', '-r', url,
From 01b06aedcf9692e03a80efe7e9857560a2a6eb45 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 Nov 2015 01:34:02 +0600
Subject: [PATCH 307/415] [kaltura] Add support for referrer protected videos
(#7409)
---
youtube_dl/extractor/kaltura.py | 47 ++++++++++++++++++++++++---------
1 file changed, 34 insertions(+), 13 deletions(-)
diff --git a/youtube_dl/extractor/kaltura.py b/youtube_dl/extractor/kaltura.py
index 0dcd6cd05..39e038659 100644
--- a/youtube_dl/extractor/kaltura.py
+++ b/youtube_dl/extractor/kaltura.py
@@ -2,12 +2,17 @@
from __future__ import unicode_literals
import re
+import base64
from .common import InfoExtractor
-from ..compat import compat_urllib_parse
+from ..compat import (
+ compat_urllib_parse,
+ compat_urlparse,
+)
from ..utils import (
ExtractorError,
int_or_none,
+ unsmuggle_url,
)
@@ -121,24 +126,40 @@ class KalturaIE(InfoExtractor):
video_id, actions, note='Downloading video info JSON')
def _real_extract(self, url):
+ url, smuggled_data = unsmuggle_url(url, {})
+
mobj = re.match(self._VALID_URL, url)
partner_id = mobj.group('partner_id_s') or mobj.group('partner_id') or mobj.group('partner_id_html5')
entry_id = mobj.group('id_s') or mobj.group('id') or mobj.group('id_html5')
info, source_data = self._get_video_info(entry_id, partner_id)
- formats = [{
- 'format_id': '%(fileExt)s-%(bitrate)s' % f,
- 'ext': f['fileExt'],
- 'tbr': f['bitrate'],
- 'fps': f.get('frameRate'),
- 'filesize_approx': int_or_none(f.get('size'), invscale=1024),
- 'container': f.get('containerFormat'),
- 'vcodec': f.get('videoCodecId'),
- 'height': f.get('height'),
- 'width': f.get('width'),
- 'url': '%s/flavorId/%s' % (info['dataUrl'], f['id']),
- } for f in source_data['flavorAssets']]
+ source_url = smuggled_data.get('source_url')
+ if source_url:
+ referrer = base64.b64encode(
+ '://'.join(compat_urlparse.urlparse(source_url)[:2])
+ .encode('utf-8')).decode('utf-8')
+ else:
+ referrer = None
+
+ formats = []
+ for f in source_data['flavorAssets']:
+ video_url = '%s/flavorId/%s' % (info['dataUrl'], f['id'])
+ if referrer:
+ video_url += '?referrer=%s' % referrer
+ formats.append({
+ 'format_id': '%(fileExt)s-%(bitrate)s' % f,
+ 'ext': f['fileExt'],
+ 'tbr': f['bitrate'],
+ 'fps': f.get('frameRate'),
+ 'filesize_approx': int_or_none(f.get('size'), invscale=1024),
+ 'container': f.get('containerFormat'),
+ 'vcodec': f.get('videoCodecId'),
+ 'height': f.get('height'),
+ 'width': f.get('width'),
+ 'url': video_url,
+ })
+ self._check_formats(formats, entry_id)
self._sort_formats(formats)
return {
From 5b5fae5f2004e5a1be36551bf477f74161d4f9ac Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 Nov 2015 01:35:58 +0600
Subject: [PATCH 308/415] [generic] Use referrer from source kaltura embed URLs
(#7409)
---
youtube_dl/extractor/generic.py | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index b483eba65..c7cee5487 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1694,7 +1694,9 @@ class GenericIE(InfoExtractor):
mobj = (re.search(r"(?s)kWidget\.(?:thumb)?[Ee]mbed\(\{.*?'wid'\s*:\s*'_?(?P<partner_id>[^']+)',.*?'entry_?[Ii]d'\s*:\s*'(?P<id>[^']+)',", webpage) or
re.search(r'(?s)(?P<q1>["\'])(?:https?:)?//cdnapi(?:sec)?\.kaltura\.com/.*?(?:p|partner_id)/(?P<partner_id>\d+).*?(?P=q1).*?entry_?[Ii]d\s*:\s*(?P<q2>["\'])(?P<id>.+?)(?P=q2)', webpage))
if mobj is not None:
- return self.url_result('kaltura:%(partner_id)s:%(id)s' % mobj.groupdict(), 'Kaltura')
+ return self.url_result(smuggle_url(
+ 'kaltura:%(partner_id)s:%(id)s' % mobj.groupdict(),
+ {'source_url': url}), 'Kaltura')
# Look for Eagle.Platform embeds
mobj = re.search(
From d80a39cec89e8ddfc3090c3d1b10d3fe105791e1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 Nov 2015 01:38:08 +0600
Subject: [PATCH 309/415] [kaltura] Improve
---
youtube_dl/extractor/kaltura.py | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git a/youtube_dl/extractor/kaltura.py b/youtube_dl/extractor/kaltura.py
index 39e038659..1eaa09fea 100644
--- a/youtube_dl/extractor/kaltura.py
+++ b/youtube_dl/extractor/kaltura.py
@@ -149,14 +149,14 @@ class KalturaIE(InfoExtractor):
video_url += '?referrer=%s' % referrer
formats.append({
'format_id': '%(fileExt)s-%(bitrate)s' % f,
- 'ext': f['fileExt'],
- 'tbr': f['bitrate'],
- 'fps': f.get('frameRate'),
+ 'ext': f.get('fileExt'),
+ 'tbr': int_or_none(f['bitrate']),
+ 'fps': int_or_none(f.get('frameRate')),
'filesize_approx': int_or_none(f.get('size'), invscale=1024),
'container': f.get('containerFormat'),
'vcodec': f.get('videoCodecId'),
- 'height': f.get('height'),
- 'width': f.get('width'),
+ 'height': int_or_none(f.get('height')),
+ 'width': int_or_none(f.get('width')),
'url': video_url,
})
self._check_formats(formats, entry_id)
From bdceea7afde4fe02fff90f84ef1f2fb6ebbac9d3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 Nov 2015 01:39:29 +0600
Subject: [PATCH 310/415] [kaltura] Clean description
---
youtube_dl/extractor/kaltura.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/kaltura.py b/youtube_dl/extractor/kaltura.py
index 1eaa09fea..583b1a5ad 100644
--- a/youtube_dl/extractor/kaltura.py
+++ b/youtube_dl/extractor/kaltura.py
@@ -10,6 +10,7 @@ from ..compat import (
compat_urlparse,
)
from ..utils import (
+ clean_html,
ExtractorError,
int_or_none,
unsmuggle_url,
@@ -166,7 +167,7 @@ class KalturaIE(InfoExtractor):
'id': entry_id,
'title': info['name'],
'formats': formats,
- 'description': info.get('description'),
+ 'description': clean_html(info.get('description')),
'thumbnail': info.get('thumbnailUrl'),
'duration': info.get('duration'),
'timestamp': info.get('createdAt'),
From 6da620de58bd1a4221bcdae5f4f2098b247038bf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 Nov 2015 01:40:28 +0600
Subject: [PATCH 311/415] [kaltura] Add test for referrer protected video
(#7409)
---
youtube_dl/extractor/generic.py | 13 +++++++++++++
1 file changed, 13 insertions(+)
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index c7cee5487..e512c1e14 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -823,6 +823,19 @@ class GenericIE(InfoExtractor):
'title': 'Os Guinness // Is It Fools Talk? // Unbelievable? Conference 2014',
},
},
+ # Kaltura embed protected with referrer
+ {
+ 'url': 'http://www.disney.nl/disney-channel/filmpjes/achter-de-schermen#/videoId/violetta-achter-de-schermen-ruggero',
+ 'info_dict': {
+ 'id': '1_g4fbemnq',
+ 'ext': 'mp4',
+ 'title': 'Violetta - Achter De Schermen - Ruggero',
+ 'description': 'Achter de schermen met Ruggero',
+ 'timestamp': 1435133761,
+ 'upload_date': '20150624',
+ 'uploader_id': 'echojecka',
+ },
+ },
# Eagle.Platform embed (generic URL)
{
'url': 'http://lenta.ru/news/2015/03/06/navalny/',
From 71bd93b89c667b7ca852b3a536dd771da7b67f67 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 Nov 2015 08:08:34 +0600
Subject: [PATCH 312/415] [pluralsight] Do not rely on argument order in query
(Closes #7583)
---
youtube_dl/extractor/pluralsight.py | 24 +++++++++++++++---------
1 file changed, 15 insertions(+), 9 deletions(-)
diff --git a/youtube_dl/extractor/pluralsight.py b/youtube_dl/extractor/pluralsight.py
index fd32836cc..6cef7c829 100644
--- a/youtube_dl/extractor/pluralsight.py
+++ b/youtube_dl/extractor/pluralsight.py
@@ -1,6 +1,5 @@
from __future__ import unicode_literals
-import re
import json
from .common import InfoExtractor
@@ -19,11 +18,11 @@ from ..utils import (
class PluralsightIE(InfoExtractor):
IE_NAME = 'pluralsight'
- _VALID_URL = r'https?://(?:www\.)?pluralsight\.com/training/player\?author=(?P<author>[^&]+)&name=(?P<name>[^&]+)(?:&mode=live)?&clip=(?P<clip>\d+)&course=(?P<course>[^&]+)'
+ _VALID_URL = r'https?://(?:(?:www|app)\.)?pluralsight\.com/training/player\?'
_LOGIN_URL = 'https://www.pluralsight.com/id/'
_NETRC_MACHINE = 'pluralsight'
- _TEST = {
+ _TESTS = [{
'url': 'http://www.pluralsight.com/training/player?author=mike-mckeown&name=hosting-sql-server-windows-azure-iaas-m7-mgmt&mode=live&clip=3&course=hosting-sql-server-windows-azure-iaas',
'md5': '4d458cf5cf4c593788672419a8dd4cf8',
'info_dict': {
@@ -33,7 +32,10 @@ class PluralsightIE(InfoExtractor):
'duration': 338,
},
'skip': 'Requires pluralsight account credentials',
- }
+ }, {
+ 'url': 'https://app.pluralsight.com/training/player?course=angularjs-get-started&author=scott-allen&name=angularjs-get-started-m1-introduction&clip=0&mode=live',
+ 'only_matching': True,
+ }]
def _real_initialize(self):
self._login()
@@ -74,11 +76,15 @@ class PluralsightIE(InfoExtractor):
raise ExtractorError('Unable to login: %s' % error, expected=True)
def _real_extract(self, url):
- mobj = re.match(self._VALID_URL, url)
- author = mobj.group('author')
- name = mobj.group('name')
- clip_id = mobj.group('clip')
- course = mobj.group('course')
+ qs = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
+
+ author = qs.get('author', [None])[0]
+ name = qs.get('name', [None])[0]
+ clip_id = qs.get('clip', [None])[0]
+ course = qs.get('course', [None])[0]
+
+ if any(not f for f in (author, name, clip_id, course,)):
+ raise ExtractorError('Invalid URL', expected=True)
display_id = '%s-%s' % (name, clip_id)
From 651acffbe57a016ae3b6b2c842aa691b1feb2d5b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 Nov 2015 08:21:33 +0600
Subject: [PATCH 313/415] [pluralsight] Update ViewClip URL
---
youtube_dl/extractor/pluralsight.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/pluralsight.py b/youtube_dl/extractor/pluralsight.py
index 6cef7c829..caba8fb79 100644
--- a/youtube_dl/extractor/pluralsight.py
+++ b/youtube_dl/extractor/pluralsight.py
@@ -138,7 +138,7 @@ class PluralsightIE(InfoExtractor):
'q': '%dx%d' % (f['width'], f['height']),
}
request = compat_urllib_request.Request(
- 'http://www.pluralsight.com/training/Player/ViewClip',
+ 'http://app.pluralsight.com/training/Player/ViewClip',
json.dumps(clip_post).encode('utf-8'))
request.add_header('Content-Type', 'application/json;charset=utf-8')
format_id = '%s-%s' % (ext, quality)
From c23e266427ef7fdcff3dc02e0d9978c58addea6c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 Nov 2015 08:25:52 +0600
Subject: [PATCH 314/415] [pluralsight] Do not require pluralsight account
Looks like some courses are available without pluralsight account
---
youtube_dl/extractor/pluralsight.py | 14 +++++++++++---
1 file changed, 11 insertions(+), 3 deletions(-)
diff --git a/youtube_dl/extractor/pluralsight.py b/youtube_dl/extractor/pluralsight.py
index caba8fb79..c6fd41d5a 100644
--- a/youtube_dl/extractor/pluralsight.py
+++ b/youtube_dl/extractor/pluralsight.py
@@ -35,6 +35,10 @@ class PluralsightIE(InfoExtractor):
}, {
'url': 'https://app.pluralsight.com/training/player?course=angularjs-get-started&author=scott-allen&name=angularjs-get-started-m1-introduction&clip=0&mode=live',
'only_matching': True,
+ }, {
+ # available without pluralsight account
+ 'url': 'http://app.pluralsight.com/training/player?author=scott-allen&name=angularjs-get-started-m1-introduction&mode=live&clip=0&course=angularjs-get-started',
+ 'only_matching': True,
}]
def _real_initialize(self):
@@ -43,7 +47,7 @@ class PluralsightIE(InfoExtractor):
def _login(self):
(username, password) = self._get_login_info()
if username is None:
- self.raise_login_required('Pluralsight account is required')
+ return
login_page = self._download_webpage(
self._LOGIN_URL, None, 'Downloading login page')
@@ -172,7 +176,7 @@ class PluralsightIE(InfoExtractor):
class PluralsightCourseIE(InfoExtractor):
IE_NAME = 'pluralsight:course'
_VALID_URL = r'https?://(?:www\.)?pluralsight\.com/courses/(?P<id>[^/]+)'
- _TEST = {
+ _TESTS = [{
# Free course from Pluralsight Starter Subscription for Microsoft TechNet
# https://offers.pluralsight.com/technet?loc=zTS3z&prod=zOTprodz&tech=zOttechz&prog=zOTprogz&type=zSOz&media=zOTmediaz&country=zUSz
'url': 'http://www.pluralsight.com/courses/hosting-sql-server-windows-azure-iaas',
@@ -182,7 +186,11 @@ class PluralsightCourseIE(InfoExtractor):
'description': 'md5:61b37e60f21c4b2f91dc621a977d0986',
},
'playlist_count': 31,
- }
+ }, {
+ # available without pluralsight account
+ 'url': 'https://www.pluralsight.com/courses/angularjs-get-started',
+ 'only_matching': True,
+ }]
def _real_extract(self, url):
course_id = self._match_id(url)
From a5cd0eb8a4f2661cfe4863e958172d8f8f42d0f1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 Nov 2015 08:32:48 +0600
Subject: [PATCH 315/415] [pluralsight:course] Improve _VALID_URL
---
youtube_dl/extractor/pluralsight.py | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/pluralsight.py b/youtube_dl/extractor/pluralsight.py
index c6fd41d5a..93244f41d 100644
--- a/youtube_dl/extractor/pluralsight.py
+++ b/youtube_dl/extractor/pluralsight.py
@@ -175,7 +175,7 @@ class PluralsightIE(InfoExtractor):
class PluralsightCourseIE(InfoExtractor):
IE_NAME = 'pluralsight:course'
- _VALID_URL = r'https?://(?:www\.)?pluralsight\.com/courses/(?P<id>[^/]+)'
+ _VALID_URL = r'https?://(?:(?:www|app)\.)?pluralsight\.com/(?:library/)?courses/(?P<id>[^/]+)'
_TESTS = [{
# Free course from Pluralsight Starter Subscription for Microsoft TechNet
# https://offers.pluralsight.com/technet?loc=zTS3z&prod=zOTprodz&tech=zOttechz&prog=zOTprogz&type=zSOz&media=zOTmediaz&country=zUSz
@@ -190,6 +190,9 @@ class PluralsightCourseIE(InfoExtractor):
# available without pluralsight account
'url': 'https://www.pluralsight.com/courses/angularjs-get-started',
'only_matching': True,
+ }, {
+ 'url': 'https://app.pluralsight.com/library/courses/understanding-microsoft-azure-amazon-aws/table-of-contents',
+ 'only_matching': True,
}]
def _real_extract(self, url):
From 6cc37c69e25f886c466d4b89f0734639e20e279b Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 21 Nov 2015 14:12:34 +0800
Subject: [PATCH 316/415] [generic] Unescape URLs from JWPlayer (#7582)
---
youtube_dl/extractor/generic.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index e512c1e14..6cffde20d 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1874,6 +1874,7 @@ class GenericIE(InfoExtractor):
entries = []
for video_url in found:
+ video_url = video_url.replace('\\/', '/')
video_url = compat_urlparse.urljoin(url, video_url)
video_id = compat_urllib_parse_unquote(os.path.basename(video_url))
From 28602e747c13a7979aedd517e491bada3856cb12 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 21 Nov 2015 16:08:54 +0800
Subject: [PATCH 317/415] [generic] Refactor
---
youtube_dl/extractor/generic.py | 25 +++++++++++--------------
1 file changed, 11 insertions(+), 14 deletions(-)
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 6cffde20d..1991a8684 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1886,25 +1886,22 @@ class GenericIE(InfoExtractor):
# here's a fun little line of code for you:
video_id = os.path.splitext(video_id)[0]
+ entry_info_dict = {
+ 'id': video_id,
+ 'uploader': video_uploader,
+ 'title': video_title,
+ 'age_limit': age_limit,
+ }
+
ext = determine_ext(video_url)
if ext == 'smil':
- entries.append({
- 'id': video_id,
- 'formats': self._extract_smil_formats(video_url, video_id),
- 'uploader': video_uploader,
- 'title': video_title,
- 'age_limit': age_limit,
- })
+ entry_info_dict['formats'] = self._extract_smil_formats(video_url, video_id)
elif ext == 'xspf':
return self.playlist_result(self._extract_xspf_playlist(video_url, video_id), video_id)
else:
- entries.append({
- 'id': video_id,
- 'url': video_url,
- 'uploader': video_uploader,
- 'title': video_title,
- 'age_limit': age_limit,
- })
+ entry_info_dict['url'] = video_url
+
+ entries.append(entry_info_dict)
if len(entries) == 1:
return entries[0]
From 750b9ff0324b04f252b05cfadd802e8b1f2cc19d Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 21 Nov 2015 16:43:01 +0800
Subject: [PATCH 318/415] [generic] Extract M3U8 formats (closes #7582)
---
youtube_dl/extractor/generic.py | 16 ++++++++++++++++
1 file changed, 16 insertions(+)
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 1991a8684..2b934148d 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1058,6 +1058,20 @@ class GenericIE(InfoExtractor):
'description': 'Tabletop: Dread, Last Thoughts',
'duration': 51690,
},
+ },
+ # JWPlayer with M3U8
+ {
+ 'url': 'http://ren.tv/novosti/2015-09-25/sluchaynyy-prohozhiy-poymal-avtougonshchika-v-murmanske-video',
+ 'info_dict': {
+ 'id': 'playlist',
+ 'ext': 'mp4',
+ 'title': 'Случайный прохожий поймал автоугонщика в Мурманске. ВИДЕО | РЕН ТВ',
+ 'uploader': 'ren.tv',
+ },
+ 'params': {
+ # m3u8 downloads
+ 'skip_download': True,
+ }
}
]
@@ -1898,6 +1912,8 @@ class GenericIE(InfoExtractor):
entry_info_dict['formats'] = self._extract_smil_formats(video_url, video_id)
elif ext == 'xspf':
return self.playlist_result(self._extract_xspf_playlist(video_url, video_id), video_id)
+ elif ext == 'm3u8':
+ entry_info_dict['formats'] = self._extract_m3u8_formats(video_url, video_id, ext='mp4')
else:
entry_info_dict['url'] = video_url
From f52183a8780a7b9e58ffb4510e234a7392ad8a2f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 Nov 2015 17:39:24 +0600
Subject: [PATCH 319/415] [rutube:embed] Extend _VALID_URL (Closes #7588)
---
youtube_dl/extractor/rutube.py | 9 ++++++---
1 file changed, 6 insertions(+), 3 deletions(-)
diff --git a/youtube_dl/extractor/rutube.py b/youtube_dl/extractor/rutube.py
index d94dc7399..e824129b4 100644
--- a/youtube_dl/extractor/rutube.py
+++ b/youtube_dl/extractor/rutube.py
@@ -74,9 +74,9 @@ class RutubeIE(InfoExtractor):
class RutubeEmbedIE(InfoExtractor):
IE_NAME = 'rutube:embed'
IE_DESC = 'Rutube embedded videos'
- _VALID_URL = 'https?://rutube\.ru/video/embed/(?P<id>[0-9]+)'
+ _VALID_URL = 'https?://rutube\.ru/(?:video|play)/embed/(?P<id>[0-9]+)'
- _TEST = {
+ _TESTS = [{
'url': 'http://rutube.ru/video/embed/6722881?vk_puid37=&vk_puid38=',
'info_dict': {
'id': 'a10e53b86e8f349080f718582ce4c661',
@@ -90,7 +90,10 @@ class RutubeEmbedIE(InfoExtractor):
'params': {
'skip_download': 'Requires ffmpeg',
},
- }
+ }, {
+ 'url': 'http://rutube.ru/play/embed/8083783',
+ 'only_matching': True,
+ }]
def _real_extract(self, url):
embed_id = self._match_id(url)
From 019839faaa28d6233097a669d063a3a42f9fbbc7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 Nov 2015 18:01:39 +0600
Subject: [PATCH 320/415] [extractor/common] Use baseURL from f4m manifest for
recursive manifest extraction
---
youtube_dl/extractor/common.py | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 5e263f8b5..71bdcad5a 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -891,6 +891,11 @@ class InfoExtractor(object):
if not media_nodes:
manifest_version = '2.0'
media_nodes = manifest.findall('{http://ns.adobe.com/f4m/2.0}media')
+ base_url = xpath_text(
+ manifest, ['{http://ns.adobe.com/f4m/1.0}baseURL', '{http://ns.adobe.com/f4m/2.0}baseURL'],
+ 'base URL', default=None)
+ if base_url:
+ base_url = base_url.strip()
for i, media_el in enumerate(media_nodes):
if manifest_version == '2.0':
media_url = media_el.attrib.get('href') or media_el.attrib.get('url')
@@ -898,7 +903,7 @@ class InfoExtractor(object):
continue
manifest_url = (
media_url if media_url.startswith('http://') or media_url.startswith('https://')
- else ('/'.join(manifest_url.split('/')[:-1]) + '/' + media_url))
+ else ((base_url or '/'.join(manifest_url.split('/')[:-1])) + '/' + media_url))
# If media_url is itself a f4m manifest do the recursive extraction
# since bitrates in parent manifest (this one) and media_url manifest
# may differ leading to inability to resolve the format by requested
From 413719689996b2642f2484dd572d551376f0d104 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 Nov 2015 18:02:52 +0600
Subject: [PATCH 321/415] [rutube] Extract all formats
---
youtube_dl/extractor/rutube.py | 26 +++++++++++++++++++++-----
1 file changed, 21 insertions(+), 5 deletions(-)
diff --git a/youtube_dl/extractor/rutube.py b/youtube_dl/extractor/rutube.py
index e824129b4..0db8c410d 100644
--- a/youtube_dl/extractor/rutube.py
+++ b/youtube_dl/extractor/rutube.py
@@ -9,7 +9,7 @@ from ..compat import (
compat_str,
)
from ..utils import (
- ExtractorError,
+ determine_ext,
unified_strdate,
)
@@ -51,10 +51,26 @@ class RutubeIE(InfoExtractor):
'http://rutube.ru/api/play/options/%s/?format=json' % video_id,
video_id, 'Downloading options JSON')
- m3u8_url = options['video_balancer'].get('m3u8')
- if m3u8_url is None:
- raise ExtractorError('Couldn\'t find m3u8 manifest url')
- formats = self._extract_m3u8_formats(m3u8_url, video_id, ext='mp4')
+ formats = []
+ for format_id, format_url in options['video_balancer'].items():
+ ext = determine_ext(format_url)
+ print(ext)
+ if ext == 'm3u8':
+ m3u8_formats = self._extract_m3u8_formats(
+ format_url, video_id, 'mp4', m3u8_id=format_id, fatal=False)
+ if m3u8_formats:
+ formats.extend(m3u8_formats)
+ elif ext == 'f4m':
+ f4m_formats = self._extract_f4m_formats(
+ format_url, video_id, f4m_id=format_id, fatal=False)
+ if f4m_formats:
+ formats.extend(f4m_formats)
+ else:
+ formats.append({
+ 'url': format_url,
+ 'format_id': format_id,
+ })
+ self._sort_formats(formats)
return {
'id': video['id'],
From 2abf7cab80a2d12a3157afef05d61f8404bce45d Mon Sep 17 00:00:00 2001
From: reiv <metareiv@gmail.com>
Date: Sat, 17 Oct 2015 18:23:46 +0200
Subject: [PATCH 322/415] [soundcloud] Add Soundcloud search extractor
---
youtube_dl/extractor/__init__.py | 3 +-
youtube_dl/extractor/soundcloud.py | 88 +++++++++++++++++++++++++++++-
2 files changed, 89 insertions(+), 2 deletions(-)
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 26e5745d6..232bcd89a 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -576,7 +576,8 @@ from .soundcloud import (
SoundcloudIE,
SoundcloudSetIE,
SoundcloudUserIE,
- SoundcloudPlaylistIE
+ SoundcloudPlaylistIE,
+ SoundcloudSearchIE
)
from .soundgasm import (
SoundgasmIE,
diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index 2b60d354a..7395a9848 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -4,7 +4,10 @@ from __future__ import unicode_literals
import re
import itertools
-from .common import InfoExtractor
+from .common import (
+ InfoExtractor,
+ SearchInfoExtractor
+)
from ..compat import (
compat_str,
compat_urlparse,
@@ -469,3 +472,86 @@ class SoundcloudPlaylistIE(SoundcloudIE):
'description': data.get('description'),
'entries': entries,
}
+
+
+class SoundcloudSearchIE(SearchInfoExtractor, SoundcloudIE):
+ IE_NAME = 'soundcloud:search'
+ IE_DESC = 'Soundcloud search'
+ _MAX_RESULTS = 200
+ _TESTS = [{
+ 'url': 'scsearch15:post-avant jazzcore',
+ 'info_dict': {
+ 'title': 'post-avant jazzcore',
+ },
+ 'playlist_count': 15,
+ }]
+
+ _SEARCH_KEY = 'scsearch'
+ _RESULTS_PER_PAGE = 50
+
+ def _get_collection(self, endpoint, collection_id, **query):
+ import itertools
+
+ query['limit'] = self._RESULTS_PER_PAGE
+ query['client_id'] = self._CLIENT_ID
+ query['linked_partitioning'] = '1'
+
+ api_base_url = '{0}//api-v2.soundcloud.com'.format(self.http_scheme())
+
+ total_results = self._MAX_RESULTS
+ collected_results = 0
+
+ next_url = None
+
+ for i in itertools.count():
+
+ if not next_url:
+ query['offset'] = i * self._RESULTS_PER_PAGE
+ data = compat_urllib_parse.urlencode(query)
+ next_url = '{0}{1}?{2}'.format(api_base_url, endpoint, data)
+
+ response = self._download_json(next_url,
+ video_id=collection_id,
+ note='Downloading page {0}'.format(i+1),
+ errnote='Unable to download API page')
+
+ total_results = int(response.get(
+ u'total_results', total_results))
+
+ collection = response['collection']
+ collected_results += len(collection)
+
+ for item in filter(bool, collection):
+ yield item
+
+ if collected_results >= total_results or not collection:
+ break
+
+ next_url = response.get(u'next_href', None)
+
+ def _get_n_results(self, query, n):
+
+ results = []
+
+ tracks = self._get_collection('/search/tracks',
+ collection_id='Query "{}"'.format(query),
+ q=query.encode('utf-8'))
+
+ for _ in range(n):
+ try:
+ track = next(tracks)
+ except StopIteration:
+ break
+ uri = track[u'uri']
+ title = track[u'title']
+ username = track[u'user'][u'username']
+ results.append(self.url_result(
+ url=uri,
+ video_title='{0} - {1}'.format(username, title)))
+
+ if not results:
+ raise ExtractorError(
+ '[soundcloud] No track results', expected=True)
+
+ return self.playlist_result(results[:n], playlist_title=query)
+
From c30943b1c06d0f8d927031fc3a7f3cda30c4dae1 Mon Sep 17 00:00:00 2001
From: reiv <metareiv@gmail.com>
Date: Sat, 17 Oct 2015 21:18:42 +0200
Subject: [PATCH 323/415] Fix some compatibility issues, cleanup.
---
youtube_dl/extractor/soundcloud.py | 24 ++++++++----------------
1 file changed, 8 insertions(+), 16 deletions(-)
diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index 7395a9848..3fe991849 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -488,27 +488,23 @@ class SoundcloudSearchIE(SearchInfoExtractor, SoundcloudIE):
_SEARCH_KEY = 'scsearch'
_RESULTS_PER_PAGE = 50
+ _API_V2_BASE = 'https://api-v2.soundcloud.com'
def _get_collection(self, endpoint, collection_id, **query):
- import itertools
-
query['limit'] = self._RESULTS_PER_PAGE
query['client_id'] = self._CLIENT_ID
query['linked_partitioning'] = '1'
- api_base_url = '{0}//api-v2.soundcloud.com'.format(self.http_scheme())
-
total_results = self._MAX_RESULTS
collected_results = 0
next_url = None
for i in itertools.count():
-
if not next_url:
query['offset'] = i * self._RESULTS_PER_PAGE
data = compat_urllib_parse.urlencode(query)
- next_url = '{0}{1}?{2}'.format(api_base_url, endpoint, data)
+ next_url = '{0}{1}?{2}'.format(self._API_V2_BASE, endpoint, data)
response = self._download_json(next_url,
video_id=collection_id,
@@ -516,7 +512,7 @@ class SoundcloudSearchIE(SearchInfoExtractor, SoundcloudIE):
errnote='Unable to download API page')
total_results = int(response.get(
- u'total_results', total_results))
+ 'total_results', total_results))
collection = response['collection']
collected_results += len(collection)
@@ -527,14 +523,13 @@ class SoundcloudSearchIE(SearchInfoExtractor, SoundcloudIE):
if collected_results >= total_results or not collection:
break
- next_url = response.get(u'next_href', None)
+ next_url = response.get('next_href', None)
def _get_n_results(self, query, n):
-
results = []
tracks = self._get_collection('/search/tracks',
- collection_id='Query "{}"'.format(query),
+ collection_id='Query "{0}"'.format(query),
q=query.encode('utf-8'))
for _ in range(n):
@@ -542,12 +537,9 @@ class SoundcloudSearchIE(SearchInfoExtractor, SoundcloudIE):
track = next(tracks)
except StopIteration:
break
- uri = track[u'uri']
- title = track[u'title']
- username = track[u'user'][u'username']
- results.append(self.url_result(
- url=uri,
- video_title='{0} - {1}'.format(username, title)))
+ uri = track['uri']
+ title = track['title']
+ results.append(self.url_result(url=uri))
if not results:
raise ExtractorError(
From b54b08c91bb7f59a9ea720abbf944809d1ea8956 Mon Sep 17 00:00:00 2001
From: reiv <metareiv@gmail.com>
Date: Sat, 17 Oct 2015 22:36:08 +0200
Subject: [PATCH 324/415] Simplify with itertools.islice().
---
youtube_dl/extractor/soundcloud.py | 6 +-----
1 file changed, 1 insertion(+), 5 deletions(-)
diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index 3fe991849..959f27975 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -532,11 +532,7 @@ class SoundcloudSearchIE(SearchInfoExtractor, SoundcloudIE):
collection_id='Query "{0}"'.format(query),
q=query.encode('utf-8'))
- for _ in range(n):
- try:
- track = next(tracks)
- except StopIteration:
- break
+ for track in itertools.islice(tracks, n):
uri = track['uri']
title = track['title']
results.append(self.url_result(url=uri))
From 6ea7190a3e6165f3d5c49e1e142eb85951b44712 Mon Sep 17 00:00:00 2001
From: reiv <metareiv@gmail.com>
Date: Sat, 17 Oct 2015 22:47:16 +0200
Subject: [PATCH 325/415] Rewrite as list comprehension.
---
youtube_dl/extractor/soundcloud.py | 8 ++------
1 file changed, 2 insertions(+), 6 deletions(-)
diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index 959f27975..c623bb6de 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -526,16 +526,12 @@ class SoundcloudSearchIE(SearchInfoExtractor, SoundcloudIE):
next_url = response.get('next_href', None)
def _get_n_results(self, query, n):
- results = []
-
tracks = self._get_collection('/search/tracks',
collection_id='Query "{0}"'.format(query),
q=query.encode('utf-8'))
- for track in itertools.islice(tracks, n):
- uri = track['uri']
- title = track['title']
- results.append(self.url_result(url=uri))
+ results = [self.url_result(url=track['uri'])
+ for track in itertools.islice(tracks, n)]
if not results:
raise ExtractorError(
From 417b4536999daaae2e2cc105012cd1c1e8046c15 Mon Sep 17 00:00:00 2001
From: reiv <metareiv@gmail.com>
Date: Sun, 18 Oct 2015 12:43:25 +0200
Subject: [PATCH 326/415] [soundcloud] Use correct error message conventions
---
youtube_dl/extractor/soundcloud.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index c623bb6de..6b510a43b 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -535,7 +535,7 @@ class SoundcloudSearchIE(SearchInfoExtractor, SoundcloudIE):
if not results:
raise ExtractorError(
- '[soundcloud] No track results', expected=True)
+ 'Soundcloud said: No track results', expected=True)
- return self.playlist_result(results[:n], playlist_title=query)
+ return self.playlist_result(results, playlist_title=query)
From 328a22e175a4e056258d50ebffe28026b9411285 Mon Sep 17 00:00:00 2001
From: reiv <metareiv@gmail.com>
Date: Fri, 30 Oct 2015 23:56:07 +0100
Subject: [PATCH 327/415] [soundcloud] Remove limit on search results
---
youtube_dl/extractor/soundcloud.py | 21 +++++++++++++--------
1 file changed, 13 insertions(+), 8 deletions(-)
diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index 6b510a43b..bba29d845 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -477,7 +477,7 @@ class SoundcloudPlaylistIE(SoundcloudIE):
class SoundcloudSearchIE(SearchInfoExtractor, SoundcloudIE):
IE_NAME = 'soundcloud:search'
IE_DESC = 'Soundcloud search'
- _MAX_RESULTS = 200
+ _MAX_RESULTS = float('inf')
_TESTS = [{
'url': 'scsearch15:post-avant jazzcore',
'info_dict': {
@@ -487,24 +487,28 @@ class SoundcloudSearchIE(SearchInfoExtractor, SoundcloudIE):
}]
_SEARCH_KEY = 'scsearch'
- _RESULTS_PER_PAGE = 50
+ _MAX_RESULTS_PER_PAGE = 200
+ _DEFAULT_RESULTS_PER_PAGE = 50
_API_V2_BASE = 'https://api-v2.soundcloud.com'
def _get_collection(self, endpoint, collection_id, **query):
- query['limit'] = self._RESULTS_PER_PAGE
+ query['limit'] = results_per_page = min(
+ query.get('limit', self._DEFAULT_RESULTS_PER_PAGE),
+ self._MAX_RESULTS_PER_PAGE)
query['client_id'] = self._CLIENT_ID
query['linked_partitioning'] = '1'
- total_results = self._MAX_RESULTS
+ total_results = None
collected_results = 0
next_url = None
for i in itertools.count():
if not next_url:
- query['offset'] = i * self._RESULTS_PER_PAGE
+ query['offset'] = i * results_per_page
data = compat_urllib_parse.urlencode(query)
- next_url = '{0}{1}?{2}'.format(self._API_V2_BASE, endpoint, data)
+ next_url = '{0}{1}?{2}'.format(
+ self._API_V2_BASE, endpoint, data)
response = self._download_json(next_url,
video_id=collection_id,
@@ -520,7 +524,8 @@ class SoundcloudSearchIE(SearchInfoExtractor, SoundcloudIE):
for item in filter(bool, collection):
yield item
- if collected_results >= total_results or not collection:
+ if (total_results is not None and
+ collected_results >= total_results) or not collection:
break
next_url = response.get('next_href', None)
@@ -528,7 +533,7 @@ class SoundcloudSearchIE(SearchInfoExtractor, SoundcloudIE):
def _get_n_results(self, query, n):
tracks = self._get_collection('/search/tracks',
collection_id='Query "{0}"'.format(query),
- q=query.encode('utf-8'))
+ limit=n, q=query)
results = [self.url_result(url=track['uri'])
for track in itertools.islice(tracks, n)]
From 7e3472758bfaa75aa413368b29a26c2615e5231b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 Nov 2015 20:04:35 +0600
Subject: [PATCH 328/415] [soundcloud:search] PEP 8
---
youtube_dl/extractor/soundcloud.py | 23 +++++++++--------------
1 file changed, 9 insertions(+), 14 deletions(-)
diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index bba29d845..a5c40514b 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -510,10 +510,9 @@ class SoundcloudSearchIE(SearchInfoExtractor, SoundcloudIE):
next_url = '{0}{1}?{2}'.format(
self._API_V2_BASE, endpoint, data)
- response = self._download_json(next_url,
- video_id=collection_id,
- note='Downloading page {0}'.format(i+1),
- errnote='Unable to download API page')
+ response = self._download_json(
+ next_url, collection_id, 'Downloading page {0}'.format(i + 1),
+ 'Unable to download API page')
total_results = int(response.get(
'total_results', total_results))
@@ -524,23 +523,19 @@ class SoundcloudSearchIE(SearchInfoExtractor, SoundcloudIE):
for item in filter(bool, collection):
yield item
- if (total_results is not None and
- collected_results >= total_results) or not collection:
+ if (total_results is not None and collected_results >= total_results) or not collection:
break
- next_url = response.get('next_href', None)
+ next_url = response.get('next_href')
def _get_n_results(self, query, n):
- tracks = self._get_collection('/search/tracks',
- collection_id='Query "{0}"'.format(query),
- limit=n, q=query)
+ tracks = self._get_collection(
+ '/search/tracks', collection_id='Query "{0}"'.format(query), limit=n, q=query)
- results = [self.url_result(url=track['uri'])
- for track in itertools.islice(tracks, n)]
+ results = [self.url_result(track['uri']) for track in itertools.islice(tracks, n)]
if not results:
raise ExtractorError(
'Soundcloud said: No track results', expected=True)
-
- return self.playlist_result(results, playlist_title=query)
+ return self.playlist_result(results, playlist_title=query)
From 7e1f5447e76e57af58bf45ce565742c813c80b99 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 Nov 2015 20:45:50 +0600
Subject: [PATCH 329/415] [utils] Improve encode_dict
---
youtube_dl/utils.py | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index d00b14b86..bff59eb73 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1668,7 +1668,9 @@ def urlencode_postdata(*args, **kargs):
def encode_dict(d, encoding='utf-8'):
- return dict((k.encode(encoding), v.encode(encoding)) for k, v in d.items())
+ def encode(v):
+ return v.encode(encoding) if isinstance(v, compat_basestring) else v
+ return dict((encode(k), encode(v)) for k, v in d.items())
US_RATINGS = {
From 4e3b3030168927bd589343c51278040a5cec6203 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 Nov 2015 20:55:48 +0600
Subject: [PATCH 330/415] [soundcloud:search] Fix non-ASCII searches
---
youtube_dl/extractor/soundcloud.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index a5c40514b..66bb1cb54 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -14,6 +14,7 @@ from ..compat import (
compat_urllib_parse,
)
from ..utils import (
+ encode_dict,
ExtractorError,
int_or_none,
unified_strdate,
@@ -506,7 +507,7 @@ class SoundcloudSearchIE(SearchInfoExtractor, SoundcloudIE):
for i in itertools.count():
if not next_url:
query['offset'] = i * results_per_page
- data = compat_urllib_parse.urlencode(query)
+ data = compat_urllib_parse.urlencode(encode_dict(query))
next_url = '{0}{1}?{2}'.format(
self._API_V2_BASE, endpoint, data)
From 7dc011c063c5c51712605ac08cf1c01841d5867c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 Nov 2015 21:00:42 +0600
Subject: [PATCH 331/415] [soundcloud:search] Remove no track results message
---
youtube_dl/extractor/soundcloud.py | 4 ----
1 file changed, 4 deletions(-)
diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index 66bb1cb54..e880872ae 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -535,8 +535,4 @@ class SoundcloudSearchIE(SearchInfoExtractor, SoundcloudIE):
results = [self.url_result(track['uri']) for track in itertools.islice(tracks, n)]
- if not results:
- raise ExtractorError(
- 'Soundcloud said: No track results', expected=True)
-
return self.playlist_result(results, playlist_title=query)
From f6c903e708bf8d9156ce5b999a220c4480a7aca5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 Nov 2015 21:21:21 +0600
Subject: [PATCH 332/415] [soundcloud:search] Simplify (Closes #7213)
---
youtube_dl/extractor/soundcloud.py | 42 +++++++++++++-----------------
1 file changed, 18 insertions(+), 24 deletions(-)
diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index e880872ae..9b252c444 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -493,46 +493,40 @@ class SoundcloudSearchIE(SearchInfoExtractor, SoundcloudIE):
_API_V2_BASE = 'https://api-v2.soundcloud.com'
def _get_collection(self, endpoint, collection_id, **query):
- query['limit'] = results_per_page = min(
+ limit = results_per_page = min(
query.get('limit', self._DEFAULT_RESULTS_PER_PAGE),
self._MAX_RESULTS_PER_PAGE)
+ query['limit'] = limit
query['client_id'] = self._CLIENT_ID
query['linked_partitioning'] = '1'
+ query['offset'] = 0
+ data = compat_urllib_parse.urlencode(encode_dict(query))
+ next_url = '{0}{1}?{2}'.format(self._API_V2_BASE, endpoint, data)
- total_results = None
collected_results = 0
- next_url = None
-
- for i in itertools.count():
- if not next_url:
- query['offset'] = i * results_per_page
- data = compat_urllib_parse.urlencode(encode_dict(query))
- next_url = '{0}{1}?{2}'.format(
- self._API_V2_BASE, endpoint, data)
-
+ for i in itertools.count(1):
response = self._download_json(
- next_url, collection_id, 'Downloading page {0}'.format(i + 1),
+ next_url, collection_id, 'Downloading page {0}'.format(i),
'Unable to download API page')
- total_results = int(response.get(
- 'total_results', total_results))
+ collection = response.get('collection', [])
+ if not collection:
+ break
- collection = response['collection']
+ collection = list(filter(bool, collection))
collected_results += len(collection)
- for item in filter(bool, collection):
- yield item
+ for item in collection:
+ yield self.url_result(item['uri'], SoundcloudIE.ie_key())
- if (total_results is not None and collected_results >= total_results) or not collection:
+ if not collection or collected_results >= limit:
break
next_url = response.get('next_href')
+ if not next_url:
+ break
def _get_n_results(self, query, n):
- tracks = self._get_collection(
- '/search/tracks', collection_id='Query "{0}"'.format(query), limit=n, q=query)
-
- results = [self.url_result(track['uri']) for track in itertools.islice(tracks, n)]
-
- return self.playlist_result(results, playlist_title=query)
+ tracks = self._get_collection('/search/tracks', query, limit=n, q=query)
+ return self.playlist_result(tracks, playlist_title=query)
From c3a227d1c406d0af8fdf6afd9e33366e903d9a8a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 Nov 2015 21:25:48 +0600
Subject: [PATCH 333/415] [pluralsight] Update _LOGIN_URL
---
youtube_dl/extractor/pluralsight.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/pluralsight.py b/youtube_dl/extractor/pluralsight.py
index 93244f41d..417dd965c 100644
--- a/youtube_dl/extractor/pluralsight.py
+++ b/youtube_dl/extractor/pluralsight.py
@@ -19,7 +19,7 @@ from ..utils import (
class PluralsightIE(InfoExtractor):
IE_NAME = 'pluralsight'
_VALID_URL = r'https?://(?:(?:www|app)\.)?pluralsight\.com/training/player\?'
- _LOGIN_URL = 'https://www.pluralsight.com/id/'
+ _LOGIN_URL = 'https://app.pluralsight.com/id/'
_NETRC_MACHINE = 'pluralsight'
_TESTS = [{
From 0533915aad95d76b6bf4f529d39167f22a7c4b90 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 Nov 2015 21:35:08 +0600
Subject: [PATCH 334/415] [pluralsight] Update some more URLs
---
youtube_dl/extractor/pluralsight.py | 9 +++++----
1 file changed, 5 insertions(+), 4 deletions(-)
diff --git a/youtube_dl/extractor/pluralsight.py b/youtube_dl/extractor/pluralsight.py
index 417dd965c..fe6850aac 100644
--- a/youtube_dl/extractor/pluralsight.py
+++ b/youtube_dl/extractor/pluralsight.py
@@ -20,6 +20,7 @@ class PluralsightIE(InfoExtractor):
IE_NAME = 'pluralsight'
_VALID_URL = r'https?://(?:(?:www|app)\.)?pluralsight\.com/training/player\?'
_LOGIN_URL = 'https://app.pluralsight.com/id/'
+ _API_BASE = 'http://app.pluralsight.com'
_NETRC_MACHINE = 'pluralsight'
_TESTS = [{
@@ -142,7 +143,7 @@ class PluralsightIE(InfoExtractor):
'q': '%dx%d' % (f['width'], f['height']),
}
request = compat_urllib_request.Request(
- 'http://app.pluralsight.com/training/Player/ViewClip',
+ '%s/training/Player/ViewClip' % self._API_BASE,
json.dumps(clip_post).encode('utf-8'))
request.add_header('Content-Type', 'application/json;charset=utf-8')
format_id = '%s-%s' % (ext, quality)
@@ -201,14 +202,14 @@ class PluralsightCourseIE(InfoExtractor):
# TODO: PSM cookie
course = self._download_json(
- 'http://www.pluralsight.com/data/course/%s' % course_id,
+ '%s/data/course/%s' % (self._API_BASE, course_id),
course_id, 'Downloading course JSON')
title = course['title']
description = course.get('description') or course.get('shortDescription')
course_data = self._download_json(
- 'http://www.pluralsight.com/data/course/content/%s' % course_id,
+ '%s/data/course/content/%s' % (self._API_BASE, course_id),
course_id, 'Downloading course data JSON')
entries = []
@@ -218,7 +219,7 @@ class PluralsightCourseIE(InfoExtractor):
if not player_parameters:
continue
entries.append(self.url_result(
- 'http://www.pluralsight.com/training/player?%s' % player_parameters,
+ '%s/training/player?%s' % (self._API_BASE, player_parameters),
'Pluralsight'))
return self.playlist_result(entries, course_id, title, description)
From 563772eda4a42b1fce1f3740fe91bf74e63bd347 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 Nov 2015 21:37:29 +0600
Subject: [PATCH 335/415] [pluralsight] Extract base class
---
youtube_dl/extractor/pluralsight.py | 10 +++++++---
1 file changed, 7 insertions(+), 3 deletions(-)
diff --git a/youtube_dl/extractor/pluralsight.py b/youtube_dl/extractor/pluralsight.py
index fe6850aac..de7dc739b 100644
--- a/youtube_dl/extractor/pluralsight.py
+++ b/youtube_dl/extractor/pluralsight.py
@@ -16,11 +16,15 @@ from ..utils import (
)
-class PluralsightIE(InfoExtractor):
+class PluralsightBaseIE(InfoExtractor):
+ _API_BASE = 'http://app.pluralsight.com'
+
+
+class PluralsightIE(PluralsightBaseIE):
IE_NAME = 'pluralsight'
_VALID_URL = r'https?://(?:(?:www|app)\.)?pluralsight\.com/training/player\?'
_LOGIN_URL = 'https://app.pluralsight.com/id/'
- _API_BASE = 'http://app.pluralsight.com'
+
_NETRC_MACHINE = 'pluralsight'
_TESTS = [{
@@ -174,7 +178,7 @@ class PluralsightIE(InfoExtractor):
}
-class PluralsightCourseIE(InfoExtractor):
+class PluralsightCourseIE(PluralsightBaseIE):
IE_NAME = 'pluralsight:course'
_VALID_URL = r'https?://(?:(?:www|app)\.)?pluralsight\.com/(?:library/)?courses/(?P<id>[^/]+)'
_TESTS = [{
From 7e508ff2cf8f8f8b1784db6fb33994839841d122 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 Nov 2015 21:49:37 +0600
Subject: [PATCH 336/415] [pluralsight] Improve login detection
---
youtube_dl/extractor/pluralsight.py | 3 +++
1 file changed, 3 insertions(+)
diff --git a/youtube_dl/extractor/pluralsight.py b/youtube_dl/extractor/pluralsight.py
index de7dc739b..7693282a5 100644
--- a/youtube_dl/extractor/pluralsight.py
+++ b/youtube_dl/extractor/pluralsight.py
@@ -84,6 +84,9 @@ class PluralsightIE(PluralsightBaseIE):
if error:
raise ExtractorError('Unable to login: %s' % error, expected=True)
+ if all(p not in response for p in ('__INITIAL_STATE__', '"currentUser"')):
+ raise ExtractorError('Unable to log in')
+
def _real_extract(self, url):
qs = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
From bea56c95699af594586095e5ea88e9857049c6a1 Mon Sep 17 00:00:00 2001
From: Andrzej Lichnerowicz <andrzej@lichnerowicz.pl>
Date: Sat, 26 Sep 2015 21:58:33 +0200
Subject: [PATCH 337/415] [pluralsight] prevent error 429 when sensing video
formats
---
youtube_dl/extractor/pluralsight.py | 3 +++
1 file changed, 3 insertions(+)
diff --git a/youtube_dl/extractor/pluralsight.py b/youtube_dl/extractor/pluralsight.py
index 7693282a5..d542a9e0e 100644
--- a/youtube_dl/extractor/pluralsight.py
+++ b/youtube_dl/extractor/pluralsight.py
@@ -156,6 +156,9 @@ class PluralsightIE(PluralsightBaseIE):
format_id = '%s-%s' % (ext, quality)
clip_url = self._download_webpage(
request, display_id, 'Downloading %s URL' % format_id, fatal=False)
+ # #6989: sleep 3 seconds to avoid 429 errors.
+ # should help with #6842.
+ self._sleep(3, display_id)
if not clip_url:
continue
f.update({
From 38eb2968abce9078f28904046b8e26b0d87ada61 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Nov 2015 00:07:09 +0600
Subject: [PATCH 338/415] [pluralsight] Clarify and randomize ViewClip sleep
interval
---
youtube_dl/extractor/pluralsight.py | 15 ++++++++++++---
1 file changed, 12 insertions(+), 3 deletions(-)
diff --git a/youtube_dl/extractor/pluralsight.py b/youtube_dl/extractor/pluralsight.py
index d542a9e0e..8481a10b0 100644
--- a/youtube_dl/extractor/pluralsight.py
+++ b/youtube_dl/extractor/pluralsight.py
@@ -1,6 +1,7 @@
from __future__ import unicode_literals
import json
+import random
from .common import InfoExtractor
from ..compat import (
@@ -156,9 +157,17 @@ class PluralsightIE(PluralsightBaseIE):
format_id = '%s-%s' % (ext, quality)
clip_url = self._download_webpage(
request, display_id, 'Downloading %s URL' % format_id, fatal=False)
- # #6989: sleep 3 seconds to avoid 429 errors.
- # should help with #6842.
- self._sleep(3, display_id)
+
+ # Pluralsight tracks multiple sequential calls to ViewClip API and start
+ # to return 429 HTTP errors after some time (see
+ # https://github.com/rg3/youtube-dl/pull/6989). Moreover it may even lead
+ # to account ban (see https://github.com/rg3/youtube-dl/issues/6842).
+ # To somewhat reduce the probability of these consequences
+ # we will sleep random amount of time before each call to ViewClip.
+ self._sleep(
+ random.randint(2, 5), display_id,
+ '%(video_id)s: Waiting for %(timeout)s seconds to avoid throttling')
+
if not clip_url:
continue
f.update({
From 4c57b4853d8c1b41d729314e316cc1b83129ea18 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Nov 2015 00:42:58 +0600
Subject: [PATCH 339/415] [pluralsight] Until listing formats request only
single format
---
youtube_dl/extractor/pluralsight.py | 24 +++++++++++++++++++++---
1 file changed, 21 insertions(+), 3 deletions(-)
diff --git a/youtube_dl/extractor/pluralsight.py b/youtube_dl/extractor/pluralsight.py
index 8481a10b0..16de0fd1c 100644
--- a/youtube_dl/extractor/pluralsight.py
+++ b/youtube_dl/extractor/pluralsight.py
@@ -2,6 +2,7 @@ from __future__ import unicode_literals
import json
import random
+import collections
from .common import InfoExtractor
from ..compat import (
@@ -131,13 +132,30 @@ class PluralsightIE(PluralsightBaseIE):
'high': {'width': 1024, 'height': 768},
}
+ AllowedQuality = collections.namedtuple('AllowedQuality', ['ext', 'qualities'])
+
ALLOWED_QUALITIES = (
- ('webm', ('high',)),
- ('mp4', ('low', 'medium', 'high',)),
+ AllowedQuality('webm', ('high',)),
+ AllowedQuality('mp4', ('low', 'medium', 'high',)),
)
+ if self._downloader.params.get('listformats', False):
+ allowed_qualities = ALLOWED_QUALITIES
+ else:
+ def guess_allowed_qualities():
+ req_format = self._downloader.params.get('format') or 'best'
+ req_format_split = req_format.split('-')
+ if len(req_format_split) > 1:
+ req_ext, req_quality = req_format_split
+ for allowed_quality in ALLOWED_QUALITIES:
+ if req_ext == allowed_quality.ext and req_quality in allowed_quality.qualities:
+ return (AllowedQuality(req_ext, (req_quality, )), )
+ req_ext = 'webm' if self._downloader.params.get('prefer_free_formats') else 'mp4'
+ return (AllowedQuality(req_ext, ('high', )), )
+ allowed_qualities = guess_allowed_qualities()
+
formats = []
- for ext, qualities in ALLOWED_QUALITIES:
+ for ext, qualities in allowed_qualities:
for quality in qualities:
f = QUALITIES[quality].copy()
clip_post = {
From a3372437bfe57f5b450bfdd60105bb6120595928 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Nov 2015 00:49:58 +0600
Subject: [PATCH 340/415] [soundcloud] Remove unused variable
---
youtube_dl/extractor/soundcloud.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index 9b252c444..02e64e094 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -493,7 +493,7 @@ class SoundcloudSearchIE(SearchInfoExtractor, SoundcloudIE):
_API_V2_BASE = 'https://api-v2.soundcloud.com'
def _get_collection(self, endpoint, collection_id, **query):
- limit = results_per_page = min(
+ limit = min(
query.get('limit', self._DEFAULT_RESULTS_PER_PAGE),
self._MAX_RESULTS_PER_PAGE)
query['limit'] = limit
From cf186b77a79a0bf6f322bc1e2f610471eab3b01c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Nov 2015 00:56:40 +0600
Subject: [PATCH 341/415] [pluralsight] Clarify allowed qualities guessing
rationale
---
youtube_dl/extractor/pluralsight.py | 3 +++
1 file changed, 3 insertions(+)
diff --git a/youtube_dl/extractor/pluralsight.py b/youtube_dl/extractor/pluralsight.py
index 16de0fd1c..3dee616c1 100644
--- a/youtube_dl/extractor/pluralsight.py
+++ b/youtube_dl/extractor/pluralsight.py
@@ -139,6 +139,9 @@ class PluralsightIE(PluralsightBaseIE):
AllowedQuality('mp4', ('low', 'medium', 'high',)),
)
+ # In order to minimize the number of calls to ViewClip API and reduce
+ # the probability of being throttled or banned by Pluralsight we will request
+ # only single format until explicit listformats was requested.
if self._downloader.params.get('listformats', False):
allowed_qualities = ALLOWED_QUALITIES
else:
From 0eebf34d9dd793bce9148dae267d690d50d89f88 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Nov 2015 00:58:25 +0600
Subject: [PATCH 342/415] [pluralsight] Rephrase
---
youtube_dl/extractor/pluralsight.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/pluralsight.py b/youtube_dl/extractor/pluralsight.py
index 3dee616c1..a11cebaf1 100644
--- a/youtube_dl/extractor/pluralsight.py
+++ b/youtube_dl/extractor/pluralsight.py
@@ -141,7 +141,7 @@ class PluralsightIE(PluralsightBaseIE):
# In order to minimize the number of calls to ViewClip API and reduce
# the probability of being throttled or banned by Pluralsight we will request
- # only single format until explicit listformats was requested.
+ # only single format until formats listing was explicitly requested.
if self._downloader.params.get('listformats', False):
allowed_qualities = ALLOWED_QUALITIES
else:
From 0c14841585db1baa2f9a4a5ff263035977cf0964 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Nov 2015 04:17:07 +0600
Subject: [PATCH 343/415] [youtube:user:playlists] Add extractor (Closes #3817)
---
youtube_dl/extractor/__init__.py | 1 +
youtube_dl/extractor/youtube.py | 26 ++++++++++++++++++++++++++
2 files changed, 27 insertions(+)
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 232bcd89a..947b83683 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -834,6 +834,7 @@ from .youtube import (
YoutubeTruncatedIDIE,
YoutubeTruncatedURLIE,
YoutubeUserIE,
+ YoutubeUserPlaylistsIE,
YoutubeWatchLaterIE,
)
from .zapiks import ZapiksIE
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 364ca102a..abc67f07f 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -224,6 +224,17 @@ class YoutubePlaylistBaseInfoExtractor(InfoExtractor):
return zip(ids_in_page, titles_in_page)
+class YoutubePlaylistsBaseInfoExtractor(InfoExtractor):
+ def _real_extract(self, url):
+ playlist_id = self._match_id(url)
+ webpage = self._download_webpage(url, playlist_id)
+ entries = [
+ self.url_result(compat_urlparse.urljoin(url, playlist), 'YoutubePlaylist')
+ for playlist in re.findall(r'href="(/playlist\?list=.+?)"', webpage)]
+ title = self._og_search_title(webpage, fatal=False)
+ return self.playlist_result(entries, playlist_id, title)
+
+
class YoutubeIE(YoutubeBaseInfoExtractor):
IE_DESC = 'YouTube.com'
_VALID_URL = r"""(?x)^
@@ -1742,6 +1753,21 @@ class YoutubeUserIE(YoutubeChannelIE):
return super(YoutubeUserIE, cls).suitable(url)
+class YoutubeUserPlaylistsIE(YoutubePlaylistsBaseInfoExtractor):
+ IE_DESC = 'YouTube.com user playlists'
+ _VALID_URL = r'https?://(?:\w+\.)?youtube\.com/user/(?P<id>[^/]+)/playlists'
+ IE_NAME = 'youtube:user:playlists'
+
+ _TEST = {
+ 'url': 'http://www.youtube.com/user/ThirstForScience/playlists',
+ 'playlist_mincount': 4,
+ 'info_dict': {
+ 'id': 'ThirstForScience',
+ 'title': 'Thirst for Science',
+ },
+ }
+
+
class YoutubeSearchIE(SearchInfoExtractor, YoutubePlaylistIE):
IE_DESC = 'YouTube.com searches'
# there doesn't appear to be a real limit, for example if you search for
From 136dadde9543a80f490b26c822dcfdff5541c335 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Nov 2015 04:18:20 +0600
Subject: [PATCH 344/415] [youtube:show] Rework in terms of playlists base
extractor
---
youtube_dl/extractor/youtube.py | 25 ++++---------------------
1 file changed, 4 insertions(+), 21 deletions(-)
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index abc67f07f..c56f8a0a2 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1863,7 +1863,7 @@ class YoutubeSearchURLIE(InfoExtractor):
}
-class YoutubeShowIE(InfoExtractor):
+class YoutubeShowIE(YoutubePlaylistsBaseInfoExtractor):
IE_DESC = 'YouTube.com (multi-season) shows'
_VALID_URL = r'https?://www\.youtube\.com/show/(?P<id>[^?#]*)'
IE_NAME = 'youtube:show'
@@ -1877,26 +1877,9 @@ class YoutubeShowIE(InfoExtractor):
}]
def _real_extract(self, url):
- mobj = re.match(self._VALID_URL, url)
- playlist_id = mobj.group('id')
- webpage = self._download_webpage(
- 'https://www.youtube.com/show/%s/playlists' % playlist_id, playlist_id, 'Downloading show webpage')
- # There's one playlist for each season of the show
- m_seasons = list(re.finditer(r'href="(/playlist\?list=.*?)"', webpage))
- self.to_screen('%s: Found %s seasons' % (playlist_id, len(m_seasons)))
- entries = [
- self.url_result(
- 'https://www.youtube.com' + season.group(1), 'YoutubePlaylist')
- for season in m_seasons
- ]
- title = self._og_search_title(webpage, fatal=False)
-
- return {
- '_type': 'playlist',
- 'id': playlist_id,
- 'title': title,
- 'entries': entries,
- }
+ playlist_id = self._match_id(url)
+ return super(YoutubeShowIE, self)._real_extract(
+ 'https://www.youtube.com/show/%s/playlists' % playlist_id)
class YoutubeFeedsInfoExtractor(YoutubeBaseInfoExtractor):
From 82c4d7b0ce49b13a9dd8491dd32e962ca307c974 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 21 Nov 2015 23:36:27 +0100
Subject: [PATCH 345/415] release 2015.11.21
---
docs/supportedsites.md | 2 ++
youtube_dl/version.py | 2 +-
2 files changed, 3 insertions(+), 1 deletion(-)
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index f56a6986b..1df408610 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -494,6 +494,7 @@
- **soompi:show**
- **soundcloud**
- **soundcloud:playlist**
+ - **soundcloud:search**: Soundcloud search
- **soundcloud:set**
- **soundcloud:user**
- **soundgasm**
@@ -707,6 +708,7 @@
- **youtube:show**: YouTube.com (multi-season) shows
- **youtube:subscriptions**: YouTube.com subscriptions feed, "ytsubs" keyword (requires authentication)
- **youtube:user**: YouTube.com user videos (URL or "ytuser" keyword)
+ - **youtube:user:playlists**: YouTube.com user playlists
- **youtube:watchlater**: Youtube watch later list, ":ytwatchlater" for short (requires authentication)
- **Zapiks**
- **ZDF**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index b3b1b90ba..2baf1ac42 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
from __future__ import unicode_literals
-__version__ = '2015.11.19'
+__version__ = '2015.11.21'
From 061a75edd6bab2f30978b458fe7402ff9e9c02a5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Nov 2015 05:01:01 +0600
Subject: [PATCH 346/415] [youtube] Extract base for entry list extractors and
support multi page lists of playlists
---
youtube_dl/extractor/youtube.py | 28 +++++++++++++++++-----------
1 file changed, 17 insertions(+), 11 deletions(-)
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index c56f8a0a2..8352ad1da 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -178,15 +178,13 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
return
-class YoutubePlaylistBaseInfoExtractor(InfoExtractor):
- # Extract the video ids from the playlist pages
+class YoutubeEntryListBaseInfoExtractor(InfoExtractor):
+ # Extract entries from page with "Load more" button
def _entries(self, page, playlist_id):
more_widget_html = content_html = page
for page_num in itertools.count(1):
- for video_id, video_title in self.extract_videos_from_page(content_html):
- yield self.url_result(
- video_id, 'Youtube', video_id=video_id,
- video_title=video_title)
+ for entry in self._process_page(content_html):
+ yield entry
mobj = re.search(r'data-uix-load-more-href="/?(?P<more>[^"]+)"', more_widget_html)
if not mobj:
@@ -203,6 +201,12 @@ class YoutubePlaylistBaseInfoExtractor(InfoExtractor):
break
more_widget_html = more['load_more_widget_html']
+
+class YoutubePlaylistBaseInfoExtractor(YoutubeEntryListBaseInfoExtractor):
+ def _process_page(self, content):
+ for video_id, video_title in self.extract_videos_from_page(content):
+ yield self.url_result(video_id, 'Youtube', video_id, video_title)
+
def extract_videos_from_page(self, page):
ids_in_page = []
titles_in_page = []
@@ -224,15 +228,17 @@ class YoutubePlaylistBaseInfoExtractor(InfoExtractor):
return zip(ids_in_page, titles_in_page)
-class YoutubePlaylistsBaseInfoExtractor(InfoExtractor):
+class YoutubePlaylistsBaseInfoExtractor(YoutubeEntryListBaseInfoExtractor):
+ def _process_page(self, content):
+ for playlist_id in re.findall(r'href="/?playlist\?list=(.+?)"', content):
+ yield self.url_result(
+ 'https://www.youtube.com/playlist?list=%s' % playlist_id, 'YoutubePlaylist')
+
def _real_extract(self, url):
playlist_id = self._match_id(url)
webpage = self._download_webpage(url, playlist_id)
- entries = [
- self.url_result(compat_urlparse.urljoin(url, playlist), 'YoutubePlaylist')
- for playlist in re.findall(r'href="(/playlist\?list=.+?)"', webpage)]
title = self._og_search_title(webpage, fatal=False)
- return self.playlist_result(entries, playlist_id, title)
+ return self.playlist_result(self._entries(webpage, playlist_id), playlist_id, title)
class YoutubeIE(YoutubeBaseInfoExtractor):
From e568c2233e7b4b27c9a5c56322ab7633a5f0b1f7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Nov 2015 05:03:23 +0600
Subject: [PATCH 347/415] [youtube] Add test for multi page list of playlists
---
youtube_dl/extractor/youtube.py | 12 ++++++++++--
1 file changed, 10 insertions(+), 2 deletions(-)
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 8352ad1da..4a0ff6e9c 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1764,14 +1764,22 @@ class YoutubeUserPlaylistsIE(YoutubePlaylistsBaseInfoExtractor):
_VALID_URL = r'https?://(?:\w+\.)?youtube\.com/user/(?P<id>[^/]+)/playlists'
IE_NAME = 'youtube:user:playlists'
- _TEST = {
+ _TESTS = [{
'url': 'http://www.youtube.com/user/ThirstForScience/playlists',
'playlist_mincount': 4,
'info_dict': {
'id': 'ThirstForScience',
'title': 'Thirst for Science',
},
- }
+ }, {
+ # with "Load more" button
+ 'url': 'http://www.youtube.com/user/igorkle1/playlists?view=1&sort=dd',
+ 'playlist_mincount': 70,
+ 'info_dict': {
+ 'id': 'igorkle1',
+ 'title': 'Игорь Клейнер',
+ },
+ }]
class YoutubeSearchIE(SearchInfoExtractor, YoutubePlaylistIE):
From 3e12bc583af9d5abf5f144ed6e092c59f4b83fdf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Nov 2015 06:29:39 +0600
Subject: [PATCH 348/415] [utils] Improve determine_ext (Closes #7593)
---
youtube_dl/utils.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index bff59eb73..7dab60bb8 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -922,7 +922,7 @@ def unified_strdate(date_str, day_first=True):
def determine_ext(url, default_ext='unknown_video'):
if url is None:
return default_ext
- guess = url.partition('?')[0].rpartition('.')[2]
+ guess = url.partition('?')[0].rpartition('.')[2].rstrip('/')
if re.match(r'^[A-Za-z0-9]+$', guess):
return guess
else:
From 5035536e3f32d4c47b2d3067c12e074cb9a4a199 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Nov 2015 06:33:52 +0600
Subject: [PATCH 349/415] [test_utils] Add tests for determine_ext
---
test/test_utils.py | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/test/test_utils.py b/test/test_utils.py
index ea1ff0547..4f0d9e481 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -21,6 +21,7 @@ from youtube_dl.utils import (
clean_html,
DateRange,
detect_exe_version,
+ determine_ext,
encodeFilename,
escape_rfc3986,
escape_url,
@@ -238,6 +239,10 @@ class TestUtil(unittest.TestCase):
self.assertEqual(unified_strdate('25-09-2014'), '20140925')
self.assertEqual(unified_strdate('UNKNOWN DATE FORMAT'), None)
+ def test_determine_ext(self):
+ self.assertEqual(determine_ext('http://example.com/foo/bar.mp4/?download'), 'mp4')
+ self.assertEqual(determine_ext('http://example.com/foo/bar/?download', None), None)
+
def test_find_xpath_attr(self):
testxml = '''<root>
<node/>
From 9cb9a5df7794579c38efff1c4b1451a7d13da3c1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Nov 2015 17:27:13 +0600
Subject: [PATCH 350/415] [utils] Check ext with trailing slash against the
list of known extensions
---
test/test_utils.py | 3 +++
youtube_dl/utils.py | 17 ++++++++++++++++-
2 files changed, 19 insertions(+), 1 deletion(-)
diff --git a/test/test_utils.py b/test/test_utils.py
index 4f0d9e481..501355c74 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -242,6 +242,9 @@ class TestUtil(unittest.TestCase):
def test_determine_ext(self):
self.assertEqual(determine_ext('http://example.com/foo/bar.mp4/?download'), 'mp4')
self.assertEqual(determine_ext('http://example.com/foo/bar/?download', None), None)
+ self.assertEqual(determine_ext('http://example.com/foo/bar.nonext/?download', None), None)
+ self.assertEqual(determine_ext('http://example.com/foo/bar/mp4?download', None), None)
+ self.assertEqual(determine_ext('http://example.com/foo/bar.m3u8//?download'), 'm3u8')
def test_find_xpath_attr(self):
testxml = '''<root>
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 7dab60bb8..c0325f054 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -922,9 +922,24 @@ def unified_strdate(date_str, day_first=True):
def determine_ext(url, default_ext='unknown_video'):
if url is None:
return default_ext
- guess = url.partition('?')[0].rpartition('.')[2].rstrip('/')
+ guess = url.partition('?')[0].rpartition('.')[2]
if re.match(r'^[A-Za-z0-9]+$', guess):
return guess
+ elif guess.rstrip('/') in (
+ 'mp4', 'm4a', 'm4p', 'm4b', 'm4r', 'm4v', 'aac',
+ 'flv', 'f4v', 'f4a', 'f4b',
+ 'webm', 'ogg', 'ogv', 'oga', 'ogx', 'spx', 'opus',
+ 'mkv', 'mka', 'mk3d',
+ 'avi', 'divx',
+ 'mov',
+ 'asf', 'wmv', 'wma',
+ '3gp', '3g2',
+ 'mp3',
+ 'flac',
+ 'ape',
+ 'wav',
+ 'f4f', 'f4m', 'm3u8', 'smil'):
+ return guess.rstrip('/')
else:
return default_ext
From 1b38185361e096d6e34db11adac7333ac9dadca0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Nov 2015 18:08:30 +0600
Subject: [PATCH 351/415] [pornhd] Fix title extraction (Closes #7596)
---
youtube_dl/extractor/pornhd.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/pornhd.py b/youtube_dl/extractor/pornhd.py
index dbb2c3bd9..57c78ba52 100644
--- a/youtube_dl/extractor/pornhd.py
+++ b/youtube_dl/extractor/pornhd.py
@@ -36,7 +36,8 @@ class PornHdIE(InfoExtractor):
webpage = self._download_webpage(url, display_id or video_id)
title = self._html_search_regex(
- r'<title>(.+) porn HD.+?</title>', webpage, 'title')
+ [r'<span[^>]+class=["\']video-name["\'][^>]*>([^<]+)',
+ r'<title>(.+?) - .*?[Pp]ornHD.*?</title>'], webpage, 'title')
description = self._html_search_regex(
r'<div class="description">([^<]+)</div>', webpage, 'description', fatal=False)
view_count = int_or_none(self._html_search_regex(
From 3cfd000849208b58dab4f78d1486d3f24552009e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Lalinsk=C3=BD?= <lukas@oxygene.sk>
Date: Sun, 22 Nov 2015 13:14:35 +0100
Subject: [PATCH 352/415] [youtube] More explicit player config JSON extraction
(fixes #7468)
---
youtube_dl/extractor/youtube.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 687e0b4db..21731188a 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1074,7 +1074,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
age_gate = False
video_info = None
# Try looking directly into the video webpage
- mobj = re.search(r';ytplayer\.config\s*=\s*({.*?});', video_webpage)
+ mobj = re.search(r';ytplayer\.config\s*=\s*({.*?});ytplayer', video_webpage)
if mobj:
json_code = uppercase_escape(mobj.group(1))
ytplayer_config = json.loads(json_code)
From 4a7d108ab3c8b5ac82b8740179dae6a454218a38 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Nov 2015 18:24:17 +0600
Subject: [PATCH 353/415] [rutube] Remove unnecessary print
---
youtube_dl/extractor/rutube.py | 1 -
1 file changed, 1 deletion(-)
diff --git a/youtube_dl/extractor/rutube.py b/youtube_dl/extractor/rutube.py
index 0db8c410d..6b09550b0 100644
--- a/youtube_dl/extractor/rutube.py
+++ b/youtube_dl/extractor/rutube.py
@@ -54,7 +54,6 @@ class RutubeIE(InfoExtractor):
formats = []
for format_id, format_url in options['video_balancer'].items():
ext = determine_ext(format_url)
- print(ext)
if ext == 'm3u8':
m3u8_formats = self._extract_m3u8_formats(
format_url, video_id, 'mp4', m3u8_id=format_id, fatal=False)
From 0e49d9a6b0216555c2a3ee063ae3d1c6d09edbd4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Lalinsk=C3=BD?= <lukas@oxygene.sk>
Date: Sun, 22 Nov 2015 13:49:33 +0100
Subject: [PATCH 354/415] [youtube] Fall back to the original regex for
ytplayer.config
---
youtube_dl/extractor/youtube.py | 39 ++++++++++++++++++++++++++-------
1 file changed, 31 insertions(+), 8 deletions(-)
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 21731188a..7e74d2368 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -674,7 +674,23 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
{
'url': 'http://vid.plus/FlRa-iH7PGw',
'only_matching': True,
- }
+ },
+ {
+ # Title with JS-like syntax "};"
+ 'url': 'https://www.youtube.com/watch?v=lsguqyKfVQg',
+ 'info_dict': {
+ 'id': 'lsguqyKfVQg',
+ 'ext': 'mp4',
+ 'title': '{dark walk}; Loki/AC/Dishonored; collab w/Elflover21',
+ 'description': 'md5:8085699c11dc3f597ce0410b0dcbb34a',
+ 'upload_date': '20151119',
+ 'uploader_id': 'IronSoulElf',
+ 'uploader': 'IronSoulElf',
+ },
+ 'params': {
+ 'skip_download': True,
+ },
+ },
]
def __init__(self, *args, **kwargs):
@@ -858,16 +874,25 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
return {}
return sub_lang_list
+ def _get_ytplayer_config(self, webpage):
+ patterns = [
+ r';ytplayer\.config\s*=\s*({.*?});ytplayer',
+ r';ytplayer\.config\s*=\s*({.*?});',
+ ]
+ for pattern in patterns:
+ config = self._search_regex(pattern, webpage, 'ytconfig.player', default=None)
+ if config is not None:
+ return json.loads(uppercase_escape(config))
+
def _get_automatic_captions(self, video_id, webpage):
"""We need the webpage for getting the captions url, pass it as an
argument to speed up the process."""
self.to_screen('%s: Looking for automatic captions' % video_id)
- mobj = re.search(r';ytplayer.config = ({.*?});', webpage)
+ player_config = self._get_ytplayer_config(webpage)
err_msg = 'Couldn\'t find automatic captions for %s' % video_id
- if mobj is None:
+ if player_config is None:
self._downloader.report_warning(err_msg)
return {}
- player_config = json.loads(mobj.group(1))
try:
args = player_config['args']
caption_url = args['ttsurl']
@@ -1074,10 +1099,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
age_gate = False
video_info = None
# Try looking directly into the video webpage
- mobj = re.search(r';ytplayer\.config\s*=\s*({.*?});ytplayer', video_webpage)
- if mobj:
- json_code = uppercase_escape(mobj.group(1))
- ytplayer_config = json.loads(json_code)
+ ytplayer_config = self._get_ytplayer_config(video_webpage)
+ if ytplayer_config is not None:
args = ytplayer_config['args']
if args.get('url_encoded_fmt_stream_map'):
# Convert to the same format returned by compat_parse_qs
From b41631c4e6e56afb2427513c84df1b13681cf4c8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Lalinsk=C3=BD?= <lukas@oxygene.sk>
Date: Sun, 22 Nov 2015 13:53:26 +0100
Subject: [PATCH 355/415] [youtube] Send the list of patterns directly to
_search_regex
---
youtube_dl/extractor/youtube.py | 7 +++----
1 file changed, 3 insertions(+), 4 deletions(-)
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 7e74d2368..247769067 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -879,10 +879,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
r';ytplayer\.config\s*=\s*({.*?});ytplayer',
r';ytplayer\.config\s*=\s*({.*?});',
]
- for pattern in patterns:
- config = self._search_regex(pattern, webpage, 'ytconfig.player', default=None)
- if config is not None:
- return json.loads(uppercase_escape(config))
+ config = self._search_regex(patterns, webpage, 'ytconfig.player', default=None)
+ if config is not None:
+ return json.loads(uppercase_escape(config))
def _get_automatic_captions(self, video_id, webpage):
"""We need the webpage for getting the captions url, pass it as an
From 02f0da20b00fa0efee6abcd1842c183b3a4cc36c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 23 Nov 2015 03:08:38 +0600
Subject: [PATCH 356/415] [pluralsight] Add support for alternative webpage
layout (Closes #7607)
---
youtube_dl/extractor/pluralsight.py | 22 ++++++++++++++++------
1 file changed, 16 insertions(+), 6 deletions(-)
diff --git a/youtube_dl/extractor/pluralsight.py b/youtube_dl/extractor/pluralsight.py
index a11cebaf1..792316db8 100644
--- a/youtube_dl/extractor/pluralsight.py
+++ b/youtube_dl/extractor/pluralsight.py
@@ -104,19 +104,29 @@ class PluralsightIE(PluralsightBaseIE):
webpage = self._download_webpage(url, display_id)
- collection = self._parse_json(
- self._search_regex(
- r'moduleCollection\s*:\s*new\s+ModuleCollection\((\[.+?\])\s*,\s*\$rootScope\)',
- webpage, 'modules'),
- display_id)
+ modules = self._search_regex(
+ r'moduleCollection\s*:\s*new\s+ModuleCollection\((\[.+?\])\s*,\s*\$rootScope\)',
+ webpage, 'modules', default=None)
+
+ if modules:
+ collection = self._parse_json(modules, display_id)
+ else:
+ # Webpage may be served in different layout (see
+ # https://github.com/rg3/youtube-dl/issues/7607)
+ collection = self._parse_json(
+ self._search_regex(
+ r'var\s+initialState\s*=\s*({.+?});\n', webpage, 'initial state'),
+ display_id)['course']['modules']
module, clip = None, None
for module_ in collection:
- if module_.get('moduleName') == name:
+ if name in (module_.get('moduleName'), module_.get('name')):
module = module_
for clip_ in module_.get('clips', []):
clip_index = clip_.get('clipIndex')
+ if clip_index is None:
+ clip_index = clip_.get('index')
if clip_index is None:
continue
if compat_str(clip_index) == clip_id:
From a72778d364022612ba88bdfd9affef0d7b0ca864 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 23 Nov 2015 21:00:06 +0600
Subject: [PATCH 357/415] [youtube] Improve ytplayer.config extraction
---
youtube_dl/extractor/youtube.py | 26 ++++++++++++++------------
1 file changed, 14 insertions(+), 12 deletions(-)
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 1580c54fe..052f6922a 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -891,22 +891,24 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
return {}
return sub_lang_list
- def _get_ytplayer_config(self, webpage):
- patterns = [
- r';ytplayer\.config\s*=\s*({.*?});ytplayer',
- r';ytplayer\.config\s*=\s*({.*?});',
- ]
- config = self._search_regex(patterns, webpage, 'ytconfig.player', default=None)
- if config is not None:
- return json.loads(uppercase_escape(config))
+ def _get_ytplayer_config(self, video_id, webpage):
+ patterns = (
+ r';ytplayer\.config\s*=\s*({.+?});ytplayer',
+ r';ytplayer\.config\s*=\s*({.+?});',
+ )
+ config = self._search_regex(
+ patterns, webpage, 'ytplayer.config', default=None)
+ if config:
+ return self._parse_json(
+ uppercase_escape(config), video_id, fatal=False)
def _get_automatic_captions(self, video_id, webpage):
"""We need the webpage for getting the captions url, pass it as an
argument to speed up the process."""
self.to_screen('%s: Looking for automatic captions' % video_id)
- player_config = self._get_ytplayer_config(webpage)
+ player_config = self._get_ytplayer_config(video_id, webpage)
err_msg = 'Couldn\'t find automatic captions for %s' % video_id
- if player_config is None:
+ if not player_config:
self._downloader.report_warning(err_msg)
return {}
try:
@@ -1115,8 +1117,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
age_gate = False
video_info = None
# Try looking directly into the video webpage
- ytplayer_config = self._get_ytplayer_config(video_webpage)
- if ytplayer_config is not None:
+ ytplayer_config = self._get_ytplayer_config(video_id, video_webpage)
+ if ytplayer_config:
args = ytplayer_config['args']
if args.get('url_encoded_fmt_stream_map'):
# Convert to the same format returned by compat_parse_qs
From 61f92af1cfacb9a5a6e368d0093fb71dbac0af6b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 23 Nov 2015 21:02:37 +0600
Subject: [PATCH 358/415] [youtube] Add test with '};' in tags
---
youtube_dl/extractor/youtube.py | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 052f6922a..824335d0a 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -693,7 +693,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
'only_matching': True,
},
{
- # Title with JS-like syntax "};"
+ # Title with JS-like syntax "};" (see https://github.com/rg3/youtube-dl/issues/7468)
'url': 'https://www.youtube.com/watch?v=lsguqyKfVQg',
'info_dict': {
'id': 'lsguqyKfVQg',
@@ -708,6 +708,11 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
'skip_download': True,
},
},
+ {
+ # Tags with '};' (see https://github.com/rg3/youtube-dl/issues/7468)
+ 'url': 'https://www.youtube.com/watch?v=Ms7iBXnlUO8',
+ 'only_matching': True,
+ },
]
def __init__(self, *args, **kwargs):
From 526b3b071632bc3c840ae4dd3579e015f41df6f5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 23 Nov 2015 21:14:03 +0600
Subject: [PATCH 359/415] [youtube] Clarify ytplayer.config extraction
rationale
---
youtube_dl/extractor/youtube.py | 7 +++++++
1 file changed, 7 insertions(+)
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 824335d0a..5482aac3b 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -898,6 +898,13 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
def _get_ytplayer_config(self, video_id, webpage):
patterns = (
+ # User data may contain arbitrary character sequences that may affect
+ # JSON extraction with regex, e.g. when '};' is contained the second
+ # regex won't capture the whole JSON. Yet working around by trying more
+ # concrete regex first keeping in mind proper quoted string handling
+ # to be implemented in future that will replace this workaround (see
+ # https://github.com/rg3/youtube-dl/issues/7468,
+ # https://github.com/rg3/youtube-dl/pull/7599)
r';ytplayer\.config\s*=\s*({.+?});ytplayer',
r';ytplayer\.config\s*=\s*({.+?});',
)
From 94bfcd23b7e2488b5e4bbf076965ab9ed980f1ca Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 23 Nov 2015 21:35:23 +0600
Subject: [PATCH 360/415] [youtube] Fix test
---
youtube_dl/extractor/youtube.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 5482aac3b..e0e345496 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -426,7 +426,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
'title': 'Principal Sexually Assaults A Teacher - Episode 117 - 8th June 2012',
'description': 'md5:09b78bd971f1e3e289601dfba15ca4f7',
'uploader': 'SET India',
- 'uploader_id': 'setindia'
+ 'uploader_id': 'setindia',
+ 'age_limit': 18,
}
},
{
From 9022726446c659f2bc38556105991e9797e0c8c8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 23 Nov 2015 21:37:21 +0600
Subject: [PATCH 361/415] [youtube] Fix test
---
youtube_dl/extractor/youtube.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index e0e345496..0246050c2 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -564,7 +564,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
'info_dict': {
'id': 'lqQg6PlCWgI',
'ext': 'mp4',
- 'upload_date': '20120724',
+ 'upload_date': '20150827',
'uploader_id': 'olympic',
'description': 'HO09 - Women - GER-AUS - Hockey - 31 July 2012 - London 2012 Olympic Games',
'uploader': 'Olympics',
From 13a10d5aa336be7c301a6d09eb4e9d7b50f51191 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 20 Nov 2015 03:08:01 +0600
Subject: [PATCH 362/415] [compat] Add compat_urllib_request_Request
This is actually not a compatibility routine but rather a workaround for URLs without protocol specified.
The protocol-less URL is treated as HTTP one since it's most probable scenario and it will most likely to
redirect to HTTPS if HTTPS was actually expected. This routine could also be useful for any Request
preprocessing that may be added in future.
---
youtube_dl/compat.py | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index a3e85264a..4e3de7f51 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -198,6 +198,14 @@ except ImportError: # Python < 3.4
return compat_urllib_response.addinfourl(io.BytesIO(data), headers, url)
+
+# Prepend protocol-less URLs with `http:` scheme in order to mitigate the number of
+# unwanted failures due to missing protocol
+def compat_urllib_request_Request(url, *args, **kwargs):
+ return compat_urllib_request.Request(
+ 'http:%s' % url if url.startswith('//') else url, *args, **kwargs)
+
+
try:
compat_basestring = basestring # Python 2
except NameError:
From 82d8a8b6e2eadb2c1ac8684ce9876f40925f4e2e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 20 Nov 2015 03:08:34 +0600
Subject: [PATCH 363/415] [YoutubeDL] Wrap plain-text URL requests in
compat_urllib_request_Request
---
youtube_dl/YoutubeDL.py | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index fba99af8d..b20837ce2 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -28,6 +28,7 @@ if os.name == 'nt':
import ctypes
from .compat import (
+ compat_basestring,
compat_cookiejar,
compat_expanduser,
compat_get_terminal_size,
@@ -38,6 +39,7 @@ from .compat import (
compat_urllib_error,
compat_urllib_request,
compat_urllib_request_DataHandler,
+ compat_urllib_request_Request,
)
from .utils import (
ContentTooShortError,
@@ -1871,6 +1873,8 @@ class YoutubeDL(object):
def urlopen(self, req):
""" Start an HTTP download """
+ if isinstance(req, compat_basestring):
+ req = compat_urllib_request_Request(req)
return self._opener.open(req, timeout=self._socket_timeout)
def print_debug_header(self):
From e4c4bcf36f4e0bb575526701f852152f4fd976c4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 20 Nov 2015 03:12:54 +0600
Subject: [PATCH 364/415] [vimeo] Use compat_urllib_request_Request
---
youtube_dl/extractor/vimeo.py | 14 +++++++-------
1 file changed, 7 insertions(+), 7 deletions(-)
diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 057c72f39..d1daf9672 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -8,7 +8,7 @@ import itertools
from .common import InfoExtractor
from ..compat import (
compat_HTTPError,
- compat_urllib_request,
+ compat_urllib_request_Request,
compat_urlparse,
)
from ..utils import (
@@ -47,7 +47,7 @@ class VimeoBaseInfoExtractor(InfoExtractor):
'service': 'vimeo',
'token': token,
}))
- login_request = compat_urllib_request.Request(self._LOGIN_URL, data)
+ login_request = compat_urllib_request_Request(self._LOGIN_URL, data)
login_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
login_request.add_header('Referer', self._LOGIN_URL)
self._set_vimeo_cookie('vuid', vuid)
@@ -222,7 +222,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
if url.startswith('http://'):
# vimeo only supports https now, but the user can give an http url
url = url.replace('http://', 'https://')
- password_request = compat_urllib_request.Request(url + '/password', data)
+ password_request = compat_urllib_request_Request(url + '/password', data)
password_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
password_request.add_header('Referer', url)
self._set_vimeo_cookie('vuid', vuid)
@@ -236,7 +236,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
raise ExtractorError('This video is protected by a password, use the --video-password option')
data = urlencode_postdata(encode_dict({'password': password}))
pass_url = url + '/check-password'
- password_request = compat_urllib_request.Request(pass_url, data)
+ password_request = compat_urllib_request_Request(pass_url, data)
password_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
return self._download_json(
password_request, video_id,
@@ -265,7 +265,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
url = 'https://vimeo.com/' + video_id
# Retrieve video webpage to extract further information
- request = compat_urllib_request.Request(url, None, headers)
+ request = compat_urllib_request_Request(url, None, headers)
try:
webpage = self._download_webpage(request, video_id)
except ExtractorError as ee:
@@ -481,7 +481,7 @@ class VimeoChannelIE(VimeoBaseInfoExtractor):
password_path = self._search_regex(
r'action="([^"]+)"', login_form, 'password URL')
password_url = compat_urlparse.urljoin(page_url, password_path)
- password_request = compat_urllib_request.Request(password_url, post)
+ password_request = compat_urllib_request_Request(password_url, post)
password_request.add_header('Content-type', 'application/x-www-form-urlencoded')
self._set_vimeo_cookie('vuid', vuid)
self._set_vimeo_cookie('xsrft', token)
@@ -640,7 +640,7 @@ class VimeoWatchLaterIE(VimeoChannelIE):
def _page_url(self, base_url, pagenum):
url = '%s/page:%d/' % (base_url, pagenum)
- request = compat_urllib_request.Request(url)
+ request = compat_urllib_request_Request(url)
# Set the header to get a partial html page with the ids,
# the normal page doesn't contain them.
request.add_header('X-Requested-With', 'XMLHttpRequest')
From 67dda51722f1ce12b956782d43047b3fff390115 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 20 Nov 2015 20:33:49 +0600
Subject: [PATCH 365/415] Rename compat_urllib_request_Request to
sanitized_Request and move to utils
---
youtube_dl/YoutubeDL.py | 4 ++--
youtube_dl/compat.py | 8 --------
youtube_dl/extractor/vimeo.py | 14 +++++++-------
youtube_dl/utils.py | 7 +++++++
4 files changed, 16 insertions(+), 17 deletions(-)
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index b20837ce2..eedab37a7 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -39,7 +39,6 @@ from .compat import (
compat_urllib_error,
compat_urllib_request,
compat_urllib_request_DataHandler,
- compat_urllib_request_Request,
)
from .utils import (
ContentTooShortError,
@@ -65,6 +64,7 @@ from .utils import (
SameFileError,
sanitize_filename,
sanitize_path,
+ sanitized_Request,
std_headers,
subtitles_filename,
UnavailableVideoError,
@@ -1874,7 +1874,7 @@ class YoutubeDL(object):
def urlopen(self, req):
""" Start an HTTP download """
if isinstance(req, compat_basestring):
- req = compat_urllib_request_Request(req)
+ req = sanitized_Request(req)
return self._opener.open(req, timeout=self._socket_timeout)
def print_debug_header(self):
diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 4e3de7f51..a3e85264a 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -198,14 +198,6 @@ except ImportError: # Python < 3.4
return compat_urllib_response.addinfourl(io.BytesIO(data), headers, url)
-
-# Prepend protocol-less URLs with `http:` scheme in order to mitigate the number of
-# unwanted failures due to missing protocol
-def compat_urllib_request_Request(url, *args, **kwargs):
- return compat_urllib_request.Request(
- 'http:%s' % url if url.startswith('//') else url, *args, **kwargs)
-
-
try:
compat_basestring = basestring # Python 2
except NameError:
diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index d1daf9672..f392ccf1c 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -8,7 +8,6 @@ import itertools
from .common import InfoExtractor
from ..compat import (
compat_HTTPError,
- compat_urllib_request_Request,
compat_urlparse,
)
from ..utils import (
@@ -17,6 +16,7 @@ from ..utils import (
InAdvancePagedList,
int_or_none,
RegexNotFoundError,
+ sanitized_Request,
smuggle_url,
std_headers,
unified_strdate,
@@ -47,7 +47,7 @@ class VimeoBaseInfoExtractor(InfoExtractor):
'service': 'vimeo',
'token': token,
}))
- login_request = compat_urllib_request_Request(self._LOGIN_URL, data)
+ login_request = sanitized_Request(self._LOGIN_URL, data)
login_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
login_request.add_header('Referer', self._LOGIN_URL)
self._set_vimeo_cookie('vuid', vuid)
@@ -222,7 +222,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
if url.startswith('http://'):
# vimeo only supports https now, but the user can give an http url
url = url.replace('http://', 'https://')
- password_request = compat_urllib_request_Request(url + '/password', data)
+ password_request = sanitized_Request(url + '/password', data)
password_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
password_request.add_header('Referer', url)
self._set_vimeo_cookie('vuid', vuid)
@@ -236,7 +236,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
raise ExtractorError('This video is protected by a password, use the --video-password option')
data = urlencode_postdata(encode_dict({'password': password}))
pass_url = url + '/check-password'
- password_request = compat_urllib_request_Request(pass_url, data)
+ password_request = sanitized_Request(pass_url, data)
password_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
return self._download_json(
password_request, video_id,
@@ -265,7 +265,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
url = 'https://vimeo.com/' + video_id
# Retrieve video webpage to extract further information
- request = compat_urllib_request_Request(url, None, headers)
+ request = sanitized_Request(url, None, headers)
try:
webpage = self._download_webpage(request, video_id)
except ExtractorError as ee:
@@ -481,7 +481,7 @@ class VimeoChannelIE(VimeoBaseInfoExtractor):
password_path = self._search_regex(
r'action="([^"]+)"', login_form, 'password URL')
password_url = compat_urlparse.urljoin(page_url, password_path)
- password_request = compat_urllib_request_Request(password_url, post)
+ password_request = sanitized_Request(password_url, post)
password_request.add_header('Content-type', 'application/x-www-form-urlencoded')
self._set_vimeo_cookie('vuid', vuid)
self._set_vimeo_cookie('xsrft', token)
@@ -640,7 +640,7 @@ class VimeoWatchLaterIE(VimeoChannelIE):
def _page_url(self, base_url, pagenum):
url = '%s/page:%d/' % (base_url, pagenum)
- request = compat_urllib_request_Request(url)
+ request = sanitized_Request(url)
# Set the header to get a partial html page with the ids,
# the normal page doesn't contain them.
request.add_header('X-Requested-With', 'XMLHttpRequest')
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index c0325f054..d7b737e21 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -373,6 +373,13 @@ def sanitize_path(s):
return os.path.join(*sanitized_path)
+# Prepend protocol-less URLs with `http:` scheme in order to mitigate the number of
+# unwanted failures due to missing protocol
+def sanitized_Request(url, *args, **kwargs):
+ return compat_urllib_request.Request(
+ 'http:%s' % url if url.startswith('//') else url, *args, **kwargs)
+
+
def orderedSet(iterable):
""" Remove all duplicates from the input iterable """
res = []
From 5c2266df4b9aeb7881ed8c026a038e2a25e43734 Mon Sep 17 00:00:00 2001
From: Sergey M? <dstftw@gmail.com>
Date: Sat, 21 Nov 2015 22:18:17 +0600
Subject: [PATCH 366/415] Switch codebase to use sanitized_Request instead of
compat_urllib_request.Request
[downloader/dash] Use sanitized_Request
[downloader/http] Use sanitized_Request
[atresplayer] Use sanitized_Request
[bambuser] Use sanitized_Request
[bliptv] Use sanitized_Request
[brightcove] Use sanitized_Request
[cbs] Use sanitized_Request
[ceskatelevize] Use sanitized_Request
[collegerama] Use sanitized_Request
[extractor/common] Use sanitized_Request
[crunchyroll] Use sanitized_Request
[dailymotion] Use sanitized_Request
[dcn] Use sanitized_Request
[dramafever] Use sanitized_Request
[dumpert] Use sanitized_Request
[eitb] Use sanitized_Request
[escapist] Use sanitized_Request
[everyonesmixtape] Use sanitized_Request
[extremetube] Use sanitized_Request
[facebook] Use sanitized_Request
[fc2] Use sanitized_Request
[flickr] Use sanitized_Request
[4tube] Use sanitized_Request
[gdcvault] Use sanitized_Request
[extractor/generic] Use sanitized_Request
[hearthisat] Use sanitized_Request
[hotnewhiphop] Use sanitized_Request
[hypem] Use sanitized_Request
[iprima] Use sanitized_Request
[ivi] Use sanitized_Request
[keezmovies] Use sanitized_Request
[letv] Use sanitized_Request
[lynda] Use sanitized_Request
[metacafe] Use sanitized_Request
[minhateca] Use sanitized_Request
[miomio] Use sanitized_Request
[meovideo] Use sanitized_Request
[mofosex] Use sanitized_Request
[moniker] Use sanitized_Request
[mooshare] Use sanitized_Request
[movieclips] Use sanitized_Request
[mtv] Use sanitized_Request
[myvideo] Use sanitized_Request
[neteasemusic] Use sanitized_Request
[nfb] Use sanitized_Request
[niconico] Use sanitized_Request
[noco] Use sanitized_Request
[nosvideo] Use sanitized_Request
[novamov] Use sanitized_Request
[nowness] Use sanitized_Request
[nuvid] Use sanitized_Request
[played] Use sanitized_Request
[pluralsight] Use sanitized_Request
[pornhub] Use sanitized_Request
[pornotube] Use sanitized_Request
[primesharetv] Use sanitized_Request
[promptfile] Use sanitized_Request
[qqmusic] Use sanitized_Request
[rtve] Use sanitized_Request
[safari] Use sanitized_Request
[sandia] Use sanitized_Request
[shared] Use sanitized_Request
[sharesix] Use sanitized_Request
[sina] Use sanitized_Request
[smotri] Use sanitized_Request
[sohu] Use sanitized_Request
[spankwire] Use sanitized_Request
[sportdeutschland] Use sanitized_Request
[streamcloud] Use sanitized_Request
[streamcz] Use sanitized_Request
[tapely] Use sanitized_Request
[tube8] Use sanitized_Request
[tubitv] Use sanitized_Request
[twitch] Use sanitized_Request
[twitter] Use sanitized_Request
[udemy] Use sanitized_Request
[vbox7] Use sanitized_Request
[veoh] Use sanitized_Request
[vessel] Use sanitized_Request
[vevo] Use sanitized_Request
[viddler] Use sanitized_Request
[videomega] Use sanitized_Request
[viewvster] Use sanitized_Request
[viki] Use sanitized_Request
[vk] Use sanitized_Request
[vodlocker] Use sanitized_Request
[voicerepublic] Use sanitized_Request
[wistia] Use sanitized_Request
[xfileshare] Use sanitized_Request
[xtube] Use sanitized_Request
[xvideos] Use sanitized_Request
[yandexmusic] Use sanitized_Request
[youku] Use sanitized_Request
[youporn] Use sanitized_Request
[youtube] Use sanitized_Request
[patreon] Use sanitized_Request
[extractor/common] Remove unused import
[nfb] PEP 8
---
youtube_dl/YoutubeDL.py | 2 +-
youtube_dl/downloader/dash.py | 4 ++--
youtube_dl/downloader/http.py | 10 ++++------
youtube_dl/extractor/atresplayer.py | 6 +++---
youtube_dl/extractor/bambuser.py | 6 +++---
youtube_dl/extractor/bliptv.py | 8 +++-----
youtube_dl/extractor/brightcove.py | 6 +++---
youtube_dl/extractor/cbs.py | 8 +++++---
youtube_dl/extractor/ceskatelevize.py | 6 +++---
youtube_dl/extractor/collegerama.py | 4 ++--
youtube_dl/extractor/common.py | 4 ++--
youtube_dl/extractor/crunchyroll.py | 9 +++++----
youtube_dl/extractor/dailymotion.py | 8 +++-----
youtube_dl/extractor/dcn.py | 8 +++-----
youtube_dl/extractor/dramafever.py | 4 ++--
youtube_dl/extractor/dumpert.py | 8 +++++---
youtube_dl/extractor/eitb.py | 4 ++--
youtube_dl/extractor/escapist.py | 5 ++---
youtube_dl/extractor/everyonesmixtape.py | 8 +++-----
youtube_dl/extractor/extremetube.py | 4 ++--
youtube_dl/extractor/facebook.py | 8 ++++----
youtube_dl/extractor/fc2.py | 5 +++--
youtube_dl/extractor/flickr.py | 4 ++--
youtube_dl/extractor/fourtube.py | 6 ++----
youtube_dl/extractor/gdcvault.py | 8 +++-----
youtube_dl/extractor/generic.py | 6 +++---
youtube_dl/extractor/hearthisat.py | 8 +++-----
youtube_dl/extractor/hotnewhiphop.py | 8 +++-----
youtube_dl/extractor/hypem.py | 10 ++++------
youtube_dl/extractor/iprima.py | 6 ++----
youtube_dl/extractor/ivi.py | 6 ++----
youtube_dl/extractor/keezmovies.py | 8 +++-----
youtube_dl/extractor/letv.py | 4 ++--
youtube_dl/extractor/lynda.py | 6 +++---
youtube_dl/extractor/metacafe.py | 6 +++---
youtube_dl/extractor/minhateca.py | 8 +++-----
youtube_dl/extractor/miomio.py | 4 ++--
youtube_dl/extractor/moevideo.py | 8 +++-----
youtube_dl/extractor/mofosex.py | 4 ++--
youtube_dl/extractor/moniker.py | 10 ++++------
youtube_dl/extractor/mooshare.py | 8 +++-----
youtube_dl/extractor/movieclips.py | 6 ++----
youtube_dl/extractor/mtv.py | 4 ++--
youtube_dl/extractor/myvideo.py | 4 ++--
youtube_dl/extractor/neteasemusic.py | 4 ++--
youtube_dl/extractor/nfb.py | 11 +++++------
youtube_dl/extractor/niconico.py | 6 +++---
youtube_dl/extractor/noco.py | 4 ++--
youtube_dl/extractor/nosvideo.py | 6 ++----
youtube_dl/extractor/novamov.py | 8 +++-----
youtube_dl/extractor/nowness.py | 10 +++++-----
youtube_dl/extractor/nuvid.py | 6 ++----
youtube_dl/extractor/patreon.py | 6 ++----
youtube_dl/extractor/played.py | 8 +++-----
youtube_dl/extractor/pluralsight.py | 6 +++---
youtube_dl/extractor/pornhub.py | 4 ++--
youtube_dl/extractor/pornotube.py | 10 ++++------
youtube_dl/extractor/primesharetv.py | 10 +++++-----
youtube_dl/extractor/promptfile.py | 8 +++-----
youtube_dl/extractor/qqmusic.py | 4 ++--
youtube_dl/extractor/rtve.py | 4 ++--
youtube_dl/extractor/safari.py | 8 +++-----
youtube_dl/extractor/sandia.py | 8 +++-----
youtube_dl/extractor/shared.py | 8 +++-----
youtube_dl/extractor/sharesix.py | 8 +++-----
youtube_dl/extractor/sina.py | 8 +++-----
youtube_dl/extractor/smotri.py | 10 ++++------
youtube_dl/extractor/sohu.py | 4 ++--
youtube_dl/extractor/spankwire.py | 4 ++--
youtube_dl/extractor/sportdeutschland.py | 6 ++----
youtube_dl/extractor/streamcloud.py | 8 +++-----
youtube_dl/extractor/streamcz.py | 6 ++----
youtube_dl/extractor/tapely.py | 6 ++----
youtube_dl/extractor/tube8.py | 8 +++-----
youtube_dl/extractor/tubitv.py | 8 +++-----
youtube_dl/extractor/twitch.py | 6 +++---
youtube_dl/extractor/twitter.py | 4 ++--
youtube_dl/extractor/udemy.py | 5 +++--
youtube_dl/extractor/vbox7.py | 4 ++--
youtube_dl/extractor/veoh.py | 6 ++----
youtube_dl/extractor/vessel.py | 4 ++--
youtube_dl/extractor/vevo.py | 8 +++-----
youtube_dl/extractor/viddler.py | 6 ++----
youtube_dl/extractor/videomega.py | 4 ++--
youtube_dl/extractor/viewster.py | 4 ++--
youtube_dl/extractor/viki.py | 6 +++---
youtube_dl/extractor/vk.py | 4 ++--
youtube_dl/extractor/vodlocker.py | 8 +++-----
youtube_dl/extractor/voicerepublic.py | 8 +++-----
youtube_dl/extractor/wistia.py | 8 +++++---
youtube_dl/extractor/xfileshare.py | 8 +++-----
youtube_dl/extractor/xtube.py | 8 +++-----
youtube_dl/extractor/xvideos.py | 8 +++-----
youtube_dl/extractor/yandexmusic.py | 4 ++--
youtube_dl/extractor/youku.py | 9 +++++----
youtube_dl/extractor/youporn.py | 4 ++--
youtube_dl/extractor/youtube.py | 6 +++---
97 files changed, 271 insertions(+), 353 deletions(-)
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index eedab37a7..9a8c7da05 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1189,7 +1189,7 @@ class YoutubeDL(object):
return res
def _calc_cookies(self, info_dict):
- pr = compat_urllib_request.Request(info_dict['url'])
+ pr = sanitized_Request(info_dict['url'])
self.cookiejar.add_cookie_header(pr)
return pr.get_header('Cookie')
diff --git a/youtube_dl/downloader/dash.py b/youtube_dl/downloader/dash.py
index 8b6fa2753..535f2a7fc 100644
--- a/youtube_dl/downloader/dash.py
+++ b/youtube_dl/downloader/dash.py
@@ -3,7 +3,7 @@ from __future__ import unicode_literals
import re
from .common import FileDownloader
-from ..compat import compat_urllib_request
+from ..utils import sanitized_Request
class DashSegmentsFD(FileDownloader):
@@ -22,7 +22,7 @@ class DashSegmentsFD(FileDownloader):
def append_url_to_file(outf, target_url, target_name, remaining_bytes=None):
self.to_screen('[DashSegments] %s: Downloading %s' % (info_dict['id'], target_name))
- req = compat_urllib_request.Request(target_url)
+ req = sanitized_Request(target_url)
if remaining_bytes is not None:
req.add_header('Range', 'bytes=0-%d' % (remaining_bytes - 1))
diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py
index a29f5cf31..56840e026 100644
--- a/youtube_dl/downloader/http.py
+++ b/youtube_dl/downloader/http.py
@@ -7,14 +7,12 @@ import time
import re
from .common import FileDownloader
-from ..compat import (
- compat_urllib_request,
- compat_urllib_error,
-)
+from ..compat import compat_urllib_error
from ..utils import (
ContentTooShortError,
encodeFilename,
sanitize_open,
+ sanitized_Request,
)
@@ -29,8 +27,8 @@ class HttpFD(FileDownloader):
add_headers = info_dict.get('http_headers')
if add_headers:
headers.update(add_headers)
- basic_request = compat_urllib_request.Request(url, None, headers)
- request = compat_urllib_request.Request(url, None, headers)
+ basic_request = sanitized_Request(url, None, headers)
+ request = sanitized_Request(url, None, headers)
is_test = self.params.get('test', False)
diff --git a/youtube_dl/extractor/atresplayer.py b/youtube_dl/extractor/atresplayer.py
index 29f8795d3..50e47ba0a 100644
--- a/youtube_dl/extractor/atresplayer.py
+++ b/youtube_dl/extractor/atresplayer.py
@@ -7,11 +7,11 @@ from .common import InfoExtractor
from ..compat import (
compat_str,
compat_urllib_parse,
- compat_urllib_request,
)
from ..utils import (
int_or_none,
float_or_none,
+ sanitized_Request,
xpath_text,
ExtractorError,
)
@@ -63,7 +63,7 @@ class AtresPlayerIE(InfoExtractor):
'j_password': password,
}
- request = compat_urllib_request.Request(
+ request = sanitized_Request(
self._LOGIN_URL, compat_urllib_parse.urlencode(login_form).encode('utf-8'))
request.add_header('Content-Type', 'application/x-www-form-urlencoded')
response = self._download_webpage(
@@ -94,7 +94,7 @@ class AtresPlayerIE(InfoExtractor):
formats = []
for fmt in ['windows', 'android_tablet']:
- request = compat_urllib_request.Request(
+ request = sanitized_Request(
self._URL_VIDEO_TEMPLATE.format(fmt, episode_id, timestamp_shifted, token))
request.add_header('User-Agent', self._USER_AGENT)
diff --git a/youtube_dl/extractor/bambuser.py b/youtube_dl/extractor/bambuser.py
index 8dff1d6e3..da986e063 100644
--- a/youtube_dl/extractor/bambuser.py
+++ b/youtube_dl/extractor/bambuser.py
@@ -6,13 +6,13 @@ import itertools
from .common import InfoExtractor
from ..compat import (
compat_urllib_parse,
- compat_urllib_request,
compat_str,
)
from ..utils import (
ExtractorError,
int_or_none,
float_or_none,
+ sanitized_Request,
)
@@ -57,7 +57,7 @@ class BambuserIE(InfoExtractor):
'pass': password,
}
- request = compat_urllib_request.Request(
+ request = sanitized_Request(
self._LOGIN_URL, compat_urllib_parse.urlencode(login_form).encode('utf-8'))
request.add_header('Referer', self._LOGIN_URL)
response = self._download_webpage(
@@ -126,7 +126,7 @@ class BambuserChannelIE(InfoExtractor):
'&sort=created&access_mode=0%2C1%2C2&limit={count}'
'&method=broadcast&format=json&vid_older_than={last}'
).format(user=user, count=self._STEP, last=last_id)
- req = compat_urllib_request.Request(req_url)
+ req = sanitized_Request(req_url)
# Without setting this header, we wouldn't get any result
req.add_header('Referer', 'http://bambuser.com/channel/%s' % user)
data = self._download_json(
diff --git a/youtube_dl/extractor/bliptv.py b/youtube_dl/extractor/bliptv.py
index c3296283d..35375f7b1 100644
--- a/youtube_dl/extractor/bliptv.py
+++ b/youtube_dl/extractor/bliptv.py
@@ -4,14 +4,12 @@ import re
from .common import InfoExtractor
-from ..compat import (
- compat_urllib_request,
- compat_urlparse,
-)
+from ..compat import compat_urlparse
from ..utils import (
clean_html,
int_or_none,
parse_iso8601,
+ sanitized_Request,
unescapeHTML,
xpath_text,
xpath_with_ns,
@@ -219,7 +217,7 @@ class BlipTVIE(InfoExtractor):
for lang, url in subtitles_urls.items():
# For some weird reason, blip.tv serves a video instead of subtitles
# when we request with a common UA
- req = compat_urllib_request.Request(url)
+ req = sanitized_Request(url)
req.add_header('User-Agent', 'youtube-dl')
subtitles[lang] = [{
# The extension is 'srt' but it's actually an 'ass' file
diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 14ee05f21..f5ebae1e6 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -11,7 +11,6 @@ from ..compat import (
compat_str,
compat_urllib_parse,
compat_urllib_parse_urlparse,
- compat_urllib_request,
compat_urlparse,
compat_xml_parse_error,
)
@@ -24,6 +23,7 @@ from ..utils import (
js_to_json,
int_or_none,
parse_iso8601,
+ sanitized_Request,
unescapeHTML,
unsmuggle_url,
)
@@ -250,7 +250,7 @@ class BrightcoveLegacyIE(InfoExtractor):
def _get_video_info(self, video_id, query_str, query, referer=None):
request_url = self._FEDERATED_URL_TEMPLATE % query_str
- req = compat_urllib_request.Request(request_url)
+ req = sanitized_Request(request_url)
linkBase = query.get('linkBaseURL')
if linkBase is not None:
referer = linkBase[0]
@@ -443,7 +443,7 @@ class BrightcoveNewIE(InfoExtractor):
r'policyKey\s*:\s*(["\'])(?P<pk>.+?)\1',
webpage, 'policy key', group='pk')
- req = compat_urllib_request.Request(
+ req = sanitized_Request(
'https://edge.api.brightcove.com/playback/v1/accounts/%s/videos/%s'
% (account_id, video_id),
headers={'Accept': 'application/json;pk=%s' % policy_key})
diff --git a/youtube_dl/extractor/cbs.py b/youtube_dl/extractor/cbs.py
index 43f05d278..40d07ab18 100644
--- a/youtube_dl/extractor/cbs.py
+++ b/youtube_dl/extractor/cbs.py
@@ -1,8 +1,10 @@
from __future__ import unicode_literals
from .common import InfoExtractor
-from ..compat import compat_urllib_request
-from ..utils import smuggle_url
+from ..utils import (
+ sanitized_Request,
+ smuggle_url,
+)
class CBSIE(InfoExtractor):
@@ -48,7 +50,7 @@ class CBSIE(InfoExtractor):
def _real_extract(self, url):
display_id = self._match_id(url)
- request = compat_urllib_request.Request(url)
+ request = sanitized_Request(url)
# Android UA is served with higher quality (720p) streams (see
# https://github.com/rg3/youtube-dl/issues/7490)
request.add_header('User-Agent', 'Mozilla/5.0 (Linux; Android 4.4; Nexus 5)')
diff --git a/youtube_dl/extractor/ceskatelevize.py b/youtube_dl/extractor/ceskatelevize.py
index e857e66f4..6f7b2a70d 100644
--- a/youtube_dl/extractor/ceskatelevize.py
+++ b/youtube_dl/extractor/ceskatelevize.py
@@ -5,7 +5,6 @@ import re
from .common import InfoExtractor
from ..compat import (
- compat_urllib_request,
compat_urllib_parse,
compat_urllib_parse_unquote,
compat_urllib_parse_urlparse,
@@ -13,6 +12,7 @@ from ..compat import (
from ..utils import (
ExtractorError,
float_or_none,
+ sanitized_Request,
)
@@ -100,7 +100,7 @@ class CeskaTelevizeIE(InfoExtractor):
'requestSource': 'iVysilani',
}
- req = compat_urllib_request.Request(
+ req = sanitized_Request(
'http://www.ceskatelevize.cz/ivysilani/ajax/get-client-playlist',
data=compat_urllib_parse.urlencode(data))
@@ -115,7 +115,7 @@ class CeskaTelevizeIE(InfoExtractor):
if playlist_url == 'error_region':
raise ExtractorError(NOT_AVAILABLE_STRING, expected=True)
- req = compat_urllib_request.Request(compat_urllib_parse_unquote(playlist_url))
+ req = sanitized_Request(compat_urllib_parse_unquote(playlist_url))
req.add_header('Referer', url)
playlist_title = self._og_search_title(webpage)
diff --git a/youtube_dl/extractor/collegerama.py b/youtube_dl/extractor/collegerama.py
index fedd48490..40667a0f1 100644
--- a/youtube_dl/extractor/collegerama.py
+++ b/youtube_dl/extractor/collegerama.py
@@ -3,10 +3,10 @@ from __future__ import unicode_literals
import json
from .common import InfoExtractor
-from ..compat import compat_urllib_request
from ..utils import (
float_or_none,
int_or_none,
+ sanitized_Request,
)
@@ -52,7 +52,7 @@ class CollegeRamaIE(InfoExtractor):
}
}
- request = compat_urllib_request.Request(
+ request = sanitized_Request(
'http://collegerama.tudelft.nl/Mediasite/PlayerService/PlayerService.svc/json/GetPlayerOptions',
json.dumps(player_options_request))
request.add_header('Content-Type', 'application/json')
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 71bdcad5a..eb9bfa3d1 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -19,7 +19,6 @@ from ..compat import (
compat_urllib_error,
compat_urllib_parse,
compat_urllib_parse_urlparse,
- compat_urllib_request,
compat_urlparse,
compat_str,
compat_etree_fromstring,
@@ -37,6 +36,7 @@ from ..utils import (
int_or_none,
RegexNotFoundError,
sanitize_filename,
+ sanitized_Request,
unescapeHTML,
unified_strdate,
url_basename,
@@ -1285,7 +1285,7 @@ class InfoExtractor(object):
def _get_cookies(self, url):
""" Return a compat_cookies.SimpleCookie with the cookies for the url """
- req = compat_urllib_request.Request(url)
+ req = sanitized_Request(url)
self._downloader.cookiejar.add_cookie_header(req)
return compat_cookies.SimpleCookie(req.get_header('Cookie'))
diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index 6e5999c72..00d943f77 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -23,6 +23,7 @@ from ..utils import (
int_or_none,
lowercase_escape,
remove_end,
+ sanitized_Request,
unified_strdate,
urlencode_postdata,
xpath_text,
@@ -46,7 +47,7 @@ class CrunchyrollBaseIE(InfoExtractor):
'name': username,
'password': password,
})
- login_request = compat_urllib_request.Request(login_url, data)
+ login_request = sanitized_Request(login_url, data)
login_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
self._download_webpage(login_request, None, False, 'Wrong login info')
@@ -55,7 +56,7 @@ class CrunchyrollBaseIE(InfoExtractor):
def _download_webpage(self, url_or_request, video_id, note=None, errnote=None, fatal=True, tries=1, timeout=5, encoding=None):
request = (url_or_request if isinstance(url_or_request, compat_urllib_request.Request)
- else compat_urllib_request.Request(url_or_request))
+ else sanitized_Request(url_or_request))
# Accept-Language must be set explicitly to accept any language to avoid issues
# similar to https://github.com/rg3/youtube-dl/issues/6797.
# Along with IP address Crunchyroll uses Accept-Language to guess whether georestriction
@@ -307,7 +308,7 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
'video_uploader', fatal=False)
playerdata_url = compat_urllib_parse_unquote(self._html_search_regex(r'"config_url":"([^"]+)', webpage, 'playerdata_url'))
- playerdata_req = compat_urllib_request.Request(playerdata_url)
+ playerdata_req = sanitized_Request(playerdata_url)
playerdata_req.data = compat_urllib_parse.urlencode({'current_page': webpage_url})
playerdata_req.add_header('Content-Type', 'application/x-www-form-urlencoded')
playerdata = self._download_webpage(playerdata_req, video_id, note='Downloading media info')
@@ -319,7 +320,7 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
for fmt in re.findall(r'showmedia\.([0-9]{3,4})p', webpage):
stream_quality, stream_format = self._FORMAT_IDS[fmt]
video_format = fmt + 'p'
- streamdata_req = compat_urllib_request.Request(
+ streamdata_req = sanitized_Request(
'http://www.crunchyroll.com/xml/?req=RpcApiVideoPlayer_GetStandardConfig&media_id=%s&video_format=%s&video_quality=%s'
% (stream_id, stream_format, stream_quality),
compat_urllib_parse.urlencode({'current_page': url}).encode('utf-8'))
diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index bc7823931..ab7f3aec4 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -7,15 +7,13 @@ import itertools
from .common import InfoExtractor
-from ..compat import (
- compat_str,
- compat_urllib_request,
-)
+from ..compat import compat_str
from ..utils import (
ExtractorError,
determine_ext,
int_or_none,
parse_iso8601,
+ sanitized_Request,
str_to_int,
unescapeHTML,
)
@@ -25,7 +23,7 @@ class DailymotionBaseInfoExtractor(InfoExtractor):
@staticmethod
def _build_request(url):
"""Build a request with the family filter disabled"""
- request = compat_urllib_request.Request(url)
+ request = sanitized_Request(url)
request.add_header('Cookie', 'family_filter=off; ff=off')
return request
diff --git a/youtube_dl/extractor/dcn.py b/youtube_dl/extractor/dcn.py
index 6f2fea5ff..9737cff14 100644
--- a/youtube_dl/extractor/dcn.py
+++ b/youtube_dl/extractor/dcn.py
@@ -2,13 +2,11 @@
from __future__ import unicode_literals
from .common import InfoExtractor
-from ..compat import (
- compat_urllib_parse,
- compat_urllib_request,
-)
+from ..compat import compat_urllib_parse
from ..utils import (
int_or_none,
parse_iso8601,
+ sanitized_Request,
)
@@ -36,7 +34,7 @@ class DCNIE(InfoExtractor):
def _real_extract(self, url):
video_id = self._match_id(url)
- request = compat_urllib_request.Request(
+ request = sanitized_Request(
'http://admin.mangomolo.com/analytics/index.php/plus/video?id=%s' % video_id,
headers={'Origin': 'http://www.dcndigital.ae'})
diff --git a/youtube_dl/extractor/dramafever.py b/youtube_dl/extractor/dramafever.py
index 38e6597c8..d836c1a6c 100644
--- a/youtube_dl/extractor/dramafever.py
+++ b/youtube_dl/extractor/dramafever.py
@@ -7,7 +7,6 @@ from .common import InfoExtractor
from ..compat import (
compat_HTTPError,
compat_urllib_parse,
- compat_urllib_request,
compat_urlparse,
)
from ..utils import (
@@ -16,6 +15,7 @@ from ..utils import (
determine_ext,
int_or_none,
parse_iso8601,
+ sanitized_Request,
)
@@ -51,7 +51,7 @@ class DramaFeverBaseIE(InfoExtractor):
'password': password,
}
- request = compat_urllib_request.Request(
+ request = sanitized_Request(
self._LOGIN_URL, compat_urllib_parse.urlencode(login_form).encode('utf-8'))
response = self._download_webpage(
request, None, 'Logging in as %s' % username)
diff --git a/youtube_dl/extractor/dumpert.py b/youtube_dl/extractor/dumpert.py
index f5a31058d..e5aadcd25 100644
--- a/youtube_dl/extractor/dumpert.py
+++ b/youtube_dl/extractor/dumpert.py
@@ -5,8 +5,10 @@ import base64
import re
from .common import InfoExtractor
-from ..compat import compat_urllib_request
-from ..utils import qualities
+from ..utils import (
+ qualities,
+ sanitized_Request,
+)
class DumpertIE(InfoExtractor):
@@ -32,7 +34,7 @@ class DumpertIE(InfoExtractor):
protocol = mobj.group('protocol')
url = '%s://www.dumpert.nl/mediabase/%s' % (protocol, video_id)
- req = compat_urllib_request.Request(url)
+ req = sanitized_Request(url)
req.add_header('Cookie', 'nsfw=1; cpc=10')
webpage = self._download_webpage(req, video_id)
diff --git a/youtube_dl/extractor/eitb.py b/youtube_dl/extractor/eitb.py
index 357a2196c..c83845fc2 100644
--- a/youtube_dl/extractor/eitb.py
+++ b/youtube_dl/extractor/eitb.py
@@ -2,11 +2,11 @@
from __future__ import unicode_literals
from .common import InfoExtractor
-from ..compat import compat_urllib_request
from ..utils import (
float_or_none,
int_or_none,
parse_iso8601,
+ sanitized_Request,
)
@@ -57,7 +57,7 @@ class EitbIE(InfoExtractor):
hls_url = media.get('HLS_SURL')
if hls_url:
- request = compat_urllib_request.Request(
+ request = sanitized_Request(
'http://mam.eitb.eus/mam/REST/ServiceMultiweb/DomainRestrictedSecurity/TokenAuth/',
headers={'Referer': url})
token_data = self._download_json(
diff --git a/youtube_dl/extractor/escapist.py b/youtube_dl/extractor/escapist.py
index c85b4c458..a3d7bbbcb 100644
--- a/youtube_dl/extractor/escapist.py
+++ b/youtube_dl/extractor/escapist.py
@@ -3,13 +3,12 @@ from __future__ import unicode_literals
import json
from .common import InfoExtractor
-from ..compat import compat_urllib_request
-
from ..utils import (
determine_ext,
clean_html,
int_or_none,
float_or_none,
+ sanitized_Request,
)
@@ -75,7 +74,7 @@ class EscapistIE(InfoExtractor):
video_id = ims_video['videoID']
key = ims_video['hash']
- config_req = compat_urllib_request.Request(
+ config_req = sanitized_Request(
'http://www.escapistmagazine.com/videos/'
'vidconfig.php?videoID=%s&hash=%s' % (video_id, key))
config_req.add_header('Referer', url)
diff --git a/youtube_dl/extractor/everyonesmixtape.py b/youtube_dl/extractor/everyonesmixtape.py
index d872d828f..493d38af8 100644
--- a/youtube_dl/extractor/everyonesmixtape.py
+++ b/youtube_dl/extractor/everyonesmixtape.py
@@ -3,11 +3,9 @@ from __future__ import unicode_literals
import re
from .common import InfoExtractor
-from ..compat import (
- compat_urllib_request,
-)
from ..utils import (
ExtractorError,
+ sanitized_Request,
)
@@ -42,7 +40,7 @@ class EveryonesMixtapeIE(InfoExtractor):
playlist_id = mobj.group('id')
pllist_url = 'http://everyonesmixtape.com/mixtape.php?a=getMixes&u=-1&linked=%s&explore=' % playlist_id
- pllist_req = compat_urllib_request.Request(pllist_url)
+ pllist_req = sanitized_Request(pllist_url)
pllist_req.add_header('X-Requested-With', 'XMLHttpRequest')
playlist_list = self._download_json(
@@ -55,7 +53,7 @@ class EveryonesMixtapeIE(InfoExtractor):
raise ExtractorError('Playlist id not found')
pl_url = 'http://everyonesmixtape.com/mixtape.php?a=getMix&id=%s&userId=null&code=' % playlist_no
- pl_req = compat_urllib_request.Request(pl_url)
+ pl_req = sanitized_Request(pl_url)
pl_req.add_header('X-Requested-With', 'XMLHttpRequest')
playlist = self._download_json(
pl_req, playlist_id, note='Downloading playlist info')
diff --git a/youtube_dl/extractor/extremetube.py b/youtube_dl/extractor/extremetube.py
index c5677c82b..3403581fd 100644
--- a/youtube_dl/extractor/extremetube.py
+++ b/youtube_dl/extractor/extremetube.py
@@ -3,9 +3,9 @@ from __future__ import unicode_literals
import re
from .common import InfoExtractor
-from ..compat import compat_urllib_request
from ..utils import (
int_or_none,
+ sanitized_Request,
str_to_int,
)
@@ -37,7 +37,7 @@ class ExtremeTubeIE(InfoExtractor):
def _real_extract(self, url):
video_id = self._match_id(url)
- req = compat_urllib_request.Request(url)
+ req = sanitized_Request(url)
req.add_header('Cookie', 'age_verified=1')
webpage = self._download_webpage(req, video_id)
diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index f53c51615..fd854411b 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -10,11 +10,11 @@ from ..compat import (
compat_str,
compat_urllib_error,
compat_urllib_parse_unquote,
- compat_urllib_request,
)
from ..utils import (
ExtractorError,
limit_length,
+ sanitized_Request,
urlencode_postdata,
get_element_by_id,
clean_html,
@@ -73,7 +73,7 @@ class FacebookIE(InfoExtractor):
if useremail is None:
return
- login_page_req = compat_urllib_request.Request(self._LOGIN_URL)
+ login_page_req = sanitized_Request(self._LOGIN_URL)
login_page_req.add_header('Cookie', 'locale=en_US')
login_page = self._download_webpage(login_page_req, None,
note='Downloading login page',
@@ -94,7 +94,7 @@ class FacebookIE(InfoExtractor):
'timezone': '-60',
'trynum': '1',
}
- request = compat_urllib_request.Request(self._LOGIN_URL, urlencode_postdata(login_form))
+ request = sanitized_Request(self._LOGIN_URL, urlencode_postdata(login_form))
request.add_header('Content-Type', 'application/x-www-form-urlencoded')
try:
login_results = self._download_webpage(request, None,
@@ -109,7 +109,7 @@ class FacebookIE(InfoExtractor):
r'name="h"\s+(?:\w+="[^"]+"\s+)*?value="([^"]+)"', login_results, 'h'),
'name_action_selected': 'dont_save',
}
- check_req = compat_urllib_request.Request(self._CHECKPOINT_URL, urlencode_postdata(check_form))
+ check_req = sanitized_Request(self._CHECKPOINT_URL, urlencode_postdata(check_form))
check_req.add_header('Content-Type', 'application/x-www-form-urlencoded')
check_response = self._download_webpage(check_req, None,
note='Confirming login')
diff --git a/youtube_dl/extractor/fc2.py b/youtube_dl/extractor/fc2.py
index a406945e8..92e8c571f 100644
--- a/youtube_dl/extractor/fc2.py
+++ b/youtube_dl/extractor/fc2.py
@@ -12,6 +12,7 @@ from ..compat import (
from ..utils import (
encode_dict,
ExtractorError,
+ sanitized_Request,
)
@@ -57,7 +58,7 @@ class FC2IE(InfoExtractor):
}
login_data = compat_urllib_parse.urlencode(encode_dict(login_form_strs)).encode('utf-8')
- request = compat_urllib_request.Request(
+ request = sanitized_Request(
'https://secure.id.fc2.com/index.php?mode=login&switch_language=en', login_data)
login_results = self._download_webpage(request, None, note='Logging in', errnote='Unable to log in')
@@ -66,7 +67,7 @@ class FC2IE(InfoExtractor):
return False
# this is also needed
- login_redir = compat_urllib_request.Request('http://id.fc2.com/?mode=redirect&login=done')
+ login_redir = sanitized_Request('http://id.fc2.com/?mode=redirect&login=done')
self._download_webpage(
login_redir, None, note='Login redirect', errnote='Login redirect failed')
diff --git a/youtube_dl/extractor/flickr.py b/youtube_dl/extractor/flickr.py
index 2fe76d661..91cd46e76 100644
--- a/youtube_dl/extractor/flickr.py
+++ b/youtube_dl/extractor/flickr.py
@@ -3,10 +3,10 @@ from __future__ import unicode_literals
import re
from .common import InfoExtractor
-from ..compat import compat_urllib_request
from ..utils import (
ExtractorError,
find_xpath_attr,
+ sanitized_Request,
)
@@ -30,7 +30,7 @@ class FlickrIE(InfoExtractor):
video_id = mobj.group('id')
video_uploader_id = mobj.group('uploader_id')
webpage_url = 'http://www.flickr.com/photos/' + video_uploader_id + '/' + video_id
- req = compat_urllib_request.Request(webpage_url)
+ req = sanitized_Request(webpage_url)
req.add_header(
'User-Agent',
# it needs a more recent version
diff --git a/youtube_dl/extractor/fourtube.py b/youtube_dl/extractor/fourtube.py
index fb6d108c0..fc4a5a0fb 100644
--- a/youtube_dl/extractor/fourtube.py
+++ b/youtube_dl/extractor/fourtube.py
@@ -3,12 +3,10 @@ from __future__ import unicode_literals
import re
from .common import InfoExtractor
-from ..compat import (
- compat_urllib_request,
-)
from ..utils import (
parse_duration,
parse_iso8601,
+ sanitized_Request,
str_to_int,
)
@@ -93,7 +91,7 @@ class FourTubeIE(InfoExtractor):
b'Content-Type': b'application/x-www-form-urlencoded',
b'Origin': b'http://www.4tube.com',
}
- token_req = compat_urllib_request.Request(token_url, b'{}', headers)
+ token_req = sanitized_Request(token_url, b'{}', headers)
tokens = self._download_json(token_req, video_id)
formats = [{
'url': tokens[format]['token'],
diff --git a/youtube_dl/extractor/gdcvault.py b/youtube_dl/extractor/gdcvault.py
index a6834db43..3befd3e7b 100644
--- a/youtube_dl/extractor/gdcvault.py
+++ b/youtube_dl/extractor/gdcvault.py
@@ -3,13 +3,11 @@ from __future__ import unicode_literals
import re
from .common import InfoExtractor
-from ..compat import (
- compat_urllib_parse,
- compat_urllib_request,
-)
+from ..compat import compat_urllib_parse
from ..utils import (
remove_end,
HEADRequest,
+ sanitized_Request,
)
@@ -125,7 +123,7 @@ class GDCVaultIE(InfoExtractor):
'password': password,
}
- request = compat_urllib_request.Request(login_url, compat_urllib_parse.urlencode(login_form))
+ request = sanitized_Request(login_url, compat_urllib_parse.urlencode(login_form))
request.add_header('Content-Type', 'application/x-www-form-urlencoded')
self._download_webpage(request, display_id, 'Logging in')
start_page = self._download_webpage(webpage_url, display_id, 'Getting authenticated video page')
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 2b934148d..5075d131e 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -11,7 +11,6 @@ from .youtube import YoutubeIE
from ..compat import (
compat_etree_fromstring,
compat_urllib_parse_unquote,
- compat_urllib_request,
compat_urlparse,
compat_xml_parse_error,
)
@@ -22,6 +21,7 @@ from ..utils import (
HEADRequest,
is_html,
orderedSet,
+ sanitized_Request,
smuggle_url,
unescapeHTML,
unified_strdate,
@@ -1215,7 +1215,7 @@ class GenericIE(InfoExtractor):
full_response = None
if head_response is False:
- request = compat_urllib_request.Request(url)
+ request = sanitized_Request(url)
request.add_header('Accept-Encoding', '*')
full_response = self._request_webpage(request, video_id)
head_response = full_response
@@ -1244,7 +1244,7 @@ class GenericIE(InfoExtractor):
'%s on generic information extractor.' % ('Forcing' if force else 'Falling back'))
if not full_response:
- request = compat_urllib_request.Request(url)
+ request = sanitized_Request(url)
# Some webservers may serve compressed content of rather big size (e.g. gzipped flac)
# making it impossible to download only chunk of the file (yet we need only 512kB to
# test whether it's HTML or not). According to youtube-dl default Accept-Encoding
diff --git a/youtube_dl/extractor/hearthisat.py b/youtube_dl/extractor/hearthisat.py
index a19b31ac0..7d8698655 100644
--- a/youtube_dl/extractor/hearthisat.py
+++ b/youtube_dl/extractor/hearthisat.py
@@ -4,12 +4,10 @@ from __future__ import unicode_literals
import re
from .common import InfoExtractor
-from ..compat import (
- compat_urllib_request,
- compat_urlparse,
-)
+from ..compat import compat_urlparse
from ..utils import (
HEADRequest,
+ sanitized_Request,
str_to_int,
urlencode_postdata,
urlhandle_detect_ext,
@@ -47,7 +45,7 @@ class HearThisAtIE(InfoExtractor):
r'intTrackId\s*=\s*(\d+)', webpage, 'track ID')
payload = urlencode_postdata({'tracks[]': track_id})
- req = compat_urllib_request.Request(self._PLAYLIST_URL, payload)
+ req = sanitized_Request(self._PLAYLIST_URL, payload)
req.add_header('Content-type', 'application/x-www-form-urlencoded')
track = self._download_json(req, track_id, 'Downloading playlist')[0]
diff --git a/youtube_dl/extractor/hotnewhiphop.py b/youtube_dl/extractor/hotnewhiphop.py
index 651784b73..31e219945 100644
--- a/youtube_dl/extractor/hotnewhiphop.py
+++ b/youtube_dl/extractor/hotnewhiphop.py
@@ -3,13 +3,11 @@ from __future__ import unicode_literals
import base64
from .common import InfoExtractor
-from ..compat import (
- compat_urllib_parse,
- compat_urllib_request,
-)
+from ..compat import compat_urllib_parse
from ..utils import (
ExtractorError,
HEADRequest,
+ sanitized_Request,
)
@@ -41,7 +39,7 @@ class HotNewHipHopIE(InfoExtractor):
('mediaType', 's'),
('mediaId', video_id),
])
- r = compat_urllib_request.Request(
+ r = sanitized_Request(
'http://www.hotnewhiphop.com/ajax/media/getActions/', data=reqdata)
r.add_header('Content-Type', 'application/x-www-form-urlencoded')
mkd = self._download_json(
diff --git a/youtube_dl/extractor/hypem.py b/youtube_dl/extractor/hypem.py
index aa0724a02..cca3dd498 100644
--- a/youtube_dl/extractor/hypem.py
+++ b/youtube_dl/extractor/hypem.py
@@ -4,12 +4,10 @@ import json
import time
from .common import InfoExtractor
-from ..compat import (
- compat_urllib_parse,
- compat_urllib_request,
-)
+from ..compat import compat_urllib_parse
from ..utils import (
ExtractorError,
+ sanitized_Request,
)
@@ -32,7 +30,7 @@ class HypemIE(InfoExtractor):
data = {'ax': 1, 'ts': time.time()}
data_encoded = compat_urllib_parse.urlencode(data)
complete_url = url + "?" + data_encoded
- request = compat_urllib_request.Request(complete_url)
+ request = sanitized_Request(complete_url)
response, urlh = self._download_webpage_handle(
request, track_id, 'Downloading webpage with the url')
cookie = urlh.headers.get('Set-Cookie', '')
@@ -52,7 +50,7 @@ class HypemIE(InfoExtractor):
title = track['song']
serve_url = "http://hypem.com/serve/source/%s/%s" % (track_id, key)
- request = compat_urllib_request.Request(
+ request = sanitized_Request(
serve_url, '', {'Content-Type': 'application/json'})
request.add_header('cookie', cookie)
song_data = self._download_json(request, track_id, 'Downloading metadata')
diff --git a/youtube_dl/extractor/iprima.py b/youtube_dl/extractor/iprima.py
index 821c8ec10..36baf3245 100644
--- a/youtube_dl/extractor/iprima.py
+++ b/youtube_dl/extractor/iprima.py
@@ -6,12 +6,10 @@ from random import random
from math import floor
from .common import InfoExtractor
-from ..compat import (
- compat_urllib_request,
-)
from ..utils import (
ExtractorError,
remove_end,
+ sanitized_Request,
)
@@ -61,7 +59,7 @@ class IPrimaIE(InfoExtractor):
(floor(random() * 1073741824), floor(random() * 1073741824))
)
- req = compat_urllib_request.Request(player_url)
+ req = sanitized_Request(player_url)
req.add_header('Referer', url)
playerpage = self._download_webpage(req, video_id)
diff --git a/youtube_dl/extractor/ivi.py b/youtube_dl/extractor/ivi.py
index e82594444..029878d24 100644
--- a/youtube_dl/extractor/ivi.py
+++ b/youtube_dl/extractor/ivi.py
@@ -5,11 +5,9 @@ import re
import json
from .common import InfoExtractor
-from ..compat import (
- compat_urllib_request,
-)
from ..utils import (
ExtractorError,
+ sanitized_Request,
)
@@ -78,7 +76,7 @@ class IviIE(InfoExtractor):
]
}
- request = compat_urllib_request.Request(api_url, json.dumps(data))
+ request = sanitized_Request(api_url, json.dumps(data))
video_json_page = self._download_webpage(
request, video_id, 'Downloading video JSON')
diff --git a/youtube_dl/extractor/keezmovies.py b/youtube_dl/extractor/keezmovies.py
index 82eddec51..d79261bb5 100644
--- a/youtube_dl/extractor/keezmovies.py
+++ b/youtube_dl/extractor/keezmovies.py
@@ -4,10 +4,8 @@ import os
import re
from .common import InfoExtractor
-from ..compat import (
- compat_urllib_parse_urlparse,
- compat_urllib_request,
-)
+from ..compat import compat_urllib_parse_urlparse
+from ..utils import sanitized_Request
class KeezMoviesIE(InfoExtractor):
@@ -26,7 +24,7 @@ class KeezMoviesIE(InfoExtractor):
def _real_extract(self, url):
video_id = self._match_id(url)
- req = compat_urllib_request.Request(url)
+ req = sanitized_Request(url)
req.add_header('Cookie', 'age_verified=1')
webpage = self._download_webpage(req, video_id)
diff --git a/youtube_dl/extractor/letv.py b/youtube_dl/extractor/letv.py
index effd9eb92..be648000e 100644
--- a/youtube_dl/extractor/letv.py
+++ b/youtube_dl/extractor/letv.py
@@ -8,13 +8,13 @@ import time
from .common import InfoExtractor
from ..compat import (
compat_urllib_parse,
- compat_urllib_request,
compat_ord,
)
from ..utils import (
determine_ext,
ExtractorError,
parse_iso8601,
+ sanitized_Request,
int_or_none,
encode_data_uri,
)
@@ -114,7 +114,7 @@ class LetvIE(InfoExtractor):
'tkey': self.calc_time_key(int(time.time())),
'domain': 'www.letv.com'
}
- play_json_req = compat_urllib_request.Request(
+ play_json_req = sanitized_Request(
'http://api.letv.com/mms/out/video/playJson?' + compat_urllib_parse.urlencode(params)
)
cn_verification_proxy = self._downloader.params.get('cn_verification_proxy')
diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py
index 3d7e7e003..d4e1ae99d 100644
--- a/youtube_dl/extractor/lynda.py
+++ b/youtube_dl/extractor/lynda.py
@@ -7,12 +7,12 @@ from .common import InfoExtractor
from ..compat import (
compat_str,
compat_urllib_parse,
- compat_urllib_request,
)
from ..utils import (
ExtractorError,
clean_html,
int_or_none,
+ sanitized_Request,
)
@@ -35,7 +35,7 @@ class LyndaBaseIE(InfoExtractor):
'remember': 'false',
'stayPut': 'false'
}
- request = compat_urllib_request.Request(
+ request = sanitized_Request(
self._LOGIN_URL, compat_urllib_parse.urlencode(login_form).encode('utf-8'))
login_page = self._download_webpage(
request, None, 'Logging in as %s' % username)
@@ -64,7 +64,7 @@ class LyndaBaseIE(InfoExtractor):
'remember': 'false',
'stayPut': 'false',
}
- request = compat_urllib_request.Request(
+ request = sanitized_Request(
self._LOGIN_URL, compat_urllib_parse.urlencode(confirm_form).encode('utf-8'))
login_page = self._download_webpage(
request, None,
diff --git a/youtube_dl/extractor/metacafe.py b/youtube_dl/extractor/metacafe.py
index 6e2e73a51..3c786a36d 100644
--- a/youtube_dl/extractor/metacafe.py
+++ b/youtube_dl/extractor/metacafe.py
@@ -7,12 +7,12 @@ from ..compat import (
compat_parse_qs,
compat_urllib_parse,
compat_urllib_parse_unquote,
- compat_urllib_request,
)
from ..utils import (
determine_ext,
ExtractorError,
int_or_none,
+ sanitized_Request,
)
@@ -117,7 +117,7 @@ class MetacafeIE(InfoExtractor):
'filters': '0',
'submit': "Continue - I'm over 18",
}
- request = compat_urllib_request.Request(self._FILTER_POST, compat_urllib_parse.urlencode(disclaimer_form))
+ request = sanitized_Request(self._FILTER_POST, compat_urllib_parse.urlencode(disclaimer_form))
request.add_header('Content-Type', 'application/x-www-form-urlencoded')
self.report_age_confirmation()
self._download_webpage(request, None, False, 'Unable to confirm age')
@@ -142,7 +142,7 @@ class MetacafeIE(InfoExtractor):
return self.url_result('theplatform:%s' % ext_id, 'ThePlatform')
# Retrieve video webpage to extract further information
- req = compat_urllib_request.Request('http://www.metacafe.com/watch/%s/' % video_id)
+ req = sanitized_Request('http://www.metacafe.com/watch/%s/' % video_id)
# AnyClip videos require the flashversion cookie so that we get the link
# to the mp4 file
diff --git a/youtube_dl/extractor/minhateca.py b/youtube_dl/extractor/minhateca.py
index 14934b7ec..e46b23a6f 100644
--- a/youtube_dl/extractor/minhateca.py
+++ b/youtube_dl/extractor/minhateca.py
@@ -2,14 +2,12 @@
from __future__ import unicode_literals
from .common import InfoExtractor
-from ..compat import (
- compat_urllib_parse,
- compat_urllib_request,
-)
+from ..compat import compat_urllib_parse
from ..utils import (
int_or_none,
parse_duration,
parse_filesize,
+ sanitized_Request,
)
@@ -39,7 +37,7 @@ class MinhatecaIE(InfoExtractor):
('fileId', video_id),
('__RequestVerificationToken', token),
]
- req = compat_urllib_request.Request(
+ req = sanitized_Request(
'http://minhateca.com.br/action/License/Download',
data=compat_urllib_parse.urlencode(token_data))
req.add_header('Content-Type', 'application/x-www-form-urlencoded')
diff --git a/youtube_dl/extractor/miomio.py b/youtube_dl/extractor/miomio.py
index ce391c759..170ebd9eb 100644
--- a/youtube_dl/extractor/miomio.py
+++ b/youtube_dl/extractor/miomio.py
@@ -4,11 +4,11 @@ from __future__ import unicode_literals
import random
from .common import InfoExtractor
-from ..compat import compat_urllib_request
from ..utils import (
xpath_text,
int_or_none,
ExtractorError,
+ sanitized_Request,
)
@@ -63,7 +63,7 @@ class MioMioIE(InfoExtractor):
'http://www.miomio.tv/mioplayer/mioplayerconfigfiles/xml.php?id=%s&r=%s' % (id, random.randint(100, 999)),
video_id)
- vid_config_request = compat_urllib_request.Request(
+ vid_config_request = sanitized_Request(
'http://www.miomio.tv/mioplayer/mioplayerconfigfiles/sina.php?{0}'.format(xml_config),
headers=http_headers)
diff --git a/youtube_dl/extractor/moevideo.py b/youtube_dl/extractor/moevideo.py
index 5a66302f6..d930b9634 100644
--- a/youtube_dl/extractor/moevideo.py
+++ b/youtube_dl/extractor/moevideo.py
@@ -5,13 +5,11 @@ import json
import re
from .common import InfoExtractor
-from ..compat import (
- compat_urllib_parse,
- compat_urllib_request,
-)
+from ..compat import compat_urllib_parse
from ..utils import (
ExtractorError,
int_or_none,
+ sanitized_Request,
)
@@ -80,7 +78,7 @@ class MoeVideoIE(InfoExtractor):
]
r_json = json.dumps(r)
post = compat_urllib_parse.urlencode({'r': r_json})
- req = compat_urllib_request.Request(self._API_URL, post)
+ req = sanitized_Request(self._API_URL, post)
req.add_header('Content-type', 'application/x-www-form-urlencoded')
response = self._download_json(req, video_id)
diff --git a/youtube_dl/extractor/mofosex.py b/youtube_dl/extractor/mofosex.py
index 9bf99a54a..f8226cbb2 100644
--- a/youtube_dl/extractor/mofosex.py
+++ b/youtube_dl/extractor/mofosex.py
@@ -7,8 +7,8 @@ from .common import InfoExtractor
from ..compat import (
compat_urllib_parse_unquote,
compat_urllib_parse_urlparse,
- compat_urllib_request,
)
+from ..utils import sanitized_Request
class MofosexIE(InfoExtractor):
@@ -29,7 +29,7 @@ class MofosexIE(InfoExtractor):
video_id = mobj.group('id')
url = 'http://www.' + mobj.group('url')
- req = compat_urllib_request.Request(url)
+ req = sanitized_Request(url)
req.add_header('Cookie', 'age_verified=1')
webpage = self._download_webpage(req, video_id)
diff --git a/youtube_dl/extractor/moniker.py b/youtube_dl/extractor/moniker.py
index 7c0c4e50e..f6bf94f2f 100644
--- a/youtube_dl/extractor/moniker.py
+++ b/youtube_dl/extractor/moniker.py
@@ -5,13 +5,11 @@ import os.path
import re
from .common import InfoExtractor
-from ..compat import (
- compat_urllib_parse,
- compat_urllib_request,
-)
+from ..compat import compat_urllib_parse
from ..utils import (
ExtractorError,
remove_start,
+ sanitized_Request,
)
@@ -81,7 +79,7 @@ class MonikerIE(InfoExtractor):
orig_webpage, 'builtin URL', default=None, group='url')
if builtin_url:
- req = compat_urllib_request.Request(builtin_url)
+ req = sanitized_Request(builtin_url)
req.add_header('Referer', url)
webpage = self._download_webpage(req, video_id, 'Downloading builtin page')
title = self._og_search_title(orig_webpage).strip()
@@ -94,7 +92,7 @@ class MonikerIE(InfoExtractor):
headers = {
b'Content-Type': b'application/x-www-form-urlencoded',
}
- req = compat_urllib_request.Request(url, post, headers)
+ req = sanitized_Request(url, post, headers)
webpage = self._download_webpage(
req, video_id, note='Downloading video page ...')
diff --git a/youtube_dl/extractor/mooshare.py b/youtube_dl/extractor/mooshare.py
index 7603af5e2..7cc7f054f 100644
--- a/youtube_dl/extractor/mooshare.py
+++ b/youtube_dl/extractor/mooshare.py
@@ -3,12 +3,10 @@ from __future__ import unicode_literals
import re
from .common import InfoExtractor
-from ..compat import (
- compat_urllib_request,
- compat_urllib_parse,
-)
+from ..compat import compat_urllib_parse
from ..utils import (
ExtractorError,
+ sanitized_Request,
)
@@ -59,7 +57,7 @@ class MooshareIE(InfoExtractor):
'hash': hash_key,
}
- request = compat_urllib_request.Request(
+ request = sanitized_Request(
'http://mooshare.biz/%s' % video_id, compat_urllib_parse.urlencode(download_form))
request.add_header('Content-Type', 'application/x-www-form-urlencoded')
diff --git a/youtube_dl/extractor/movieclips.py b/youtube_dl/extractor/movieclips.py
index b8c43a163..1564cb71f 100644
--- a/youtube_dl/extractor/movieclips.py
+++ b/youtube_dl/extractor/movieclips.py
@@ -2,9 +2,7 @@
from __future__ import unicode_literals
from .common import InfoExtractor
-from ..compat import (
- compat_urllib_request,
-)
+from ..utils import sanitized_Request
class MovieClipsIE(InfoExtractor):
@@ -25,7 +23,7 @@ class MovieClipsIE(InfoExtractor):
def _real_extract(self, url):
display_id = self._match_id(url)
- req = compat_urllib_request.Request(url)
+ req = sanitized_Request(url)
# it doesn't work if it thinks the browser it's too old
req.add_header('User-Agent', 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20150101 Firefox/43.0 (Chrome)')
webpage = self._download_webpage(req, display_id)
diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index 302c9bf35..d887583e6 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -5,7 +5,6 @@ import re
from .common import InfoExtractor
from ..compat import (
compat_urllib_parse,
- compat_urllib_request,
compat_str,
)
from ..utils import (
@@ -13,6 +12,7 @@ from ..utils import (
find_xpath_attr,
fix_xml_ampersands,
HEADRequest,
+ sanitized_Request,
unescapeHTML,
url_basename,
RegexNotFoundError,
@@ -53,7 +53,7 @@ class MTVServicesInfoExtractor(InfoExtractor):
def _extract_mobile_video_formats(self, mtvn_id):
webpage_url = self._MOBILE_TEMPLATE % mtvn_id
- req = compat_urllib_request.Request(webpage_url)
+ req = sanitized_Request(webpage_url)
# Otherwise we get a webpage that would execute some javascript
req.add_header('User-Agent', 'curl/7')
webpage = self._download_webpage(req, mtvn_id,
diff --git a/youtube_dl/extractor/myvideo.py b/youtube_dl/extractor/myvideo.py
index c96f472a3..36ab388b2 100644
--- a/youtube_dl/extractor/myvideo.py
+++ b/youtube_dl/extractor/myvideo.py
@@ -11,10 +11,10 @@ from ..compat import (
compat_ord,
compat_urllib_parse,
compat_urllib_parse_unquote,
- compat_urllib_request,
)
from ..utils import (
ExtractorError,
+ sanitized_Request,
)
@@ -83,7 +83,7 @@ class MyVideoIE(InfoExtractor):
mobj = re.search(r'data-video-service="/service/data/video/%s/config' % video_id, webpage)
if mobj is not None:
- request = compat_urllib_request.Request('http://www.myvideo.de/service/data/video/%s/config' % video_id, '')
+ request = sanitized_Request('http://www.myvideo.de/service/data/video/%s/config' % video_id, '')
response = self._download_webpage(request, video_id,
'Downloading video info')
info = json.loads(base64.b64decode(response).decode('utf-8'))
diff --git a/youtube_dl/extractor/neteasemusic.py b/youtube_dl/extractor/neteasemusic.py
index bb3362069..15eca825a 100644
--- a/youtube_dl/extractor/neteasemusic.py
+++ b/youtube_dl/extractor/neteasemusic.py
@@ -8,11 +8,11 @@ import re
from .common import InfoExtractor
from ..compat import (
- compat_urllib_request,
compat_urllib_parse,
compat_str,
compat_itertools_count,
)
+from ..utils import sanitized_Request
class NetEaseMusicBaseIE(InfoExtractor):
@@ -56,7 +56,7 @@ class NetEaseMusicBaseIE(InfoExtractor):
return int(round(ms / 1000.0))
def query_api(self, endpoint, video_id, note):
- req = compat_urllib_request.Request('%s%s' % (self._API_BASE, endpoint))
+ req = sanitized_Request('%s%s' % (self._API_BASE, endpoint))
req.add_header('Referer', self._API_BASE)
return self._download_json(req, video_id, note)
diff --git a/youtube_dl/extractor/nfb.py b/youtube_dl/extractor/nfb.py
index ea077254b..5bd15f7a7 100644
--- a/youtube_dl/extractor/nfb.py
+++ b/youtube_dl/extractor/nfb.py
@@ -1,10 +1,8 @@
from __future__ import unicode_literals
from .common import InfoExtractor
-from ..compat import (
- compat_urllib_request,
- compat_urllib_parse,
-)
+from ..compat import compat_urllib_parse
+from ..utils import sanitized_Request
class NFBIE(InfoExtractor):
@@ -40,8 +38,9 @@ class NFBIE(InfoExtractor):
uploader = self._html_search_regex(r'<em class="director-name" itemprop="name">([^<]+)</em>',
page, 'director name', fatal=False)
- request = compat_urllib_request.Request('https://www.nfb.ca/film/%s/player_config' % video_id,
- compat_urllib_parse.urlencode({'getConfig': 'true'}).encode('ascii'))
+ request = sanitized_Request(
+ 'https://www.nfb.ca/film/%s/player_config' % video_id,
+ compat_urllib_parse.urlencode({'getConfig': 'true'}).encode('ascii'))
request.add_header('Content-Type', 'application/x-www-form-urlencoded')
request.add_header('X-NFB-Referer', 'http://www.nfb.ca/medias/flash/NFBVideoPlayer.swf')
diff --git a/youtube_dl/extractor/niconico.py b/youtube_dl/extractor/niconico.py
index bda1cff05..586e52a4a 100644
--- a/youtube_dl/extractor/niconico.py
+++ b/youtube_dl/extractor/niconico.py
@@ -8,7 +8,6 @@ import datetime
from .common import InfoExtractor
from ..compat import (
compat_urllib_parse,
- compat_urllib_request,
compat_urlparse,
)
from ..utils import (
@@ -17,6 +16,7 @@ from ..utils import (
int_or_none,
parse_duration,
parse_iso8601,
+ sanitized_Request,
xpath_text,
determine_ext,
)
@@ -102,7 +102,7 @@ class NiconicoIE(InfoExtractor):
'password': password,
}
login_data = compat_urllib_parse.urlencode(encode_dict(login_form_strs)).encode('utf-8')
- request = compat_urllib_request.Request(
+ request = sanitized_Request(
'https://secure.nicovideo.jp/secure/login', login_data)
login_results = self._download_webpage(
request, None, note='Logging in', errnote='Unable to log in')
@@ -145,7 +145,7 @@ class NiconicoIE(InfoExtractor):
'k': thumb_play_key,
'v': video_id
})
- flv_info_request = compat_urllib_request.Request(
+ flv_info_request = sanitized_Request(
'http://ext.nicovideo.jp/thumb_watch', flv_info_data,
{'Content-Type': 'application/x-www-form-urlencoded'})
flv_info_webpage = self._download_webpage(
diff --git a/youtube_dl/extractor/noco.py b/youtube_dl/extractor/noco.py
index a53e27b27..76bd21e6d 100644
--- a/youtube_dl/extractor/noco.py
+++ b/youtube_dl/extractor/noco.py
@@ -9,7 +9,6 @@ from .common import InfoExtractor
from ..compat import (
compat_str,
compat_urllib_parse,
- compat_urllib_request,
)
from ..utils import (
clean_html,
@@ -17,6 +16,7 @@ from ..utils import (
int_or_none,
float_or_none,
parse_iso8601,
+ sanitized_Request,
)
@@ -74,7 +74,7 @@ class NocoIE(InfoExtractor):
'username': username,
'password': password,
}
- request = compat_urllib_request.Request(self._LOGIN_URL, compat_urllib_parse.urlencode(login_form))
+ request = sanitized_Request(self._LOGIN_URL, compat_urllib_parse.urlencode(login_form))
request.add_header('Content-Type', 'application/x-www-form-urlencoded; charset=UTF-8')
login = self._download_json(request, None, 'Logging in as %s' % username)
diff --git a/youtube_dl/extractor/nosvideo.py b/youtube_dl/extractor/nosvideo.py
index f5ef856db..eab816e49 100644
--- a/youtube_dl/extractor/nosvideo.py
+++ b/youtube_dl/extractor/nosvideo.py
@@ -4,11 +4,9 @@ from __future__ import unicode_literals
import re
from .common import InfoExtractor
-from ..compat import (
- compat_urllib_request,
-)
from ..utils import (
ExtractorError,
+ sanitized_Request,
urlencode_postdata,
xpath_text,
xpath_with_ns,
@@ -41,7 +39,7 @@ class NosVideoIE(InfoExtractor):
'op': 'download1',
'method_free': 'Continue to Video',
}
- req = compat_urllib_request.Request(url, urlencode_postdata(fields))
+ req = sanitized_Request(url, urlencode_postdata(fields))
req.add_header('Content-type', 'application/x-www-form-urlencoded')
webpage = self._download_webpage(req, video_id,
'Downloading download page')
diff --git a/youtube_dl/extractor/novamov.py b/youtube_dl/extractor/novamov.py
index 6b15fc2e5..6163e8855 100644
--- a/youtube_dl/extractor/novamov.py
+++ b/youtube_dl/extractor/novamov.py
@@ -3,14 +3,12 @@ from __future__ import unicode_literals
import re
from .common import InfoExtractor
-from ..compat import (
- compat_urllib_request,
- compat_urlparse,
-)
+from ..compat import compat_urlparse
from ..utils import (
ExtractorError,
NO_DEFAULT,
encode_dict,
+ sanitized_Request,
urlencode_postdata,
)
@@ -65,7 +63,7 @@ class NovaMovIE(InfoExtractor):
'post url', default=url, group='url')
if not post_url.startswith('http'):
post_url = compat_urlparse.urljoin(url, post_url)
- request = compat_urllib_request.Request(
+ request = sanitized_Request(
post_url, urlencode_postdata(encode_dict(fields)))
request.add_header('Content-Type', 'application/x-www-form-urlencoded')
request.add_header('Referer', post_url)
diff --git a/youtube_dl/extractor/nowness.py b/youtube_dl/extractor/nowness.py
index 0fba55833..d480fb58c 100644
--- a/youtube_dl/extractor/nowness.py
+++ b/youtube_dl/extractor/nowness.py
@@ -3,10 +3,10 @@ from __future__ import unicode_literals
from .brightcove import BrightcoveLegacyIE
from .common import InfoExtractor
-from ..utils import ExtractorError
-from ..compat import (
- compat_str,
- compat_urllib_request,
+from ..compat import compat_str
+from ..utils import (
+ ExtractorError,
+ sanitized_Request,
)
@@ -37,7 +37,7 @@ class NownessBaseIE(InfoExtractor):
def _api_request(self, url, request_path):
display_id = self._match_id(url)
- request = compat_urllib_request.Request(
+ request = sanitized_Request(
'http://api.nowness.com/api/' + request_path % display_id,
headers={
'X-Nowness-Language': 'zh-cn' if 'cn.nowness.com' in url else 'en-us',
diff --git a/youtube_dl/extractor/nuvid.py b/youtube_dl/extractor/nuvid.py
index 57928f2ae..9fa7cefad 100644
--- a/youtube_dl/extractor/nuvid.py
+++ b/youtube_dl/extractor/nuvid.py
@@ -3,11 +3,9 @@ from __future__ import unicode_literals
import re
from .common import InfoExtractor
-from ..compat import (
- compat_urllib_request,
-)
from ..utils import (
parse_duration,
+ sanitized_Request,
unified_strdate,
)
@@ -33,7 +31,7 @@ class NuvidIE(InfoExtractor):
formats = []
for dwnld_speed, format_id in [(0, '3gp'), (5, 'mp4')]:
- request = compat_urllib_request.Request(
+ request = sanitized_Request(
'http://m.nuvid.com/play/%s' % video_id)
request.add_header('Cookie', 'skip_download_page=1; dwnld_speed=%d; adv_show=1' % dwnld_speed)
webpage = self._download_webpage(
diff --git a/youtube_dl/extractor/patreon.py b/youtube_dl/extractor/patreon.py
index 6cdc2638b..ec8876c28 100644
--- a/youtube_dl/extractor/patreon.py
+++ b/youtube_dl/extractor/patreon.py
@@ -2,9 +2,7 @@
from __future__ import unicode_literals
from .common import InfoExtractor
-from ..utils import (
- js_to_json,
-)
+from ..utils import js_to_json
class PatreonIE(InfoExtractor):
@@ -65,7 +63,7 @@ class PatreonIE(InfoExtractor):
'password': password,
}
- request = compat_urllib_request.Request(
+ request = sanitized_Request(
'https://www.patreon.com/processLogin',
compat_urllib_parse.urlencode(login_form).encode('utf-8')
)
diff --git a/youtube_dl/extractor/played.py b/youtube_dl/extractor/played.py
index 8a1c296dd..2856af96f 100644
--- a/youtube_dl/extractor/played.py
+++ b/youtube_dl/extractor/played.py
@@ -5,12 +5,10 @@ import re
import os.path
from .common import InfoExtractor
-from ..compat import (
- compat_urllib_parse,
- compat_urllib_request,
-)
+from ..compat import compat_urllib_parse
from ..utils import (
ExtractorError,
+ sanitized_Request,
)
@@ -46,7 +44,7 @@ class PlayedIE(InfoExtractor):
headers = {
b'Content-Type': b'application/x-www-form-urlencoded',
}
- req = compat_urllib_request.Request(url, post, headers)
+ req = sanitized_Request(url, post, headers)
webpage = self._download_webpage(
req, video_id, note='Downloading video page ...')
diff --git a/youtube_dl/extractor/pluralsight.py b/youtube_dl/extractor/pluralsight.py
index 792316db8..aa7dbcb63 100644
--- a/youtube_dl/extractor/pluralsight.py
+++ b/youtube_dl/extractor/pluralsight.py
@@ -8,13 +8,13 @@ from .common import InfoExtractor
from ..compat import (
compat_str,
compat_urllib_parse,
- compat_urllib_request,
compat_urlparse,
)
from ..utils import (
ExtractorError,
int_or_none,
parse_duration,
+ sanitized_Request,
)
@@ -73,7 +73,7 @@ class PluralsightIE(PluralsightBaseIE):
if not post_url.startswith('http'):
post_url = compat_urlparse.urljoin(self._LOGIN_URL, post_url)
- request = compat_urllib_request.Request(
+ request = sanitized_Request(
post_url, compat_urllib_parse.urlencode(login_form).encode('utf-8'))
request.add_header('Content-Type', 'application/x-www-form-urlencoded')
@@ -181,7 +181,7 @@ class PluralsightIE(PluralsightBaseIE):
'mt': ext,
'q': '%dx%d' % (f['width'], f['height']),
}
- request = compat_urllib_request.Request(
+ request = sanitized_Request(
'%s/training/Player/ViewClip' % self._API_BASE,
json.dumps(clip_post).encode('utf-8'))
request.add_header('Content-Type', 'application/json;charset=utf-8')
diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index a656ad85a..965940a4b 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -8,10 +8,10 @@ from ..compat import (
compat_urllib_parse_unquote,
compat_urllib_parse_unquote_plus,
compat_urllib_parse_urlparse,
- compat_urllib_request,
)
from ..utils import (
ExtractorError,
+ sanitized_Request,
str_to_int,
)
from ..aes import (
@@ -53,7 +53,7 @@ class PornHubIE(InfoExtractor):
def _real_extract(self, url):
video_id = self._match_id(url)
- req = compat_urllib_request.Request(
+ req = sanitized_Request(
'http://www.pornhub.com/view_video.php?viewkey=%s' % video_id)
req.add_header('Cookie', 'age_verified=1')
webpage = self._download_webpage(req, video_id)
diff --git a/youtube_dl/extractor/pornotube.py b/youtube_dl/extractor/pornotube.py
index 34735c51e..5398e708b 100644
--- a/youtube_dl/extractor/pornotube.py
+++ b/youtube_dl/extractor/pornotube.py
@@ -3,11 +3,9 @@ from __future__ import unicode_literals
import json
from .common import InfoExtractor
-from ..compat import (
- compat_urllib_request,
-)
from ..utils import (
int_or_none,
+ sanitized_Request,
)
@@ -46,7 +44,7 @@ class PornotubeIE(InfoExtractor):
'authenticationSpaceKey': originAuthenticationSpaceKey,
'credentials': 'Clip Application',
}
- token_req = compat_urllib_request.Request(
+ token_req = sanitized_Request(
'https://api.aebn.net/auth/v1/token/primal',
data=json.dumps(token_req_data).encode('utf-8'))
token_req.add_header('Content-Type', 'application/json')
@@ -56,7 +54,7 @@ class PornotubeIE(InfoExtractor):
token = token_answer['tokenKey']
# Get video URL
- delivery_req = compat_urllib_request.Request(
+ delivery_req = sanitized_Request(
'https://api.aebn.net/delivery/v1/clips/%s/MP4' % video_id)
delivery_req.add_header('Authorization', token)
delivery_info = self._download_json(
@@ -64,7 +62,7 @@ class PornotubeIE(InfoExtractor):
video_url = delivery_info['mediaUrl']
# Get additional info (title etc.)
- info_req = compat_urllib_request.Request(
+ info_req = sanitized_Request(
'https://api.aebn.net/content/v1/clips/%s?expand='
'title,description,primaryImageNumber,startSecond,endSecond,'
'movie.title,movie.MovieId,movie.boxCoverFront,movie.stars,'
diff --git a/youtube_dl/extractor/primesharetv.py b/youtube_dl/extractor/primesharetv.py
index 304359dc5..85aae9576 100644
--- a/youtube_dl/extractor/primesharetv.py
+++ b/youtube_dl/extractor/primesharetv.py
@@ -1,11 +1,11 @@
from __future__ import unicode_literals
from .common import InfoExtractor
-from ..compat import (
- compat_urllib_parse,
- compat_urllib_request,
+from ..compat import compat_urllib_parse
+from ..utils import (
+ ExtractorError,
+ sanitized_Request,
)
-from ..utils import ExtractorError
class PrimeShareTVIE(InfoExtractor):
@@ -41,7 +41,7 @@ class PrimeShareTVIE(InfoExtractor):
webpage, 'wait time', default=7)) + 1
self._sleep(wait_time, video_id)
- req = compat_urllib_request.Request(
+ req = sanitized_Request(
url, compat_urllib_parse.urlencode(fields), headers)
video_page = self._download_webpage(
req, video_id, 'Downloading video page')
diff --git a/youtube_dl/extractor/promptfile.py b/youtube_dl/extractor/promptfile.py
index 8190ed676..d5357283a 100644
--- a/youtube_dl/extractor/promptfile.py
+++ b/youtube_dl/extractor/promptfile.py
@@ -4,13 +4,11 @@ from __future__ import unicode_literals
import re
from .common import InfoExtractor
-from ..compat import (
- compat_urllib_parse,
- compat_urllib_request,
-)
+from ..compat import compat_urllib_parse
from ..utils import (
determine_ext,
ExtractorError,
+ sanitized_Request,
)
@@ -37,7 +35,7 @@ class PromptFileIE(InfoExtractor):
fields = self._hidden_inputs(webpage)
post = compat_urllib_parse.urlencode(fields)
- req = compat_urllib_request.Request(url, post)
+ req = sanitized_Request(url, post)
req.add_header('Content-type', 'application/x-www-form-urlencoded')
webpage = self._download_webpage(
req, video_id, 'Downloading video page')
diff --git a/youtube_dl/extractor/qqmusic.py b/youtube_dl/extractor/qqmusic.py
index c98539f6a..1ba3bbddf 100644
--- a/youtube_dl/extractor/qqmusic.py
+++ b/youtube_dl/extractor/qqmusic.py
@@ -7,11 +7,11 @@ import re
from .common import InfoExtractor
from ..utils import (
+ sanitized_Request,
strip_jsonp,
unescapeHTML,
clean_html,
)
-from ..compat import compat_urllib_request
class QQMusicIE(InfoExtractor):
@@ -201,7 +201,7 @@ class QQMusicSingerIE(QQPlaylistBaseIE):
singer_desc = None
if singer_id:
- req = compat_urllib_request.Request(
+ req = sanitized_Request(
'http://s.plcloud.music.qq.com/fcgi-bin/fcg_get_singer_desc.fcg?utf8=1&outCharset=utf-8&format=xml&singerid=%s' % singer_id)
req.add_header(
'Referer', 'http://s.plcloud.music.qq.com/xhr_proxy_utf8.html')
diff --git a/youtube_dl/extractor/rtve.py b/youtube_dl/extractor/rtve.py
index 0fe6356db..603d7bd00 100644
--- a/youtube_dl/extractor/rtve.py
+++ b/youtube_dl/extractor/rtve.py
@@ -6,11 +6,11 @@ import re
import time
from .common import InfoExtractor
-from ..compat import compat_urllib_request
from ..utils import (
ExtractorError,
float_or_none,
remove_end,
+ sanitized_Request,
std_headers,
struct_unpack,
)
@@ -102,7 +102,7 @@ class RTVEALaCartaIE(InfoExtractor):
if info['state'] == 'DESPU':
raise ExtractorError('The video is no longer available', expected=True)
png_url = 'http://www.rtve.es/ztnr/movil/thumbnail/%s/videos/%s.png' % (self._manager, video_id)
- png_request = compat_urllib_request.Request(png_url)
+ png_request = sanitized_Request(png_url)
png_request.add_header('Referer', url)
png = self._download_webpage(png_request, video_id, 'Downloading url information')
video_url = _decrypt_url(png)
diff --git a/youtube_dl/extractor/safari.py b/youtube_dl/extractor/safari.py
index e9e33d0a3..919704261 100644
--- a/youtube_dl/extractor/safari.py
+++ b/youtube_dl/extractor/safari.py
@@ -6,12 +6,10 @@ import re
from .common import InfoExtractor
from .brightcove import BrightcoveLegacyIE
-from ..compat import (
- compat_urllib_parse,
- compat_urllib_request,
-)
+from ..compat import compat_urllib_parse
from ..utils import (
ExtractorError,
+ sanitized_Request,
smuggle_url,
std_headers,
)
@@ -58,7 +56,7 @@ class SafariBaseIE(InfoExtractor):
'next': '',
}
- request = compat_urllib_request.Request(
+ request = sanitized_Request(
self._LOGIN_URL, compat_urllib_parse.urlencode(login_form), headers=headers)
login_page = self._download_webpage(
request, None, 'Logging in as %s' % username)
diff --git a/youtube_dl/extractor/sandia.py b/youtube_dl/extractor/sandia.py
index 9c88167f0..759898a49 100644
--- a/youtube_dl/extractor/sandia.py
+++ b/youtube_dl/extractor/sandia.py
@@ -6,14 +6,12 @@ import json
import re
from .common import InfoExtractor
-from ..compat import (
- compat_urllib_request,
- compat_urlparse,
-)
+from ..compat import compat_urlparse
from ..utils import (
int_or_none,
js_to_json,
mimetype2ext,
+ sanitized_Request,
unified_strdate,
)
@@ -37,7 +35,7 @@ class SandiaIE(InfoExtractor):
def _real_extract(self, url):
video_id = self._match_id(url)
- req = compat_urllib_request.Request(url)
+ req = sanitized_Request(url)
req.add_header('Cookie', 'MediasitePlayerCaps=ClientPlugins=4')
webpage = self._download_webpage(req, video_id)
diff --git a/youtube_dl/extractor/shared.py b/youtube_dl/extractor/shared.py
index c5636e8e9..8eda3c864 100644
--- a/youtube_dl/extractor/shared.py
+++ b/youtube_dl/extractor/shared.py
@@ -3,13 +3,11 @@ from __future__ import unicode_literals
import base64
from .common import InfoExtractor
-from ..compat import (
- compat_urllib_parse,
- compat_urllib_request,
-)
+from ..compat import compat_urllib_parse
from ..utils import (
ExtractorError,
int_or_none,
+ sanitized_Request,
)
@@ -46,7 +44,7 @@ class SharedIE(InfoExtractor):
'Video %s does not exist' % video_id, expected=True)
download_form = self._hidden_inputs(webpage)
- request = compat_urllib_request.Request(
+ request = sanitized_Request(
url, compat_urllib_parse.urlencode(download_form))
request.add_header('Content-Type', 'application/x-www-form-urlencoded')
diff --git a/youtube_dl/extractor/sharesix.py b/youtube_dl/extractor/sharesix.py
index ac3e3adf2..f1ea9bdb2 100644
--- a/youtube_dl/extractor/sharesix.py
+++ b/youtube_dl/extractor/sharesix.py
@@ -4,12 +4,10 @@ from __future__ import unicode_literals
import re
from .common import InfoExtractor
-from ..compat import (
- compat_urllib_parse,
- compat_urllib_request,
-)
+from ..compat import compat_urllib_parse
from ..utils import (
parse_duration,
+ sanitized_Request,
)
@@ -50,7 +48,7 @@ class ShareSixIE(InfoExtractor):
'method_free': 'Free'
}
post = compat_urllib_parse.urlencode(fields)
- req = compat_urllib_request.Request(url, post)
+ req = sanitized_Request(url, post)
req.add_header('Content-type', 'application/x-www-form-urlencoded')
webpage = self._download_webpage(req, video_id,
diff --git a/youtube_dl/extractor/sina.py b/youtube_dl/extractor/sina.py
index 0891a441f..b2258a0f6 100644
--- a/youtube_dl/extractor/sina.py
+++ b/youtube_dl/extractor/sina.py
@@ -4,10 +4,8 @@ from __future__ import unicode_literals
import re
from .common import InfoExtractor
-from ..compat import (
- compat_urllib_request,
- compat_urllib_parse,
-)
+from ..compat import compat_urllib_parse
+from ..utils import sanitized_Request
class SinaIE(InfoExtractor):
@@ -61,7 +59,7 @@ class SinaIE(InfoExtractor):
if mobj.group('token') is not None:
# The video id is in the redirected url
self.to_screen('Getting video id')
- request = compat_urllib_request.Request(url)
+ request = sanitized_Request(url)
request.get_method = lambda: 'HEAD'
(_, urlh) = self._download_webpage_handle(request, 'NA', False)
return self._real_extract(urlh.geturl())
diff --git a/youtube_dl/extractor/smotri.py b/youtube_dl/extractor/smotri.py
index 35a81ee87..30210c8a3 100644
--- a/youtube_dl/extractor/smotri.py
+++ b/youtube_dl/extractor/smotri.py
@@ -7,13 +7,11 @@ import hashlib
import uuid
from .common import InfoExtractor
-from ..compat import (
- compat_urllib_parse,
- compat_urllib_request,
-)
+from ..compat import compat_urllib_parse
from ..utils import (
ExtractorError,
int_or_none,
+ sanitized_Request,
unified_strdate,
)
@@ -176,7 +174,7 @@ class SmotriIE(InfoExtractor):
if video_password:
video_form['pass'] = hashlib.md5(video_password.encode('utf-8')).hexdigest()
- request = compat_urllib_request.Request(
+ request = sanitized_Request(
'http://smotri.com/video/view/url/bot/', compat_urllib_parse.urlencode(video_form))
request.add_header('Content-Type', 'application/x-www-form-urlencoded')
@@ -339,7 +337,7 @@ class SmotriBroadcastIE(InfoExtractor):
'password': password,
}
- request = compat_urllib_request.Request(
+ request = sanitized_Request(
broadcast_url + '/?no_redirect=1', compat_urllib_parse.urlencode(login_form))
request.add_header('Content-Type', 'application/x-www-form-urlencoded')
broadcast_page = self._download_webpage(
diff --git a/youtube_dl/extractor/sohu.py b/youtube_dl/extractor/sohu.py
index ba2d5e19b..daf6ad555 100644
--- a/youtube_dl/extractor/sohu.py
+++ b/youtube_dl/extractor/sohu.py
@@ -6,11 +6,11 @@ import re
from .common import InfoExtractor
from ..compat import (
compat_str,
- compat_urllib_request,
compat_urllib_parse,
)
from ..utils import (
ExtractorError,
+ sanitized_Request,
)
@@ -96,7 +96,7 @@ class SohuIE(InfoExtractor):
else:
base_data_url = 'http://hot.vrs.sohu.com/vrs_flash.action?vid='
- req = compat_urllib_request.Request(base_data_url + vid_id)
+ req = sanitized_Request(base_data_url + vid_id)
cn_verification_proxy = self._downloader.params.get('cn_verification_proxy')
if cn_verification_proxy:
diff --git a/youtube_dl/extractor/spankwire.py b/youtube_dl/extractor/spankwire.py
index 9e8fb35b2..692fd78e8 100644
--- a/youtube_dl/extractor/spankwire.py
+++ b/youtube_dl/extractor/spankwire.py
@@ -6,9 +6,9 @@ from .common import InfoExtractor
from ..compat import (
compat_urllib_parse_unquote,
compat_urllib_parse_urlparse,
- compat_urllib_request,
)
from ..utils import (
+ sanitized_Request,
str_to_int,
unified_strdate,
)
@@ -51,7 +51,7 @@ class SpankwireIE(InfoExtractor):
mobj = re.match(self._VALID_URL, url)
video_id = mobj.group('id')
- req = compat_urllib_request.Request('http://www.' + mobj.group('url'))
+ req = sanitized_Request('http://www.' + mobj.group('url'))
req.add_header('Cookie', 'age_verified=1')
webpage = self._download_webpage(req, video_id)
diff --git a/youtube_dl/extractor/sportdeutschland.py b/youtube_dl/extractor/sportdeutschland.py
index 7ec6c613f..ebb75f059 100644
--- a/youtube_dl/extractor/sportdeutschland.py
+++ b/youtube_dl/extractor/sportdeutschland.py
@@ -4,11 +4,9 @@ from __future__ import unicode_literals
import re
from .common import InfoExtractor
-from ..compat import (
- compat_urllib_request,
-)
from ..utils import (
parse_iso8601,
+ sanitized_Request,
)
@@ -54,7 +52,7 @@ class SportDeutschlandIE(InfoExtractor):
api_url = 'http://proxy.vidibusdynamic.net/sportdeutschland.tv/api/permalinks/%s/%s?access_token=true' % (
sport_id, video_id)
- req = compat_urllib_request.Request(api_url, headers={
+ req = sanitized_Request(api_url, headers={
'Accept': 'application/vnd.vidibus.v2.html+json',
'Referer': url,
})
diff --git a/youtube_dl/extractor/streamcloud.py b/youtube_dl/extractor/streamcloud.py
index d4e134015..77841b946 100644
--- a/youtube_dl/extractor/streamcloud.py
+++ b/youtube_dl/extractor/streamcloud.py
@@ -4,10 +4,8 @@ from __future__ import unicode_literals
import re
from .common import InfoExtractor
-from ..compat import (
- compat_urllib_parse,
- compat_urllib_request,
-)
+from ..compat import compat_urllib_parse
+from ..utils import sanitized_Request
class StreamcloudIE(InfoExtractor):
@@ -43,7 +41,7 @@ class StreamcloudIE(InfoExtractor):
headers = {
b'Content-Type': b'application/x-www-form-urlencoded',
}
- req = compat_urllib_request.Request(url, post, headers)
+ req = sanitized_Request(url, post, headers)
webpage = self._download_webpage(
req, video_id, note='Downloading video page ...')
diff --git a/youtube_dl/extractor/streamcz.py b/youtube_dl/extractor/streamcz.py
index e92b93285..d3d2b7eb7 100644
--- a/youtube_dl/extractor/streamcz.py
+++ b/youtube_dl/extractor/streamcz.py
@@ -5,11 +5,9 @@ import hashlib
import time
from .common import InfoExtractor
-from ..compat import (
- compat_urllib_request,
-)
from ..utils import (
int_or_none,
+ sanitized_Request,
)
@@ -54,7 +52,7 @@ class StreamCZIE(InfoExtractor):
video_id = self._match_id(url)
api_path = '/episode/%s' % video_id
- req = compat_urllib_request.Request(self._API_URL + api_path)
+ req = sanitized_Request(self._API_URL + api_path)
req.add_header('Api-Password', _get_api_key(api_path))
data = self._download_json(req, video_id)
diff --git a/youtube_dl/extractor/tapely.py b/youtube_dl/extractor/tapely.py
index 744f9db38..ed560bd24 100644
--- a/youtube_dl/extractor/tapely.py
+++ b/youtube_dl/extractor/tapely.py
@@ -4,14 +4,12 @@ from __future__ import unicode_literals
import re
from .common import InfoExtractor
-from ..compat import (
- compat_urllib_request,
-)
from ..utils import (
clean_html,
ExtractorError,
float_or_none,
parse_iso8601,
+ sanitized_Request,
)
@@ -53,7 +51,7 @@ class TapelyIE(InfoExtractor):
display_id = mobj.group('id')
playlist_url = self._API_URL.format(display_id)
- request = compat_urllib_request.Request(playlist_url)
+ request = sanitized_Request(playlist_url)
request.add_header('X-Requested-With', 'XMLHttpRequest')
request.add_header('Accept', 'application/json')
request.add_header('Referer', url)
diff --git a/youtube_dl/extractor/tube8.py b/youtube_dl/extractor/tube8.py
index c9cb69333..46ef61ff5 100644
--- a/youtube_dl/extractor/tube8.py
+++ b/youtube_dl/extractor/tube8.py
@@ -4,12 +4,10 @@ import json
import re
from .common import InfoExtractor
-from ..compat import (
- compat_urllib_parse_urlparse,
- compat_urllib_request,
-)
+from ..compat import compat_urllib_parse_urlparse
from ..utils import (
int_or_none,
+ sanitized_Request,
str_to_int,
)
from ..aes import aes_decrypt_text
@@ -42,7 +40,7 @@ class Tube8IE(InfoExtractor):
video_id = mobj.group('id')
display_id = mobj.group('display_id')
- req = compat_urllib_request.Request(url)
+ req = sanitized_Request(url)
req.add_header('Cookie', 'age_verified=1')
webpage = self._download_webpage(req, display_id)
diff --git a/youtube_dl/extractor/tubitv.py b/youtube_dl/extractor/tubitv.py
index 4f86b3ee9..6d78b5dfe 100644
--- a/youtube_dl/extractor/tubitv.py
+++ b/youtube_dl/extractor/tubitv.py
@@ -5,13 +5,11 @@ import codecs
import re
from .common import InfoExtractor
-from ..compat import (
- compat_urllib_parse,
- compat_urllib_request
-)
+from ..compat import compat_urllib_parse
from ..utils import (
ExtractorError,
int_or_none,
+ sanitized_Request,
)
@@ -44,7 +42,7 @@ class TubiTvIE(InfoExtractor):
'password': password,
}
payload = compat_urllib_parse.urlencode(form_data).encode('utf-8')
- request = compat_urllib_request.Request(self._LOGIN_URL, payload)
+ request = sanitized_Request(self._LOGIN_URL, payload)
request.add_header('Content-Type', 'application/x-www-form-urlencoded')
login_page = self._download_webpage(
request, None, False, 'Wrong login info')
diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 3ec08b674..69882da63 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -11,7 +11,6 @@ from ..compat import (
compat_str,
compat_urllib_parse,
compat_urllib_parse_urlparse,
- compat_urllib_request,
compat_urlparse,
)
from ..utils import (
@@ -20,6 +19,7 @@ from ..utils import (
int_or_none,
parse_duration,
parse_iso8601,
+ sanitized_Request,
)
@@ -48,7 +48,7 @@ class TwitchBaseIE(InfoExtractor):
for cookie in self._downloader.cookiejar:
if cookie.name == 'api_token':
headers['Twitch-Api-Token'] = cookie.value
- request = compat_urllib_request.Request(url, headers=headers)
+ request = sanitized_Request(url, headers=headers)
response = super(TwitchBaseIE, self)._download_json(request, video_id, note)
self._handle_error(response)
return response
@@ -80,7 +80,7 @@ class TwitchBaseIE(InfoExtractor):
if not post_url.startswith('http'):
post_url = compat_urlparse.urljoin(redirect_url, post_url)
- request = compat_urllib_request.Request(
+ request = sanitized_Request(
post_url, compat_urllib_parse.urlencode(encode_dict(login_form)).encode('utf-8'))
request.add_header('Referer', redirect_url)
response = self._download_webpage(
diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index 055047340..a161f046b 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -4,13 +4,13 @@ from __future__ import unicode_literals
import re
from .common import InfoExtractor
-from ..compat import compat_urllib_request
from ..utils import (
float_or_none,
xpath_text,
remove_end,
int_or_none,
ExtractorError,
+ sanitized_Request,
)
@@ -81,7 +81,7 @@ class TwitterCardIE(InfoExtractor):
config = None
formats = []
for user_agent in USER_AGENTS:
- request = compat_urllib_request.Request(url)
+ request = sanitized_Request(url)
request.add_header('User-Agent', user_agent)
webpage = self._download_webpage(request, video_id)
diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index 365d8b4bf..825172806 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -9,6 +9,7 @@ from ..compat import (
)
from ..utils import (
ExtractorError,
+ sanitized_Request,
)
@@ -58,7 +59,7 @@ class UdemyIE(InfoExtractor):
for header, value in headers.items():
url_or_request.add_header(header, value)
else:
- url_or_request = compat_urllib_request.Request(url_or_request, headers=headers)
+ url_or_request = sanitized_Request(url_or_request, headers=headers)
response = super(UdemyIE, self)._download_json(url_or_request, video_id, note)
self._handle_error(response)
@@ -89,7 +90,7 @@ class UdemyIE(InfoExtractor):
'password': password.encode('utf-8'),
})
- request = compat_urllib_request.Request(
+ request = sanitized_Request(
self._LOGIN_URL, compat_urllib_parse.urlencode(login_form).encode('utf-8'))
request.add_header('Referer', self._ORIGIN_URL)
request.add_header('Origin', self._ORIGIN_URL)
diff --git a/youtube_dl/extractor/vbox7.py b/youtube_dl/extractor/vbox7.py
index 722eb5236..1e740fbe6 100644
--- a/youtube_dl/extractor/vbox7.py
+++ b/youtube_dl/extractor/vbox7.py
@@ -4,11 +4,11 @@ from __future__ import unicode_literals
from .common import InfoExtractor
from ..compat import (
compat_urllib_parse,
- compat_urllib_request,
compat_urlparse,
)
from ..utils import (
ExtractorError,
+ sanitized_Request,
)
@@ -49,7 +49,7 @@ class Vbox7IE(InfoExtractor):
info_url = "http://vbox7.com/play/magare.do"
data = compat_urllib_parse.urlencode({'as3': '1', 'vid': video_id})
- info_request = compat_urllib_request.Request(info_url, data)
+ info_request = sanitized_Request(info_url, data)
info_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
info_response = self._download_webpage(info_request, video_id, 'Downloading info webpage')
if info_response is None:
diff --git a/youtube_dl/extractor/veoh.py b/youtube_dl/extractor/veoh.py
index 01e258e32..9633f7ffe 100644
--- a/youtube_dl/extractor/veoh.py
+++ b/youtube_dl/extractor/veoh.py
@@ -4,12 +4,10 @@ import re
import json
from .common import InfoExtractor
-from ..compat import (
- compat_urllib_request,
-)
from ..utils import (
int_or_none,
ExtractorError,
+ sanitized_Request,
)
@@ -110,7 +108,7 @@ class VeohIE(InfoExtractor):
if 'class="adultwarning-container"' in webpage:
self.report_age_confirmation()
age_limit = 18
- request = compat_urllib_request.Request(url)
+ request = sanitized_Request(url)
request.add_header('Cookie', 'confirmedAdult=true')
webpage = self._download_webpage(request, video_id)
diff --git a/youtube_dl/extractor/vessel.py b/youtube_dl/extractor/vessel.py
index 3c8d2a943..1a0ff3395 100644
--- a/youtube_dl/extractor/vessel.py
+++ b/youtube_dl/extractor/vessel.py
@@ -4,10 +4,10 @@ from __future__ import unicode_literals
import json
from .common import InfoExtractor
-from ..compat import compat_urllib_request
from ..utils import (
ExtractorError,
parse_iso8601,
+ sanitized_Request,
)
@@ -33,7 +33,7 @@ class VesselIE(InfoExtractor):
@staticmethod
def make_json_request(url, data):
payload = json.dumps(data).encode('utf-8')
- req = compat_urllib_request.Request(url, payload)
+ req = sanitized_Request(url, payload)
req.add_header('Content-Type', 'application/json; charset=utf-8')
return req
diff --git a/youtube_dl/extractor/vevo.py b/youtube_dl/extractor/vevo.py
index 4c0de354f..571289421 100644
--- a/youtube_dl/extractor/vevo.py
+++ b/youtube_dl/extractor/vevo.py
@@ -3,13 +3,11 @@ from __future__ import unicode_literals
import re
from .common import InfoExtractor
-from ..compat import (
- compat_etree_fromstring,
- compat_urllib_request,
-)
+from ..compat import compat_etree_fromstring
from ..utils import (
ExtractorError,
int_or_none,
+ sanitized_Request,
)
@@ -73,7 +71,7 @@ class VevoIE(InfoExtractor):
_SMIL_BASE_URL = 'http://smil.lvl3.vevo.com/'
def _real_initialize(self):
- req = compat_urllib_request.Request(
+ req = sanitized_Request(
'http://www.vevo.com/auth', data=b'')
webpage = self._download_webpage(
req, None,
diff --git a/youtube_dl/extractor/viddler.py b/youtube_dl/extractor/viddler.py
index 8516a2940..40ffbad2a 100644
--- a/youtube_dl/extractor/viddler.py
+++ b/youtube_dl/extractor/viddler.py
@@ -4,9 +4,7 @@ from .common import InfoExtractor
from ..utils import (
float_or_none,
int_or_none,
-)
-from ..compat import (
- compat_urllib_request
+ sanitized_Request,
)
@@ -65,7 +63,7 @@ class ViddlerIE(InfoExtractor):
'http://api.viddler.com/api/v2/viddler.videos.getPlaybackDetails.json?video_id=%s&key=v0vhrt7bg2xq1vyxhkct' %
video_id)
headers = {'Referer': 'http://static.cdn-ec.viddler.com/js/arpeggio/v2/embed.html'}
- request = compat_urllib_request.Request(json_url, None, headers)
+ request = sanitized_Request(json_url, None, headers)
data = self._download_json(request, video_id)['video']
formats = []
diff --git a/youtube_dl/extractor/videomega.py b/youtube_dl/extractor/videomega.py
index 78ff6310a..87aca327b 100644
--- a/youtube_dl/extractor/videomega.py
+++ b/youtube_dl/extractor/videomega.py
@@ -4,7 +4,7 @@ from __future__ import unicode_literals
import re
from .common import InfoExtractor
-from ..compat import compat_urllib_request
+from ..utils import sanitized_Request
class VideoMegaIE(InfoExtractor):
@@ -30,7 +30,7 @@ class VideoMegaIE(InfoExtractor):
video_id = self._match_id(url)
iframe_url = 'http://videomega.tv/cdn.php?ref=%s' % video_id
- req = compat_urllib_request.Request(iframe_url)
+ req = sanitized_Request(iframe_url)
req.add_header('Referer', url)
req.add_header('Cookie', 'noadvtday=0')
webpage = self._download_webpage(req, video_id)
diff --git a/youtube_dl/extractor/viewster.py b/youtube_dl/extractor/viewster.py
index 7cf930d69..185b1c119 100644
--- a/youtube_dl/extractor/viewster.py
+++ b/youtube_dl/extractor/viewster.py
@@ -4,7 +4,6 @@ from __future__ import unicode_literals
from .common import InfoExtractor
from ..compat import (
compat_HTTPError,
- compat_urllib_request,
compat_urllib_parse,
compat_urllib_parse_unquote,
)
@@ -13,6 +12,7 @@ from ..utils import (
ExtractorError,
int_or_none,
parse_iso8601,
+ sanitized_Request,
HEADRequest,
)
@@ -76,7 +76,7 @@ class ViewsterIE(InfoExtractor):
_ACCEPT_HEADER = 'application/json, text/javascript, */*; q=0.01'
def _download_json(self, url, video_id, note='Downloading JSON metadata', fatal=True):
- request = compat_urllib_request.Request(url)
+ request = sanitized_Request(url)
request.add_header('Accept', self._ACCEPT_HEADER)
request.add_header('Auth-token', self._AUTH_TOKEN)
return super(ViewsterIE, self)._download_json(request, video_id, note, fatal=fatal)
diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py
index ddbd395c8..a63c23617 100644
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@@ -7,14 +7,14 @@ import hmac
import hashlib
import itertools
+from .common import InfoExtractor
from ..utils import (
ExtractorError,
int_or_none,
parse_age_limit,
parse_iso8601,
+ sanitized_Request,
)
-from ..compat import compat_urllib_request
-from .common import InfoExtractor
class VikiBaseIE(InfoExtractor):
@@ -43,7 +43,7 @@ class VikiBaseIE(InfoExtractor):
hashlib.sha1
).hexdigest()
url = self._API_URL_TEMPLATE % (query, sig)
- return compat_urllib_request.Request(
+ return sanitized_Request(
url, json.dumps(post_data).encode('utf-8')) if post_data else url
def _call_api(self, path, video_id, note, timestamp=None, post_data=None):
diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 01960b827..d99a42a9f 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -8,11 +8,11 @@ from .common import InfoExtractor
from ..compat import (
compat_str,
compat_urllib_parse,
- compat_urllib_request,
)
from ..utils import (
ExtractorError,
orderedSet,
+ sanitized_Request,
str_to_int,
unescapeHTML,
unified_strdate,
@@ -182,7 +182,7 @@ class VKIE(InfoExtractor):
'pass': password.encode('cp1251'),
})
- request = compat_urllib_request.Request(
+ request = sanitized_Request(
'https://login.vk.com/?act=login',
compat_urllib_parse.urlencode(login_form).encode('utf-8'))
login_page = self._download_webpage(
diff --git a/youtube_dl/extractor/vodlocker.py b/youtube_dl/extractor/vodlocker.py
index ccf1928b5..be0a2780f 100644
--- a/youtube_dl/extractor/vodlocker.py
+++ b/youtube_dl/extractor/vodlocker.py
@@ -2,10 +2,8 @@
from __future__ import unicode_literals
from .common import InfoExtractor
-from ..compat import (
- compat_urllib_parse,
- compat_urllib_request,
-)
+from ..compat import compat_urllib_parse
+from ..utils import sanitized_Request
class VodlockerIE(InfoExtractor):
@@ -31,7 +29,7 @@ class VodlockerIE(InfoExtractor):
if fields['op'] == 'download1':
self._sleep(3, video_id) # they do detect when requests happen too fast!
post = compat_urllib_parse.urlencode(fields)
- req = compat_urllib_request.Request(url, post)
+ req = sanitized_Request(url, post)
req.add_header('Content-type', 'application/x-www-form-urlencoded')
webpage = self._download_webpage(
req, video_id, 'Downloading video page')
diff --git a/youtube_dl/extractor/voicerepublic.py b/youtube_dl/extractor/voicerepublic.py
index 254383d6c..93d15a556 100644
--- a/youtube_dl/extractor/voicerepublic.py
+++ b/youtube_dl/extractor/voicerepublic.py
@@ -3,14 +3,12 @@ from __future__ import unicode_literals
import re
from .common import InfoExtractor
-from ..compat import (
- compat_urllib_request,
- compat_urlparse,
-)
+from ..compat import compat_urlparse
from ..utils import (
ExtractorError,
determine_ext,
int_or_none,
+ sanitized_Request,
)
@@ -37,7 +35,7 @@ class VoiceRepublicIE(InfoExtractor):
def _real_extract(self, url):
display_id = self._match_id(url)
- req = compat_urllib_request.Request(
+ req = sanitized_Request(
compat_urlparse.urljoin(url, '/talks/%s' % display_id))
# Older versions of Firefox get redirected to an "upgrade browser" page
req.add_header('User-Agent', 'youtube-dl')
diff --git a/youtube_dl/extractor/wistia.py b/youtube_dl/extractor/wistia.py
index 13a079151..fdb16d91c 100644
--- a/youtube_dl/extractor/wistia.py
+++ b/youtube_dl/extractor/wistia.py
@@ -1,8 +1,10 @@
from __future__ import unicode_literals
from .common import InfoExtractor
-from ..compat import compat_urllib_request
-from ..utils import ExtractorError
+from ..utils import (
+ ExtractorError,
+ sanitized_Request,
+)
class WistiaIE(InfoExtractor):
@@ -23,7 +25,7 @@ class WistiaIE(InfoExtractor):
def _real_extract(self, url):
video_id = self._match_id(url)
- request = compat_urllib_request.Request(self._API_URL.format(video_id))
+ request = sanitized_Request(self._API_URL.format(video_id))
request.add_header('Referer', url) # Some videos require this.
data_json = self._download_json(request, video_id)
if data_json.get('error'):
diff --git a/youtube_dl/extractor/xfileshare.py b/youtube_dl/extractor/xfileshare.py
index 952515c98..a3236e66c 100644
--- a/youtube_dl/extractor/xfileshare.py
+++ b/youtube_dl/extractor/xfileshare.py
@@ -4,14 +4,12 @@ from __future__ import unicode_literals
import re
from .common import InfoExtractor
-from ..compat import (
- compat_urllib_parse,
- compat_urllib_request,
-)
+from ..compat import compat_urllib_parse
from ..utils import (
ExtractorError,
encode_dict,
int_or_none,
+ sanitized_Request,
)
@@ -106,7 +104,7 @@ class XFileShareIE(InfoExtractor):
post = compat_urllib_parse.urlencode(encode_dict(fields))
- req = compat_urllib_request.Request(url, post)
+ req = sanitized_Request(url, post)
req.add_header('Content-type', 'application/x-www-form-urlencoded')
webpage = self._download_webpage(req, video_id, 'Downloading video page')
diff --git a/youtube_dl/extractor/xtube.py b/youtube_dl/extractor/xtube.py
index 779e4f46a..a1fe24050 100644
--- a/youtube_dl/extractor/xtube.py
+++ b/youtube_dl/extractor/xtube.py
@@ -3,12 +3,10 @@ from __future__ import unicode_literals
import re
from .common import InfoExtractor
-from ..compat import (
- compat_urllib_request,
- compat_urllib_parse_unquote,
-)
+from ..compat import compat_urllib_parse_unquote
from ..utils import (
parse_duration,
+ sanitized_Request,
str_to_int,
)
@@ -32,7 +30,7 @@ class XTubeIE(InfoExtractor):
def _real_extract(self, url):
video_id = self._match_id(url)
- req = compat_urllib_request.Request(url)
+ req = sanitized_Request(url)
req.add_header('Cookie', 'age_verified=1')
webpage = self._download_webpage(req, video_id)
diff --git a/youtube_dl/extractor/xvideos.py b/youtube_dl/extractor/xvideos.py
index 5dcf2fdd1..710ad5041 100644
--- a/youtube_dl/extractor/xvideos.py
+++ b/youtube_dl/extractor/xvideos.py
@@ -3,14 +3,12 @@ from __future__ import unicode_literals
import re
from .common import InfoExtractor
-from ..compat import (
- compat_urllib_parse_unquote,
- compat_urllib_request,
-)
+from ..compat import compat_urllib_parse_unquote
from ..utils import (
clean_html,
ExtractorError,
determine_ext,
+ sanitized_Request,
)
@@ -48,7 +46,7 @@ class XVideosIE(InfoExtractor):
'url': video_url,
}]
- android_req = compat_urllib_request.Request(url)
+ android_req = sanitized_Request(url)
android_req.add_header('User-Agent', self._ANDROID_USER_AGENT)
android_webpage = self._download_webpage(android_req, video_id, fatal=False)
diff --git a/youtube_dl/extractor/yandexmusic.py b/youtube_dl/extractor/yandexmusic.py
index 08dc81f3a..d3cc1a29f 100644
--- a/youtube_dl/extractor/yandexmusic.py
+++ b/youtube_dl/extractor/yandexmusic.py
@@ -8,11 +8,11 @@ from .common import InfoExtractor
from ..compat import (
compat_str,
compat_urllib_parse,
- compat_urllib_request,
)
from ..utils import (
int_or_none,
float_or_none,
+ sanitized_Request,
)
@@ -154,7 +154,7 @@ class YandexMusicPlaylistIE(YandexMusicPlaylistBaseIE):
if len(tracks) < len(track_ids):
present_track_ids = set([compat_str(track['id']) for track in tracks if track.get('id')])
missing_track_ids = set(map(compat_str, track_ids)) - set(present_track_ids)
- request = compat_urllib_request.Request(
+ request = sanitized_Request(
'https://music.yandex.ru/handlers/track-entries.jsx',
compat_urllib_parse.urlencode({
'entries': ','.join(missing_track_ids),
diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index 2e81d9223..69ecc837a 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -4,12 +4,13 @@ from __future__ import unicode_literals
import base64
from .common import InfoExtractor
-from ..utils import ExtractorError
-
from ..compat import (
compat_urllib_parse,
compat_ord,
- compat_urllib_request,
+)
+from ..utils import (
+ ExtractorError,
+ sanitized_Request,
)
@@ -187,7 +188,7 @@ class YoukuIE(InfoExtractor):
video_id = self._match_id(url)
def retrieve_data(req_url, note):
- req = compat_urllib_request.Request(req_url)
+ req = sanitized_Request(req_url)
cn_verification_proxy = self._downloader.params.get('cn_verification_proxy')
if cn_verification_proxy:
diff --git a/youtube_dl/extractor/youporn.py b/youtube_dl/extractor/youporn.py
index 9bf8d1eeb..dd724085a 100644
--- a/youtube_dl/extractor/youporn.py
+++ b/youtube_dl/extractor/youporn.py
@@ -3,9 +3,9 @@ from __future__ import unicode_literals
import re
from .common import InfoExtractor
-from ..compat import compat_urllib_request
from ..utils import (
int_or_none,
+ sanitized_Request,
str_to_int,
unescapeHTML,
unified_strdate,
@@ -63,7 +63,7 @@ class YouPornIE(InfoExtractor):
video_id = mobj.group('id')
display_id = mobj.group('display_id')
- request = compat_urllib_request.Request(url)
+ request = sanitized_Request(url)
request.add_header('Cookie', 'age_verified=1')
webpage = self._download_webpage(request, display_id)
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 0246050c2..cfe9eed55 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -20,7 +20,6 @@ from ..compat import (
compat_urllib_parse_unquote,
compat_urllib_parse_unquote_plus,
compat_urllib_parse_urlparse,
- compat_urllib_request,
compat_urlparse,
compat_str,
)
@@ -35,6 +34,7 @@ from ..utils import (
orderedSet,
parse_duration,
remove_start,
+ sanitized_Request,
smuggle_url,
str_to_int,
unescapeHTML,
@@ -114,7 +114,7 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
login_data = compat_urllib_parse.urlencode(encode_dict(login_form_strs)).encode('ascii')
- req = compat_urllib_request.Request(self._LOGIN_URL, login_data)
+ req = sanitized_Request(self._LOGIN_URL, login_data)
login_results = self._download_webpage(
req, None,
note='Logging in', errnote='unable to log in', fatal=False)
@@ -147,7 +147,7 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
tfa_data = compat_urllib_parse.urlencode(encode_dict(tfa_form_strs)).encode('ascii')
- tfa_req = compat_urllib_request.Request(self._TWOFACTOR_URL, tfa_data)
+ tfa_req = sanitized_Request(self._TWOFACTOR_URL, tfa_data)
tfa_results = self._download_webpage(
tfa_req, None,
note='Submitting TFA code', errnote='unable to submit tfa', fatal=False)
From 20e98bf6c0df1dc10651d5b835d42385db9bbdc7 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 23 Nov 2015 18:07:58 +0100
Subject: [PATCH 367/415] release 2015.11.23
---
youtube_dl/version.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 2baf1ac42..3e2c0b4ac 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
from __future__ import unicode_literals
-__version__ = '2015.11.21'
+__version__ = '2015.11.23'
From 032f2f260f7ef7a04d1499b26b97c1643fe993b0 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 24 Nov 2015 03:38:46 +0100
Subject: [PATCH 368/415] README: Document which other programs may be helpful
(Fixes #7621)
---
README.md | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/README.md b/README.md
index b286651cd..957059f8c 100644
--- a/README.md
+++ b/README.md
@@ -534,6 +534,12 @@ Most people asking this question are not aware that youtube-dl now defaults to d
Apparently YouTube requires you to pass a CAPTCHA test if you download too much. We're [considering to provide a way to let you solve the CAPTCHA](https://github.com/rg3/youtube-dl/issues/154), but at the moment, your best course of action is pointing a webbrowser to the youtube URL, solving the CAPTCHA, and restart youtube-dl.
+### Do I need any other programs?
+
+youtube-dl works fine on its own on most sites. However, if you want to convert video/audio, you'll need [avconv](https://libav.org/) or [ffmpeg](https://www.ffmpeg.org/). On some sites - most notably YouTube - videos can be retrieved in a higher quality format without sound. youtube-dl will detect whether avconv/ffmpeg is present and automatically pick the best option.
+
+Some videos or video formats can also be only downloaded when [rtmpdump](https://rtmpdump.mplayerhq.hu/) is installed.
+
### I have downloaded a video but how can I play it?
Once the video is fully downloaded, use any video player, such as [vlc](http://www.videolan.org) or [mplayer](http://www.mplayerhq.hu/).
From 4c7d816dd70a7b10dc096d815da55d41c46f9b59 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 24 Nov 2015 07:45:02 +0100
Subject: [PATCH 369/415] [jsinterp] Adapt to updated YouTube code generation
(Fixes #7623, fixes #7624, fixes #7625, fixes #7626)
---
youtube_dl/jsinterp.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index 9bc855144..2191e8b89 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -214,7 +214,7 @@ class JSInterpreter(object):
obj = {}
obj_m = re.search(
(r'(?:var\s+)?%s\s*=\s*\{' % re.escape(objname)) +
- r'\s*(?P<fields>([a-zA-Z$0-9]+\s*:\s*function\(.*?\)\s*\{.*?\})*)' +
+ r'\s*(?P<fields>([a-zA-Z$0-9]+\s*:\s*function\(.*?\)\s*\{.*?\}(?:,\s*)?)*)' +
r'\}\s*;',
self.code)
fields = obj_m.group('fields')
From ba7a92b0ced8cce0c800e287777f1654b728f17c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 24 Nov 2015 07:46:38 +0100
Subject: [PATCH 370/415] release 2015.11.24
---
youtube_dl/version.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 3e2c0b4ac..5ecff39fb 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
from __future__ import unicode_literals
-__version__ = '2015.11.23'
+__version__ = '2015.11.24'
From 7689413e42334ff79e671a4c90869077e9f9835b Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Tue, 24 Nov 2015 23:06:21 +0600
Subject: [PATCH 371/415] [README.md] Mention mplayer and mpv in "other
programs" question
---
README.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/README.md b/README.md
index 957059f8c..b85e08e78 100644
--- a/README.md
+++ b/README.md
@@ -538,7 +538,7 @@ Apparently YouTube requires you to pass a CAPTCHA test if you download too much.
youtube-dl works fine on its own on most sites. However, if you want to convert video/audio, you'll need [avconv](https://libav.org/) or [ffmpeg](https://www.ffmpeg.org/). On some sites - most notably YouTube - videos can be retrieved in a higher quality format without sound. youtube-dl will detect whether avconv/ffmpeg is present and automatically pick the best option.
-Some videos or video formats can also be only downloaded when [rtmpdump](https://rtmpdump.mplayerhq.hu/) is installed.
+Videos or video formats streamed via RTMP protocol can only be downloaded when [rtmpdump](https://rtmpdump.mplayerhq.hu/) is installed. Downloading MMS and RTSP videos requires either [mplayer](http://mplayerhq.hu/) or [mpv](https://mpv.io/) to be installed.
### I have downloaded a video but how can I play it?
From 9bff48a0e7128b4d274a5dab4aaac8da0c900ede Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 27 Nov 2015 21:24:39 +0800
Subject: [PATCH 372/415] [options] Clarify --list-formats needs videos (closes
#7669)
---
youtube_dl/options.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 079fe7e8a..359e8d300 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -338,7 +338,7 @@ def parseOpts(overrideArguments=None):
video_format.add_option(
'-F', '--list-formats',
action='store_true', dest='listformats',
- help='List all available formats')
+ help='List all available formats of specified videos')
video_format.add_option(
'--youtube-include-dash-manifest',
action='store_true', dest='youtube_include_dash_manifest', default=True,
From 9d0e3668804fe91e77275e6bd859cbd0d784ab33 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 27 Nov 2015 21:37:45 +0800
Subject: [PATCH 373/415] [downloader/hls] Remove Accept-encoding from headers
passed to ffmpeg
Fails for Youtube Gaming live streams (#7671)
---
youtube_dl/downloader/hls.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index 9a83a73dd..92765a3f9 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -35,7 +35,7 @@ class HlsFD(FileDownloader):
# [http @ 00000000003d2fa0] No trailing CRLF found in HTTP header.
args += [
'-headers',
- ''.join('%s: %s\r\n' % (key, val) for key, val in info_dict['http_headers'].items())]
+ ''.join('%s: %s\r\n' % (key, val) for key, val in info_dict['http_headers'].items() if key.lower() != 'accept-encoding')]
args += ['-i', url, '-f', 'mp4', '-c', 'copy', '-bsf:a', 'aac_adtstoasc']
From bb6ac8369858e79b04e30e6ea25016500083628f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 27 Nov 2015 16:32:51 +0100
Subject: [PATCH 374/415] release 2015.11.27
---
README.md | 3 ++-
youtube_dl/version.py | 2 +-
2 files changed, 3 insertions(+), 2 deletions(-)
diff --git a/README.md b/README.md
index b85e08e78..df419abe8 100644
--- a/README.md
+++ b/README.md
@@ -319,7 +319,8 @@ which means you can modify it, redistribute it or use it however you like.
--all-formats Download all available video formats
--prefer-free-formats Prefer free video formats unless a specific
one is requested
- -F, --list-formats List all available formats
+ -F, --list-formats List all available formats of specified
+ videos
--youtube-skip-dash-manifest Do not download the DASH manifests and
related data on YouTube videos
--merge-output-format FORMAT If a merge is required (e.g.
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 5ecff39fb..288b76e28 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
from __future__ import unicode_literals
-__version__ = '2015.11.24'
+__version__ = '2015.11.27'
From 78a55d7a28662a65cba36c48eb69ecc4b150b336 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 27 Nov 2015 16:39:59 +0100
Subject: [PATCH 375/415] release 2015.11.27.1
---
youtube_dl/version.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 288b76e28..bd0de9f53 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
from __future__ import unicode_literals
-__version__ = '2015.11.27'
+__version__ = '2015.11.27.1'
From 313dfc45f500db3acc348d58f431197ce7c153a3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Lalinsk=C3=BD?= <lukas@oxygene.sk>
Date: Sat, 28 Nov 2015 01:07:07 +0100
Subject: [PATCH 376/415] [youtube] Ignore yt:stretch with zero width/height
---
youtube_dl/extractor/youtube.py | 27 +++++++++++++++++++++++----
1 file changed, 23 insertions(+), 4 deletions(-)
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index cfe9eed55..726b5ba0a 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -714,6 +714,22 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
'url': 'https://www.youtube.com/watch?v=Ms7iBXnlUO8',
'only_matching': True,
},
+ {
+ # Video with yt:stretch=17:0
+ 'url': 'https://www.youtube.com/watch?v=Q39EVAstoRM',
+ 'info_dict': {
+ 'id': 'Q39EVAstoRM',
+ 'ext': 'mp4',
+ 'title': 'Clash Of Clans#14 Dicas De Ataque Para CV 4',
+ 'description': 'md5:ee18a25c350637c8faff806845bddee9',
+ 'upload_date': '20151107',
+ 'uploader_id': 'UCCr7TALkRbo3EtFzETQF1LA',
+ 'uploader': 'CH GAMER DROID',
+ },
+ 'params': {
+ 'skip_download': True,
+ },
+ },
]
def __init__(self, *args, **kwargs):
@@ -1496,10 +1512,13 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
r'<meta\s+property="og:video:tag".*?content="yt:stretch=(?P<w>[0-9]+):(?P<h>[0-9]+)">',
video_webpage)
if stretched_m:
- ratio = float(stretched_m.group('w')) / float(stretched_m.group('h'))
- for f in formats:
- if f.get('vcodec') != 'none':
- f['stretched_ratio'] = ratio
+ w = float(stretched_m.group('w'))
+ h = float(stretched_m.group('h'))
+ if w > 0 and h > 0:
+ ratio = float(stretched_m.group('w')) / float(stretched_m.group('h'))
+ for f in formats:
+ if f.get('vcodec') != 'none':
+ f['stretched_ratio'] = ratio
self._sort_formats(formats)
From 7ac40086f5f9fdc01c92d1b76acb742c661cf0ff Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 28 Nov 2015 08:44:13 +0600
Subject: [PATCH 377/415] [dbtv] Expand _VALID_URL (Closes #7645)
---
youtube_dl/extractor/dbtv.py | 14 ++++++++++----
1 file changed, 10 insertions(+), 4 deletions(-)
diff --git a/youtube_dl/extractor/dbtv.py b/youtube_dl/extractor/dbtv.py
index 212217625..133cdc50b 100644
--- a/youtube_dl/extractor/dbtv.py
+++ b/youtube_dl/extractor/dbtv.py
@@ -13,8 +13,8 @@ from ..utils import (
class DBTVIE(InfoExtractor):
- _VALID_URL = r'http://dbtv\.no/(?P<id>[0-9]+)#(?P<display_id>.+)'
- _TEST = {
+ _VALID_URL = r'https?://(?:www\.)?dbtv\.no/(?:(?:lazyplayer|player)/)?(?P<id>[0-9]+)(?:#(?P<display_id>.+))?'
+ _TESTS = [{
'url': 'http://dbtv.no/3649835190001#Skulle_teste_ut_fornøyelsespark,_men_kollegaen_var_bare_opptatt_av_bikinikroppen',
'md5': 'b89953ed25dacb6edb3ef6c6f430f8bc',
'info_dict': {
@@ -30,12 +30,18 @@ class DBTVIE(InfoExtractor):
'view_count': int,
'categories': list,
}
- }
+ }, {
+ 'url': 'http://dbtv.no/3649835190001',
+ 'only_matching': True,
+ }, {
+ 'url': 'http://www.dbtv.no/lazyplayer/4631135248001',
+ 'only_matching': True,
+ }]
def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url)
video_id = mobj.group('id')
- display_id = mobj.group('display_id')
+ display_id = mobj.group('display_id') or video_id
data = self._download_json(
'http://api.dbtv.no/discovery/%s' % video_id, display_id)
From 4b3fbafdd261405c12e044a59d20554838c9c4c2 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 28 Nov 2015 14:14:20 +0800
Subject: [PATCH 378/415] [options] Changed wording for --list-formats
As proposed by @dstftw at 9bff48a0e7128b4d274a5dab4aaac8da0c900ede
---
youtube_dl/options.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 359e8d300..c46e136bf 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -338,7 +338,7 @@ def parseOpts(overrideArguments=None):
video_format.add_option(
'-F', '--list-formats',
action='store_true', dest='listformats',
- help='List all available formats of specified videos')
+ help='List all available formats of requested videos')
video_format.add_option(
'--youtube-include-dash-manifest',
action='store_true', dest='youtube_include_dash_manifest', default=True,
From 41f24c321d04108a32f457d5d5445f2cfce705a8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Lalinsk=C3=BD?= <lukas@oxygene.sk>
Date: Sat, 28 Nov 2015 08:16:46 +0100
Subject: [PATCH 379/415] [youtube] Use the existing `w` and `h` variables
---
youtube_dl/extractor/youtube.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 726b5ba0a..9da8d4bc5 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1515,7 +1515,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
w = float(stretched_m.group('w'))
h = float(stretched_m.group('h'))
if w > 0 and h > 0:
- ratio = float(stretched_m.group('w')) / float(stretched_m.group('h'))
+ ratio = w / h
for f in formats:
if f.get('vcodec') != 'none':
f['stretched_ratio'] = ratio
From 0cc71785461d273294f2ab96d50854172ba29c04 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 28 Nov 2015 11:48:18 +0100
Subject: [PATCH 380/415] [skynewsarabia] Add new extractor
---
youtube_dl/extractor/__init__.py | 4 +
youtube_dl/extractor/skynewsarabia.py | 115 ++++++++++++++++++++++++++
2 files changed, 119 insertions(+)
create mode 100644 youtube_dl/extractor/skynewsarabia.py
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 947b83683..d981acb8e 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -554,6 +554,10 @@ from .shahid import ShahidIE
from .shared import SharedIE
from .sharesix import ShareSixIE
from .sina import SinaIE
+from .skynewsarabia import (
+ SkyNewsArabiaIE,
+ SkyNewsArabiaArticleIE,
+)
from .slideshare import SlideshareIE
from .slutload import SlutloadIE
from .smotri import (
diff --git a/youtube_dl/extractor/skynewsarabia.py b/youtube_dl/extractor/skynewsarabia.py
new file mode 100644
index 000000000..3499b8cd5
--- /dev/null
+++ b/youtube_dl/extractor/skynewsarabia.py
@@ -0,0 +1,115 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..compat import compat_str
+from ..utils import (
+ parse_iso8601,
+ parse_duration,
+)
+
+
+class SkyNewArabiaBaseIE(InfoExtractor):
+ _IMAGE_BASE_URL = 'http://www.skynewsarabia.com/web/images'
+
+ def _call_api(self, path, value):
+ return self._download_json('http://api.skynewsarabia.com/web/rest/v2/%s/%s.json' % (path, value), value)
+
+ def _get_limelight_media_id(self, url):
+ return self._search_regex(r'/media/[^/]+/([a-z0-9]{32})', url, 'limelight media id')
+
+ def _get_image_url(self, image_path_template, width='1600', height='1200'):
+ return self._IMAGE_BASE_URL + image_path_template.format(width=width, height=height)
+
+ def _extract_video_info(self, video_data):
+ video_id = compat_str(video_data['id'])
+ return {
+ '_type': 'url_transparent',
+ 'url': 'limelight:media:%s' % self._get_limelight_media_id(video_data['videoUrl'][0]['url']),
+ 'id': video_id,
+ 'title': video_data['headline'],
+ 'description': video_data.get('summary'),
+ 'thumbnail': self._get_image_url(video_data['mediaAsset']['imageUrl']),
+ 'timestamp': parse_iso8601(video_data.get('date')),
+ 'duration': parse_duration(video_data.get('runTime')),
+ 'tags': video_data.get('tags'),
+ 'categories': [video_data.get('topicTitle')],
+ 'webpage_url': 'http://www.skynewsarabia.com/web/video/%s' % video_id,
+ 'ie_key': 'LimelightMedia',
+ }
+
+
+class SkyNewsArabiaIE(SkyNewArabiaBaseIE):
+ _IE_NAME = 'skynewsarabia:video'
+ _VALID_URL = r'https?://(?:www\.)?skynewsarabia\.com/web/video/(?P<id>[0-9]+)'
+ _TEST = {
+ 'url': 'http://www.skynewsarabia.com/web/video/794902/%D9%86%D8%B5%D9%81-%D9%85%D9%84%D9%8A%D9%88%D9%86-%D9%85%D8%B5%D8%A8%D8%A7%D8%AD-%D8%B4%D8%AC%D8%B1%D8%A9-%D9%83%D8%B1%D9%8A%D8%B3%D9%85%D8%A7%D8%B3',
+ 'info_dict': {
+ 'id': '794902',
+ 'ext': 'flv',
+ 'title': 'نصف مليون مصباح على شجرة كريسماس',
+ 'description': 'md5:22f1b27f0850eeb10c7e59b1f16eb7c6',
+ 'upload_date': '20151128',
+ 'timestamp': 1448697198,
+ 'duration': 2119,
+ },
+ 'params': {
+ # rtmp download
+ 'skip_download': True,
+ },
+ }
+
+ def _real_extract(self, url):
+ video_id = self._match_id(url)
+ video_data = self._call_api('video', video_id)
+ return self._extract_video_info(video_data)
+
+
+class SkyNewsArabiaArticleIE(SkyNewArabiaBaseIE):
+ _IE_NAME = 'skynewsarabia:video'
+ _VALID_URL = r'https?://(?:www\.)?skynewsarabia\.com/web/article/(?P<id>[0-9]+)'
+ _TESTS = [{
+ 'url': 'http://www.skynewsarabia.com/web/article/794549/%D8%A7%D9%94%D8%AD%D8%AF%D8%A7%D8%AB-%D8%A7%D9%84%D8%B4%D8%B1%D9%82-%D8%A7%D9%84%D8%A7%D9%94%D9%88%D8%B3%D8%B7-%D8%AE%D8%B1%D9%8A%D8%B7%D8%A9-%D8%A7%D9%84%D8%A7%D9%94%D9%84%D8%B9%D8%A7%D8%A8-%D8%A7%D9%84%D8%B0%D9%83%D9%8A%D8%A9',
+ 'info_dict': {
+ 'id': '794549',
+ 'ext': 'flv',
+ 'title': 'بالفيديو.. ألعاب ذكية تحاكي واقع المنطقة',
+ 'description': 'md5:0c373d29919a851e080ee4edd0c5d97f',
+ 'upload_date': '20151126',
+ 'timestamp': 1448559336,
+ 'duration': 281.6,
+ },
+ 'params': {
+ # rtmp download
+ 'skip_download': True,
+ },
+ }, {
+ 'url': 'http://www.skynewsarabia.com/web/article/794844/%D8%A7%D8%B3%D8%AA%D9%87%D8%AF%D8%A7%D9%81-%D9%82%D9%88%D8%A7%D8%B1%D8%A8-%D8%A7%D9%94%D8%B3%D9%84%D8%AD%D8%A9-%D9%84%D9%85%D9%8A%D9%84%D9%8A%D8%B4%D9%8A%D8%A7%D8%AA-%D8%A7%D9%84%D8%AD%D9%88%D8%AB%D9%8A-%D9%88%D8%B5%D8%A7%D9%84%D8%AD',
+ 'info_dict': {
+ 'id': '794844',
+ 'title': 'إحباط تهريب أسلحة لميليشيات الحوثي وصالح بجنوب اليمن',
+ 'description': 'md5:5c927b8b2e805796e7f693538d96fc7e',
+ },
+ 'playlist_mincount': 2,
+ }]
+
+ def _real_extract(self, url):
+ article_id = self._match_id(url)
+ article_data = self._call_api('article', article_id)
+ media_asset = article_data['mediaAsset']
+ if media_asset['type'] == 'VIDEO':
+ return {
+ '_type': 'url_transparent',
+ 'url': 'limelight:media:%s' % self._get_limelight_media_id(media_asset['videoUrl'][0]['url']),
+ 'id': article_id,
+ 'title': article_data['headline'],
+ 'description': article_data.get('summary'),
+ 'thumbnail': self._get_image_url(media_asset['imageUrl']),
+ 'timestamp': parse_iso8601(article_data.get('date')),
+ 'tags': article_data.get('tags'),
+ 'categories': [article_data.get('topicTitle')],
+ 'webpage_url': url,
+ 'ie_key': 'LimelightMedia',
+ }
+ entries = [self._extract_video_info(item) for item in article_data.get('inlineItems', []) if item['type'] == 'VIDEO']
+ return self.playlist_result(entries, article_id, article_data['headline'], article_data.get('summary'))
From 4975650e002f3b9c20cc54940684c39c8f68e786 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 28 Nov 2015 12:20:39 +0100
Subject: [PATCH 381/415] [skynewsarabia] fix IE_NAME
---
youtube_dl/extractor/skynewsarabia.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/youtube_dl/extractor/skynewsarabia.py b/youtube_dl/extractor/skynewsarabia.py
index 3499b8cd5..d1c737020 100644
--- a/youtube_dl/extractor/skynewsarabia.py
+++ b/youtube_dl/extractor/skynewsarabia.py
@@ -40,7 +40,7 @@ class SkyNewArabiaBaseIE(InfoExtractor):
class SkyNewsArabiaIE(SkyNewArabiaBaseIE):
- _IE_NAME = 'skynewsarabia:video'
+ IE_NAME = 'skynewsarabia:video'
_VALID_URL = r'https?://(?:www\.)?skynewsarabia\.com/web/video/(?P<id>[0-9]+)'
_TEST = {
'url': 'http://www.skynewsarabia.com/web/video/794902/%D9%86%D8%B5%D9%81-%D9%85%D9%84%D9%8A%D9%88%D9%86-%D9%85%D8%B5%D8%A8%D8%A7%D8%AD-%D8%B4%D8%AC%D8%B1%D8%A9-%D9%83%D8%B1%D9%8A%D8%B3%D9%85%D8%A7%D8%B3',
@@ -66,7 +66,7 @@ class SkyNewsArabiaIE(SkyNewArabiaBaseIE):
class SkyNewsArabiaArticleIE(SkyNewArabiaBaseIE):
- _IE_NAME = 'skynewsarabia:video'
+ IE_NAME = 'skynewsarabia:video'
_VALID_URL = r'https?://(?:www\.)?skynewsarabia\.com/web/article/(?P<id>[0-9]+)'
_TESTS = [{
'url': 'http://www.skynewsarabia.com/web/article/794549/%D8%A7%D9%94%D8%AD%D8%AF%D8%A7%D8%AB-%D8%A7%D9%84%D8%B4%D8%B1%D9%82-%D8%A7%D9%84%D8%A7%D9%94%D9%88%D8%B3%D8%B7-%D8%AE%D8%B1%D9%8A%D8%B7%D8%A9-%D8%A7%D9%84%D8%A7%D9%94%D9%84%D8%B9%D8%A7%D8%A8-%D8%A7%D9%84%D8%B0%D9%83%D9%8A%D8%A9',
From 5faf9fed7e1c7922578467cfd48db5867ef9b91b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 28 Nov 2015 18:50:21 +0600
Subject: [PATCH 382/415] [youtube] Clarify rationale for yt:stretch validation
---
youtube_dl/extractor/youtube.py | 2 ++
1 file changed, 2 insertions(+)
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 9da8d4bc5..1c2420a33 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1514,6 +1514,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
if stretched_m:
w = float(stretched_m.group('w'))
h = float(stretched_m.group('h'))
+ # yt:stretch may hold invalid ratio data (e.g. for Q39EVAstoRM ratio is 17:0).
+ # We will only process correct ratios.
if w > 0 and h > 0:
ratio = w / h
for f in formats:
From 9945c4994c8cdbfa9971df942332f2c75bc2734d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 28 Nov 2015 20:21:03 +0600
Subject: [PATCH 383/415] Credit @reiv for soundcloud:search
---
AUTHORS | 1 +
1 file changed, 1 insertion(+)
diff --git a/AUTHORS b/AUTHORS
index f465d20ed..cdb56de3b 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -146,3 +146,4 @@ Lukáš Lalinský
Qijiang Fan
Rémy Léone
Marco Ferragina
+reiv
From 7ad4258add964b2604ada611d380b5594126d85c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 28 Nov 2015 22:39:36 +0600
Subject: [PATCH 384/415] [bloomberg] Relax _VALID_URL even more (Closes #7685)
---
youtube_dl/extractor/bloomberg.py | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/bloomberg.py b/youtube_dl/extractor/bloomberg.py
index 11ace91dd..8b2f94211 100644
--- a/youtube_dl/extractor/bloomberg.py
+++ b/youtube_dl/extractor/bloomberg.py
@@ -6,7 +6,7 @@ from .common import InfoExtractor
class BloombergIE(InfoExtractor):
- _VALID_URL = r'https?://www\.bloomberg\.com/news/[^/]+/[^/]+/(?P<id>[^/?#]+)'
+ _VALID_URL = r'https?://(?:www\.)?bloomberg\.com/(?:[^/]+/)*(?P<id>[^/?#]+)'
_TESTS = [{
'url': 'http://www.bloomberg.com/news/videos/b/aaeae121-5949-481e-a1ce-4562db6f5df2',
@@ -20,6 +20,9 @@ class BloombergIE(InfoExtractor):
}, {
'url': 'http://www.bloomberg.com/news/articles/2015-11-12/five-strange-things-that-have-been-happening-in-financial-markets',
'only_matching': True,
+ }, {
+ 'url': 'http://www.bloomberg.com/politics/videos/2015-11-25/karl-rove-on-jeb-bush-s-struggles-stopping-trump',
+ 'only_matching': True,
}]
def _real_extract(self, url):
From 9a4f12be98a0d59de9b5da65d2cc24b04fe26929 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 28 Nov 2015 22:40:29 +0600
Subject: [PATCH 385/415] [bloomberg] Modernize
---
youtube_dl/extractor/bloomberg.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/bloomberg.py b/youtube_dl/extractor/bloomberg.py
index 8b2f94211..b6197d3f1 100644
--- a/youtube_dl/extractor/bloomberg.py
+++ b/youtube_dl/extractor/bloomberg.py
@@ -35,7 +35,7 @@ class BloombergIE(InfoExtractor):
'http://www.bloomberg.com/api/embed?id=%s' % video_id, video_id)
formats = []
for stream in embed_info['streams']:
- if stream["muxing_format"] == "TS":
+ if stream['muxing_format'] == 'TS':
formats.extend(self._extract_m3u8_formats(stream['url'], video_id))
else:
formats.extend(self._extract_f4m_formats(stream['url'], video_id))
From 4191fdf147729728ca9c487f25ca66bf3d037cd5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 28 Nov 2015 22:41:39 +0600
Subject: [PATCH 386/415] [bloomberg] Improve video id regex
---
youtube_dl/extractor/bloomberg.py | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/bloomberg.py b/youtube_dl/extractor/bloomberg.py
index b6197d3f1..c28ed0c79 100644
--- a/youtube_dl/extractor/bloomberg.py
+++ b/youtube_dl/extractor/bloomberg.py
@@ -28,7 +28,9 @@ class BloombergIE(InfoExtractor):
def _real_extract(self, url):
name = self._match_id(url)
webpage = self._download_webpage(url, name)
- video_id = self._search_regex(r'"bmmrId":"(.+?)"', webpage, 'id')
+ video_id = self._search_regex(
+ r'["\']bmmrId["\']\s*:\s*(["\'])(?P<url>.+?)\1',
+ webpage, 'id', group='url')
title = re.sub(': Video$', '', self._og_search_title(webpage))
embed_info = self._download_json(
From b7faebbac87430106d0249a09fcca55a5ab112de Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 28 Nov 2015 22:45:19 +0600
Subject: [PATCH 387/415] [bloomberg] Improve formats extraction
---
youtube_dl/extractor/bloomberg.py | 13 +++++++++++--
1 file changed, 11 insertions(+), 2 deletions(-)
diff --git a/youtube_dl/extractor/bloomberg.py b/youtube_dl/extractor/bloomberg.py
index c28ed0c79..ebeef8f2a 100644
--- a/youtube_dl/extractor/bloomberg.py
+++ b/youtube_dl/extractor/bloomberg.py
@@ -37,10 +37,19 @@ class BloombergIE(InfoExtractor):
'http://www.bloomberg.com/api/embed?id=%s' % video_id, video_id)
formats = []
for stream in embed_info['streams']:
+ stream_url = stream.get('url')
+ if not stream_url:
+ continue
if stream['muxing_format'] == 'TS':
- formats.extend(self._extract_m3u8_formats(stream['url'], video_id))
+ m3u8_formats = self._extract_m3u8_formats(
+ stream_url, video_id, 'mp4', m3u8_id='hls', fatal=False)
+ if m3u8_formats:
+ formats.extend(m3u8_formats)
else:
- formats.extend(self._extract_f4m_formats(stream['url'], video_id))
+ f4m_formats = self._extract_f4m_formats(
+ stream_url, video_id, f4m_id='hds', fatal=False)
+ if f4m_formats:
+ formats.extend(f4m_formats)
self._sort_formats(formats)
return {
From f4c7ef98624bd407a8f8f91215cb95a2c5db9472 Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 28 Nov 2015 18:20:44 +0100
Subject: [PATCH 388/415] [skynewsarabia] return empty categories array if
there is no topic
---
youtube_dl/extractor/skynewsarabia.py | 10 ++++++----
1 file changed, 6 insertions(+), 4 deletions(-)
diff --git a/youtube_dl/extractor/skynewsarabia.py b/youtube_dl/extractor/skynewsarabia.py
index d1c737020..f09fee102 100644
--- a/youtube_dl/extractor/skynewsarabia.py
+++ b/youtube_dl/extractor/skynewsarabia.py
@@ -23,6 +23,7 @@ class SkyNewArabiaBaseIE(InfoExtractor):
def _extract_video_info(self, video_data):
video_id = compat_str(video_data['id'])
+ topic = video_data.get('topicTitle')
return {
'_type': 'url_transparent',
'url': 'limelight:media:%s' % self._get_limelight_media_id(video_data['videoUrl'][0]['url']),
@@ -32,8 +33,8 @@ class SkyNewArabiaBaseIE(InfoExtractor):
'thumbnail': self._get_image_url(video_data['mediaAsset']['imageUrl']),
'timestamp': parse_iso8601(video_data.get('date')),
'duration': parse_duration(video_data.get('runTime')),
- 'tags': video_data.get('tags'),
- 'categories': [video_data.get('topicTitle')],
+ 'tags': video_data.get('tags', []),
+ 'categories': [topic] if topic else [],
'webpage_url': 'http://www.skynewsarabia.com/web/video/%s' % video_id,
'ie_key': 'LimelightMedia',
}
@@ -98,6 +99,7 @@ class SkyNewsArabiaArticleIE(SkyNewArabiaBaseIE):
article_data = self._call_api('article', article_id)
media_asset = article_data['mediaAsset']
if media_asset['type'] == 'VIDEO':
+ topic = article_data.get('topicTitle')
return {
'_type': 'url_transparent',
'url': 'limelight:media:%s' % self._get_limelight_media_id(media_asset['videoUrl'][0]['url']),
@@ -106,8 +108,8 @@ class SkyNewsArabiaArticleIE(SkyNewArabiaBaseIE):
'description': article_data.get('summary'),
'thumbnail': self._get_image_url(media_asset['imageUrl']),
'timestamp': parse_iso8601(article_data.get('date')),
- 'tags': article_data.get('tags'),
- 'categories': [article_data.get('topicTitle')],
+ 'tags': article_data.get('tags', []),
+ 'categories': [topic] if topic else [],
'webpage_url': url,
'ie_key': 'LimelightMedia',
}
From 2a776f978849e0c66f70133747e7fd244f516f7f Mon Sep 17 00:00:00 2001
From: remitamine <remitamine@gmail.com>
Date: Sat, 28 Nov 2015 20:22:31 +0100
Subject: [PATCH 389/415] [cspan] change into a function
---
youtube_dl/extractor/cspan.py | 20 ++++++++++----------
1 file changed, 10 insertions(+), 10 deletions(-)
diff --git a/youtube_dl/extractor/cspan.py b/youtube_dl/extractor/cspan.py
index 388460a32..7b685d157 100644
--- a/youtube_dl/extractor/cspan.py
+++ b/youtube_dl/extractor/cspan.py
@@ -56,9 +56,6 @@ class CSpanIE(InfoExtractor):
}
}]
- def get_text_attr(self, d, attr):
- return d.get(attr, {}).get('#text')
-
def _real_extract(self, url):
video_id = self._match_id(url)
webpage = self._download_webpage(url, video_id)
@@ -74,11 +71,14 @@ class CSpanIE(InfoExtractor):
surl = smuggle_url(senate_isvp_url, {'force_title': title})
return self.url_result(surl, 'SenateISVP', video_id, title)
+ def get_text_attr(d, attr):
+ return d.get(attr, {}).get('#text')
+
data = self._download_json(
'http://www.c-span.org/assets/player/ajax-player.php?os=android&html5=%s&id=%s' % (video_type, video_id),
video_id)['video']
if data['@status'] != 'Success':
- raise ExtractorError('%s said: %s' % (self.IE_NAME, self.get_text_attr(data, 'error')), expected=True)
+ raise ExtractorError('%s said: %s' % (self.IE_NAME, get_text_attr(data, 'error')), expected=True)
doc = self._download_xml(
'http://www.c-span.org/common/services/flashXml.php?%sid=%s' % (video_type, video_id),
@@ -90,17 +90,17 @@ class CSpanIE(InfoExtractor):
thumbnail = find_xpath_attr(doc, './/string', 'name', 'poster').text
files = data['files']
- capfile = self.get_text_attr(data, 'capfile')
+ capfile = get_text_attr(data, 'capfile')
entries = []
for partnum, f in enumerate(files):
formats = []
for quality in f['qualities']:
formats.append({
- 'format_id': '%s-%sp' % (self.get_text_attr(quality, 'bitrate'), self.get_text_attr(quality, 'height')),
- 'url': unescapeHTML(self.get_text_attr(quality, 'file')),
- 'height': int_or_none(self.get_text_attr(quality, 'height')),
- 'tbr': int_or_none(self.get_text_attr(quality, 'bitrate')),
+ 'format_id': '%s-%sp' % (get_text_attr(quality, 'bitrate'), get_text_attr(quality, 'height')),
+ 'url': unescapeHTML(get_text_attr(quality, 'file')),
+ 'height': int_or_none(get_text_attr(quality, 'height')),
+ 'tbr': int_or_none(get_text_attr(quality, 'bitrate')),
})
self._sort_formats(formats)
entries.append({
@@ -111,7 +111,7 @@ class CSpanIE(InfoExtractor):
'formats': formats,
'description': description,
'thumbnail': thumbnail,
- 'duration': int_or_none(self.get_text_attr(f, 'length')),
+ 'duration': int_or_none(get_text_attr(f, 'length')),
'subtitles': {
'en': [{
'url': capfile,
From 87f0e62d94e0486598d123e26db3173e6f1d18e6 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 29 Nov 2015 12:42:50 +0800
Subject: [PATCH 390/415] [utils] Separate codes for handling Youtubedl-*
headers
---
youtube_dl/utils.py | 15 +++++++++++----
1 file changed, 11 insertions(+), 4 deletions(-)
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index d7b737e21..653a49055 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -663,6 +663,15 @@ def _create_http_connection(ydl_handler, http_class, is_https, *args, **kwargs):
return hc
+def handle_youtubedl_headers(headers):
+ if 'Youtubedl-no-compression' in headers:
+ filtered_headers = dict((k, v) for k, v in headers.items() if k.lower() != 'accept-encoding')
+ del filtered_headers['Youtubedl-no-compression']
+ return filtered_headers
+
+ return headers
+
+
class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
"""Handler for HTTP requests and responses.
@@ -731,10 +740,8 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
# The dict keys are capitalized because of this bug by urllib
if h.capitalize() not in req.headers:
req.add_header(h, v)
- if 'Youtubedl-no-compression' in req.headers:
- if 'Accept-encoding' in req.headers:
- del req.headers['Accept-encoding']
- del req.headers['Youtubedl-no-compression']
+
+ req.headers = handle_youtubedl_headers(req.headers)
if sys.version_info < (2, 7) and '#' in req.get_full_url():
# Python 2.6 is brain-dead when it comes to fragments
From 94e8c8047353eb541fa20dcf55819cc6ee6d3303 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 29 Nov 2015 12:43:59 +0800
Subject: [PATCH 391/415] [downloader/hls] Respect Youtubedl-* headers
---
youtube_dl/downloader/hls.py | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index 92765a3f9..b5a3e1167 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -13,6 +13,7 @@ from ..utils import (
encodeArgument,
encodeFilename,
sanitize_open,
+ handle_youtubedl_headers,
)
@@ -33,9 +34,10 @@ class HlsFD(FileDownloader):
if info_dict['http_headers'] and re.match(r'^https?://', url):
# Trailing \r\n after each HTTP header is important to prevent warning from ffmpeg/avconv:
# [http @ 00000000003d2fa0] No trailing CRLF found in HTTP header.
+ headers = handle_youtubedl_headers(info_dict['http_headers'])
args += [
'-headers',
- ''.join('%s: %s\r\n' % (key, val) for key, val in info_dict['http_headers'].items() if key.lower() != 'accept-encoding')]
+ ''.join('%s: %s\r\n' % (key, val) for key, val in headers.items())]
args += ['-i', url, '-f', 'mp4', '-c', 'copy', '-bsf:a', 'aac_adtstoasc']
From ac5a69af45307b583a9a6088abe5939bec18d562 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 29 Nov 2015 12:44:24 +0800
Subject: [PATCH 392/415] [youtube] Disable compression for live streams
---
youtube_dl/extractor/youtube.py | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 1c2420a33..52f4fe36d 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1475,6 +1475,11 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
manifest_url = video_info['hlsvp'][0]
url_map = self._extract_from_m3u8(manifest_url, video_id)
formats = _map_to_format_list(url_map)
+ # Accept-Encoding header causes failures in live streams on Youtube and Youtube Gaming
+ for a_format in formats:
+ if 'http_headers' not in a_format:
+ a_format['http_headers'] = {}
+ a_format['http_headers']['Youtubedl-no-compression'] = True
else:
raise ExtractorError('no conn, hlsvp or url_encoded_fmt_stream_map information found in video info')
From 0424ec307bb920a2a7c217a741241f3d2af84efa Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 29 Nov 2015 12:46:04 +0800
Subject: [PATCH 393/415] [utils] Correct docstring of YoutubeDLHandler
---
youtube_dl/utils.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 653a49055..c43e9e3a1 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -679,7 +679,7 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
the standard headers to every HTTP request and handles gzipped and
deflated responses from web servers. If compression is to be avoided in
a particular request, the original request in the program code only has
- to include the HTTP header "Youtubedl-No-Compression", which will be
+ to include the HTTP header "Youtubedl-no-compression", which will be
removed before making the real request.
Part of this code was copied from:
From 8639f89f516c5bd1e4fda38c40e2a5a9b940ad85 Mon Sep 17 00:00:00 2001
From: Ryan Schmidt <github@ryandesign.com>
Date: Sat, 28 Nov 2015 22:56:24 -0600
Subject: [PATCH 394/415] Always use PYTHON env var in Makefile
---
Makefile | 14 +++++++-------
1 file changed, 7 insertions(+), 7 deletions(-)
diff --git a/Makefile b/Makefile
index 0636fc4cb..337a2eefb 100644
--- a/Makefile
+++ b/Makefile
@@ -60,34 +60,34 @@ youtube-dl: youtube_dl/*.py youtube_dl/*/*.py
chmod a+x youtube-dl
README.md: youtube_dl/*.py youtube_dl/*/*.py
- COLUMNS=80 python youtube_dl/__main__.py --help | python devscripts/make_readme.py
+ COLUMNS=80 $(PYTHON) youtube_dl/__main__.py --help | $(PYTHON) devscripts/make_readme.py
CONTRIBUTING.md: README.md
- python devscripts/make_contributing.py README.md CONTRIBUTING.md
+ $(PYTHON) devscripts/make_contributing.py README.md CONTRIBUTING.md
supportedsites:
- python devscripts/make_supportedsites.py docs/supportedsites.md
+ $(PYTHON) devscripts/make_supportedsites.py docs/supportedsites.md
README.txt: README.md
pandoc -f markdown -t plain README.md -o README.txt
youtube-dl.1: README.md
- python devscripts/prepare_manpage.py >youtube-dl.1.temp.md
+ $(PYTHON) devscripts/prepare_manpage.py >youtube-dl.1.temp.md
pandoc -s -f markdown -t man youtube-dl.1.temp.md -o youtube-dl.1
rm -f youtube-dl.1.temp.md
youtube-dl.bash-completion: youtube_dl/*.py youtube_dl/*/*.py devscripts/bash-completion.in
- python devscripts/bash-completion.py
+ $(PYTHON) devscripts/bash-completion.py
bash-completion: youtube-dl.bash-completion
youtube-dl.zsh: youtube_dl/*.py youtube_dl/*/*.py devscripts/zsh-completion.in
- python devscripts/zsh-completion.py
+ $(PYTHON) devscripts/zsh-completion.py
zsh-completion: youtube-dl.zsh
youtube-dl.fish: youtube_dl/*.py youtube_dl/*/*.py devscripts/fish-completion.in
- python devscripts/fish-completion.py
+ $(PYTHON) devscripts/fish-completion.py
fish-completion: youtube-dl.fish
From 992fc9d6e124b910ff3d720e252ef9aad99b2a8b Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 29 Nov 2015 12:58:29 +0800
Subject: [PATCH 395/415] [utils] Refactor handle_youtubedl_headers for future
extension
---
youtube_dl/utils.py | 11 ++++++-----
1 file changed, 6 insertions(+), 5 deletions(-)
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index c43e9e3a1..d0606b4bc 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -664,12 +664,13 @@ def _create_http_connection(ydl_handler, http_class, is_https, *args, **kwargs):
def handle_youtubedl_headers(headers):
- if 'Youtubedl-no-compression' in headers:
- filtered_headers = dict((k, v) for k, v in headers.items() if k.lower() != 'accept-encoding')
- del filtered_headers['Youtubedl-no-compression']
- return filtered_headers
+ filtered_headers = headers
- return headers
+ if 'Youtubedl-no-compression' in filtered_headers:
+ filtered_headers = dict((k, v) for k, v in filtered_headers.items() if k.lower() != 'accept-encoding')
+ del filtered_headers['Youtubedl-no-compression']
+
+ return filtered_headers
class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
From bf2c8c8f82ff54f9594673c48e531661a72dbdcd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 29 Nov 2015 17:03:33 +0600
Subject: [PATCH 396/415] [spiegel] Fix extraction (Closes #7693)
---
youtube_dl/extractor/spiegel.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/spiegel.py b/youtube_dl/extractor/spiegel.py
index 5bd3c0087..39a7aaf9d 100644
--- a/youtube_dl/extractor/spiegel.py
+++ b/youtube_dl/extractor/spiegel.py
@@ -58,7 +58,8 @@ class SpiegelIE(InfoExtractor):
description = self._html_search_meta('description', webpage, 'description')
base_url = self._search_regex(
- r'var\s+server\s*=\s*"([^"]+)\"', webpage, 'server URL')
+ [r'server\s*:\s*(["\'])(?P<url>.+?)\1', r'var\s+server\s*=\s*"(?P<url>[^"]+)\"'],
+ webpage, 'server URL', group='url')
xml_url = base_url + video_id + '.xml'
idoc = self._download_xml(xml_url, video_id)
From 049d71d8745014bf5ec23e25e51d6b92556baa8c Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 29 Nov 2015 19:52:48 +0800
Subject: [PATCH 397/415] [youtube] Simplify and make sure header values are
strings
---
youtube_dl/extractor/youtube.py | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 52f4fe36d..4f375e2c8 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1477,9 +1477,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
formats = _map_to_format_list(url_map)
# Accept-Encoding header causes failures in live streams on Youtube and Youtube Gaming
for a_format in formats:
- if 'http_headers' not in a_format:
- a_format['http_headers'] = {}
- a_format['http_headers']['Youtubedl-no-compression'] = True
+ a_format.setdefault('http_headers', {})['Youtubedl-no-compression'] = 'True'
else:
raise ExtractorError('no conn, hlsvp or url_encoded_fmt_stream_map information found in video info')
From 040ac686798fdc922157cca64d654933e3f6d096 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 29 Nov 2015 21:01:59 +0600
Subject: [PATCH 398/415] [youtube] Extend _VALID_URL (Closes #7694)
---
youtube_dl/extractor/youtube.py | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 4f375e2c8..55a06eb68 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -258,7 +258,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
|(?: # or the v= param in all its forms
(?:(?:watch|movie)(?:_popup)?(?:\.php)?/?)? # preceding watch(_popup|.php) or nothing (like /?v=xxxx)
(?:\?|\#!?) # the params delimiter ? or # or #!
- (?:.*?&)?? # any other preceding param (like /?s=tuff&v=xxxx)
+ (?:.*?[&;])?? # any other preceding param (like /?s=tuff&v=xxxx or ?s=tuff&v=V36LpHqtcDY)
v=
)
))
@@ -730,6 +730,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
'skip_download': True,
},
},
+ {
+ 'url': 'https://www.youtube.com/watch?feature=player_embedded&amp;v=V36LpHqtcDY',
+ 'only_matching': True,
+ }
]
def __init__(self, *args, **kwargs):
From 2e1b92854000662e554413df0c34c1cbc0d7ffc8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 29 Nov 2015 21:04:11 +0600
Subject: [PATCH 399/415] [youtube:playlist] Extend _VALID_URL
---
youtube_dl/extractor/youtube.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 55a06eb68..032691e7f 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1566,7 +1566,7 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor, YoutubePlaylistBaseInfoExtract
youtube\.com/
(?:
(?:course|view_play_list|my_playlists|artist|playlist|watch|embed/videoseries)
- \? (?:.*?&)*? (?:p|a|list)=
+ \? (?:.*?[&;])*? (?:p|a|list)=
| p/
)
(
From d53a4af1a49413a38d639aeb7f522c4ebff8f5c8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 30 Nov 2015 03:47:01 +0600
Subject: [PATCH 400/415] [pornhub:playlist] Allow alphanumeric viewkeys
(Closes #7695)
---
youtube_dl/extractor/pornhub.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index 965940a4b..08275687d 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -147,7 +147,8 @@ class PornHubPlaylistIE(InfoExtractor):
entries = [
self.url_result('http://www.pornhub.com/%s' % video_url, 'PornHub')
- for video_url in set(re.findall('href="/?(view_video\.php\?viewkey=\d+[^"]*)"', webpage))
+ for video_url in set(re.findall(
+ r'href="/?(view_video\.php\?.*\bviewkey=[\da-z]+[^"]*)"', webpage))
]
playlist = self._parse_json(
From af284305d58a9915a8ef00d056484b3a59548dda Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 30 Nov 2015 03:58:39 +0600
Subject: [PATCH 401/415] [vodlocker] Capture file not found error (Closes
#7696)
---
youtube_dl/extractor/vodlocker.py | 13 +++++++++++--
1 file changed, 11 insertions(+), 2 deletions(-)
diff --git a/youtube_dl/extractor/vodlocker.py b/youtube_dl/extractor/vodlocker.py
index be0a2780f..357594a11 100644
--- a/youtube_dl/extractor/vodlocker.py
+++ b/youtube_dl/extractor/vodlocker.py
@@ -3,11 +3,14 @@ from __future__ import unicode_literals
from .common import InfoExtractor
from ..compat import compat_urllib_parse
-from ..utils import sanitized_Request
+from ..utils import (
+ ExtractorError,
+ sanitized_Request,
+)
class VodlockerIE(InfoExtractor):
- _VALID_URL = r'https?://(?:www\.)?vodlocker\.com/(?P<id>[0-9a-zA-Z]+)(?:\..*?)?'
+ _VALID_URL = r'https?://(?:www\.)?vodlocker\.com/(?:embed-)?(?P<id>[0-9a-zA-Z]+)(?:\..*?)?'
_TESTS = [{
'url': 'http://vodlocker.com/e8wvyzz4sl42',
@@ -24,6 +27,12 @@ class VodlockerIE(InfoExtractor):
video_id = self._match_id(url)
webpage = self._download_webpage(url, video_id)
+ if any(p in webpage for p in (
+ '>THIS FILE WAS DELETED<',
+ '>File Not Found<',
+ 'The file you were looking for could not be found, sorry for any inconvenience.<')):
+ raise ExtractorError('Video %s does not exist' % video_id, expected=True)
+
fields = self._hidden_inputs(webpage)
if fields['op'] == 'download1':
From 59ee8a86471af488c2ee16dcacf7a913636f0150 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 30 Nov 2015 20:10:09 +0600
Subject: [PATCH 402/415] [facebook] Make alternative title optional (Closes
#7700)
---
youtube_dl/extractor/facebook.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index fd854411b..321eec59e 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -164,7 +164,7 @@ class FacebookIE(InfoExtractor):
if not video_title:
video_title = self._html_search_regex(
r'(?s)<span class="fbPhotosPhotoCaption".*?id="fbPhotoPageCaption"><span class="hasCaption">(.*?)</span>',
- webpage, 'alternative title', fatal=False)
+ webpage, 'alternative title', default=None)
video_title = limit_length(video_title, 80)
if not video_title:
video_title = 'Facebook video #%s' % video_id
From 4c6b4764f0260808d321cfb6cca1daa5e3eb13d0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 30 Nov 2015 20:42:05 +0600
Subject: [PATCH 403/415] [youtube] Clarify itag 272 possible resolutions
(#7699)
---
youtube_dl/extractor/youtube.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 032691e7f..9b39505ba 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -346,6 +346,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
'247': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
'248': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
'271': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
+ # itag 272 videos are either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug)
'272': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
'302': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'vp9'},
'303': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'vp9'},
From 874ae0354e96a578dc44846fffd91b01079ccf1e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 1 Dec 2015 18:35:24 +0600
Subject: [PATCH 404/415] [nrk] Extract f4m formats and impose geo restriction
only when not media URL (Closes #7715)
---
youtube_dl/extractor/nrk.py | 24 +++++++++++++++++-------
1 file changed, 17 insertions(+), 7 deletions(-)
diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index 8ac38a174..6ff13050d 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -6,6 +6,7 @@ import re
from .common import InfoExtractor
from ..compat import compat_urlparse
from ..utils import (
+ determine_ext,
ExtractorError,
float_or_none,
parse_duration,
@@ -48,12 +49,22 @@ class NRKIE(InfoExtractor):
'http://v8.psapi.nrk.no/mediaelement/%s' % video_id,
video_id, 'Downloading media JSON')
- if data['usageRights']['isGeoBlocked']:
- raise ExtractorError(
- 'NRK har ikke rettigheter til å vise dette programmet utenfor Norge',
- expected=True)
+ media_url = data.get('mediaUrl')
- video_url = data['mediaUrl'] + '?hdcore=3.5.0&plugin=aasp-3.5.0.151.81'
+ if not media_url:
+ if data['usageRights']['isGeoBlocked']:
+ raise ExtractorError(
+ 'NRK har ikke rettigheter til å vise dette programmet utenfor Norge',
+ expected=True)
+
+ if determine_ext(media_url) == 'f4m':
+ formats = self._extract_f4m_formats(
+ media_url + '?hdcore=3.5.0&plugin=aasp-3.5.0.151.81', video_id, f4m_id='hds')
+ else:
+ formats = [{
+ 'url': media_url,
+ 'ext': 'flv',
+ }]
duration = parse_duration(data.get('duration'))
@@ -67,12 +78,11 @@ class NRKIE(InfoExtractor):
return {
'id': video_id,
- 'url': video_url,
- 'ext': 'flv',
'title': data['title'],
'description': data['description'],
'duration': duration,
'thumbnail': thumbnail,
+ 'formats': formats,
}
From 3b35c3425e2c4d9836a3efe5cf73e35a60f674d1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 1 Dec 2015 20:35:46 +0600
Subject: [PATCH 405/415] [udemy] Extract formats from data.outputs (#7704)
---
youtube_dl/extractor/udemy.py | 46 ++++++++++++++++++++++++++---------
1 file changed, 34 insertions(+), 12 deletions(-)
diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index 825172806..9d31a3f5b 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -9,6 +9,7 @@ from ..compat import (
)
from ..utils import (
ExtractorError,
+ int_or_none,
sanitized_Request,
)
@@ -127,26 +128,47 @@ class UdemyIE(InfoExtractor):
video_id = asset['id']
thumbnail = asset.get('thumbnailUrl') or asset.get('thumbnail_url')
- duration = asset['data']['duration']
+ duration = int_or_none(asset.get('data', {}).get('duration'))
download_url = asset.get('downloadUrl') or asset.get('download_url')
video = download_url.get('Video') or download_url.get('video')
video_480p = download_url.get('Video480p') or download_url.get('video_480p')
- formats = [
- {
- 'url': video_480p[0],
- 'format_id': '360p',
- },
- {
- 'url': video[0],
- 'format_id': '720p',
- },
- ]
+ formats = [{
+ 'url': video_480p[0],
+ 'format_id': 'download-360p',
+ }, {
+ 'url': video[0],
+ 'format_id': 'download-720p',
+ }]
+
+ # Some videos also contain formats in asset['data']['outputs'] (e.g.
+ # https://www.udemy.com/ios9-swift/learn/#/lecture/3383208)
+ outputs = asset.get('data', {}).get('outputs')
+ if isinstance(outputs, dict):
+ for format_id, f in outputs.items():
+ video_url = f.get('url')
+ if video_url:
+ formats.append({
+ 'url': video_url,
+ 'format_id': '%sp' % (f.get('labe1l') or format_id),
+ 'width': int_or_none(f.get('width')),
+ 'height': int_or_none(f.get('height')),
+ 'vbr': int_or_none(f.get('video_bitrate_in_kbps')),
+ 'vcodec': f.get('video_codec'),
+ 'fps': int_or_none(f.get('frame_rate')),
+ 'abr': int_or_none(f.get('audio_bitrate_in_kbps')),
+ 'acodec': f.get('audio_codec'),
+ 'asr': int_or_none(f.get('audio_sample_rate')),
+ 'tbr': int_or_none(f.get('total_bitrate_in_kbps')),
+ 'filesize': int_or_none(f.get('file_size_in_bytes')),
+ })
+
+ self._sort_formats(formats)
title = lecture['title']
- description = lecture['description']
+ description = lecture.get('description')
return {
'id': video_id,
From 78717fc32802cb1e0f240cfbdf02207c6356ff2b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 1 Dec 2015 22:10:10 +0600
Subject: [PATCH 406/415] [udemy] Allow authentication via cookies
---
youtube_dl/extractor/udemy.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index 9d31a3f5b..e743c0262 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -72,7 +72,7 @@ class UdemyIE(InfoExtractor):
def _login(self):
(username, password) = self._get_login_info()
if username is None:
- self.raise_login_required('Udemy account is required')
+ return
login_popup = self._download_webpage(
self._LOGIN_URL, None, 'Downloading login popup')
From 328f82d59a0d3b743280c3ab532949c9220383ed Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 2 Dec 2015 00:48:27 +0600
Subject: [PATCH 407/415] [udemy] Semi-switch to api 2.0 (Closes #7704)
* Use api 2.0 to get lectures since it provides more formats
* Fix authorization for api 2.0
* Autotry enrolling in the course for single lectures
* Extract additional metadata rom asset['data']['outputs']
---
youtube_dl/extractor/udemy.py | 160 ++++++++++++++++++++--------------
1 file changed, 95 insertions(+), 65 deletions(-)
diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index e743c0262..26ae8cc36 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -1,14 +1,15 @@
from __future__ import unicode_literals
-import re
-
from .common import InfoExtractor
from ..compat import (
+ compat_HTTPError,
compat_urllib_parse,
compat_urllib_request,
+ compat_urlparse,
)
from ..utils import (
ExtractorError,
+ float_or_none,
int_or_none,
sanitized_Request,
)
@@ -19,6 +20,8 @@ class UdemyIE(InfoExtractor):
_VALID_URL = r'https?://www\.udemy\.com/(?:[^#]+#/lecture/|lecture/view/?\?lectureId=)(?P<id>\d+)'
_LOGIN_URL = 'https://www.udemy.com/join/login-popup/?displayType=ajax&showSkipButton=1'
_ORIGIN_URL = 'https://www.udemy.com'
+ _SUCCESSFULLY_ENROLLED = '>You have enrolled in this course!<'
+ _ALREADY_ENROLLED = '>You are already taking this course.<'
_NETRC_MACHINE = 'udemy'
_TESTS = [{
@@ -34,6 +37,29 @@ class UdemyIE(InfoExtractor):
'skip': 'Requires udemy account credentials',
}]
+ def _enroll_course(self, webpage, course_id):
+ enroll_url = self._search_regex(
+ r'href=(["\'])(?P<url>https?://(?:www\.)?udemy\.com/course/subscribe/.+?)\1',
+ webpage, 'enroll url', group='url',
+ default='https://www.udemy.com/course/subscribe/?courseId=%s' % course_id)
+ webpage = self._download_webpage(enroll_url, course_id, 'Enrolling in the course')
+ if self._SUCCESSFULLY_ENROLLED in webpage:
+ self.to_screen('%s: Successfully enrolled in' % course_id)
+ elif self._ALREADY_ENROLLED in webpage:
+ self.to_screen('%s: Already enrolled in' % course_id)
+
+ def _download_lecture(self, course_id, lecture_id):
+ return self._download_json(
+ 'https://www.udemy.com/api-2.0/users/me/subscribed-courses/%s/lectures/%s?%s' % (
+ course_id, lecture_id, compat_urllib_parse.urlencode({
+ 'video_only': '',
+ 'auto_play': '',
+ 'fields[lecture]': 'title,description,asset',
+ 'fields[asset]': 'asset_type,stream_url,thumbnail_url,download_urls,data',
+ 'instructorPreviewMode': 'False',
+ })),
+ lecture_id, 'Downloading lecture JSON', fatal=False)
+
def _handle_error(self, response):
if not isinstance(response, dict):
return
@@ -45,7 +71,7 @@ class UdemyIE(InfoExtractor):
error_str += ' - %s' % error_data.get('formErrors')
raise ExtractorError(error_str, expected=True)
- def _download_json(self, url_or_request, video_id, note='Downloading JSON metadata'):
+ def _download_json(self, url_or_request, video_id, note='Downloading JSON metadata', *args, **kwargs):
headers = {
'X-Udemy-Snail-Case': 'true',
'X-Requested-With': 'XMLHttpRequest',
@@ -55,6 +81,7 @@ class UdemyIE(InfoExtractor):
headers['X-Udemy-Client-Id'] = cookie.value
elif cookie.name == 'access_token':
headers['X-Udemy-Bearer-Token'] = cookie.value
+ headers['X-Udemy-Authorization'] = 'Bearer %s' % cookie.value
if isinstance(url_or_request, compat_urllib_request.Request):
for header, value in headers.items():
@@ -62,7 +89,7 @@ class UdemyIE(InfoExtractor):
else:
url_or_request = sanitized_Request(url_or_request, headers=headers)
- response = super(UdemyIE, self)._download_json(url_or_request, video_id, note)
+ response = super(UdemyIE, self)._download_json(url_or_request, video_id, note, *args, **kwargs)
self._handle_error(response)
return response
@@ -110,66 +137,77 @@ class UdemyIE(InfoExtractor):
def _real_extract(self, url):
lecture_id = self._match_id(url)
- lecture = self._download_json(
- 'https://www.udemy.com/api-1.1/lectures/%s' % lecture_id,
- lecture_id, 'Downloading lecture JSON')
+ webpage = self._download_webpage(url, lecture_id)
- asset_type = lecture.get('assetType') or lecture.get('asset_type')
+ course_id = self._search_regex(
+ r'data-course-id=["\'](\d+)', webpage, 'course id')
+
+ try:
+ lecture = self._download_lecture(course_id, lecture_id)
+ except ExtractorError as e:
+ # Error could possibly mean we are not enrolled in the course
+ if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
+ self._enroll_course(webpage, course_id)
+ lecture_id = self._download_lecture(course_id, lecture_id)
+ else:
+ raise
+
+ title = lecture['title']
+ description = lecture.get('description')
+
+ asset = lecture['asset']
+
+ asset_type = asset.get('assetType') or asset.get('asset_type')
if asset_type != 'Video':
raise ExtractorError(
'Lecture %s is not a video' % lecture_id, expected=True)
- asset = lecture['asset']
-
stream_url = asset.get('streamUrl') or asset.get('stream_url')
- mobj = re.search(r'(https?://www\.youtube\.com/watch\?v=.*)', stream_url)
- if mobj:
- return self.url_result(mobj.group(1), 'Youtube')
+ if stream_url:
+ youtube_url = self._search_regex(
+ r'(https?://www\.youtube\.com/watch\?v=.*)', stream_url, 'youtube URL', default=None)
+ if youtube_url:
+ return self.url_result(youtube_url, 'Youtube')
video_id = asset['id']
thumbnail = asset.get('thumbnailUrl') or asset.get('thumbnail_url')
- duration = int_or_none(asset.get('data', {}).get('duration'))
+ duration = float_or_none(asset.get('data', {}).get('duration'))
+ outputs = asset.get('data', {}).get('outputs', {})
- download_url = asset.get('downloadUrl') or asset.get('download_url')
-
- video = download_url.get('Video') or download_url.get('video')
- video_480p = download_url.get('Video480p') or download_url.get('video_480p')
-
- formats = [{
- 'url': video_480p[0],
- 'format_id': 'download-360p',
- }, {
- 'url': video[0],
- 'format_id': 'download-720p',
- }]
-
- # Some videos also contain formats in asset['data']['outputs'] (e.g.
- # https://www.udemy.com/ios9-swift/learn/#/lecture/3383208)
- outputs = asset.get('data', {}).get('outputs')
- if isinstance(outputs, dict):
- for format_id, f in outputs.items():
- video_url = f.get('url')
- if video_url:
- formats.append({
- 'url': video_url,
- 'format_id': '%sp' % (f.get('labe1l') or format_id),
- 'width': int_or_none(f.get('width')),
- 'height': int_or_none(f.get('height')),
- 'vbr': int_or_none(f.get('video_bitrate_in_kbps')),
- 'vcodec': f.get('video_codec'),
- 'fps': int_or_none(f.get('frame_rate')),
- 'abr': int_or_none(f.get('audio_bitrate_in_kbps')),
- 'acodec': f.get('audio_codec'),
- 'asr': int_or_none(f.get('audio_sample_rate')),
- 'tbr': int_or_none(f.get('total_bitrate_in_kbps')),
- 'filesize': int_or_none(f.get('file_size_in_bytes')),
+ formats = []
+ for format_ in asset.get('download_urls', {}).get('Video', []):
+ video_url = format_.get('file')
+ if not video_url:
+ continue
+ format_id = format_.get('label')
+ f = {
+ 'url': format_['file'],
+ 'height': int_or_none(format_id),
+ }
+ if format_id:
+ # Some videos contain additional metadata (e.g.
+ # https://www.udemy.com/ios9-swift/learn/#/lecture/3383208)
+ output = outputs.get(format_id)
+ if isinstance(output, dict):
+ f.update({
+ 'format_id': '%sp' % (output.get('label') or format_id),
+ 'width': int_or_none(output.get('width')),
+ 'height': int_or_none(output.get('height')),
+ 'vbr': int_or_none(output.get('video_bitrate_in_kbps')),
+ 'vcodec': output.get('video_codec'),
+ 'fps': int_or_none(output.get('frame_rate')),
+ 'abr': int_or_none(output.get('audio_bitrate_in_kbps')),
+ 'acodec': output.get('audio_codec'),
+ 'asr': int_or_none(output.get('audio_sample_rate')),
+ 'tbr': int_or_none(output.get('total_bitrate_in_kbps')),
+ 'filesize': int_or_none(output.get('file_size_in_bytes')),
})
+ else:
+ f['format_id'] = '%sp' % format_id
+ formats.append(f)
self._sort_formats(formats)
- title = lecture['title']
- description = lecture.get('description')
-
return {
'id': video_id,
'title': title,
@@ -182,9 +220,7 @@ class UdemyIE(InfoExtractor):
class UdemyCourseIE(UdemyIE):
IE_NAME = 'udemy:course'
- _VALID_URL = r'https?://www\.udemy\.com/(?P<coursepath>[\da-z-]+)'
- _SUCCESSFULLY_ENROLLED = '>You have enrolled in this course!<'
- _ALREADY_ENROLLED = '>You are already taking this course.<'
+ _VALID_URL = r'https?://www\.udemy\.com/(?P<id>[\da-z-]+)'
_TESTS = []
@classmethod
@@ -192,24 +228,18 @@ class UdemyCourseIE(UdemyIE):
return False if UdemyIE.suitable(url) else super(UdemyCourseIE, cls).suitable(url)
def _real_extract(self, url):
- mobj = re.match(self._VALID_URL, url)
- course_path = mobj.group('coursepath')
+ course_path = self._match_id(url)
+
+ webpage = self._download_webpage(url, course_path)
response = self._download_json(
'https://www.udemy.com/api-1.1/courses/%s' % course_path,
course_path, 'Downloading course JSON')
- course_id = int(response['id'])
- course_title = response['title']
+ course_id = response['id']
+ course_title = response.get('title')
- webpage = self._download_webpage(
- 'https://www.udemy.com/course/subscribe/?courseId=%s' % course_id,
- course_id, 'Enrolling in the course')
-
- if self._SUCCESSFULLY_ENROLLED in webpage:
- self.to_screen('%s: Successfully enrolled in' % course_id)
- elif self._ALREADY_ENROLLED in webpage:
- self.to_screen('%s: Already enrolled in' % course_id)
+ self._enroll_course(webpage, course_id)
response = self._download_json(
'https://www.udemy.com/api-1.1/courses/%s/curriculum' % course_id,
From 9fc87fa767425ee7d56b2ce4650c9b06b480e005 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 2 Dec 2015 00:51:47 +0600
Subject: [PATCH 408/415] [udemy] Remove unused import
---
youtube_dl/extractor/udemy.py | 1 -
1 file changed, 1 deletion(-)
diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index 26ae8cc36..2a54f3764 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -5,7 +5,6 @@ from ..compat import (
compat_HTTPError,
compat_urllib_parse,
compat_urllib_request,
- compat_urlparse,
)
from ..utils import (
ExtractorError,
From 24121bc703152312cfbb70f01ebd39e2fe1197e5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 2 Dec 2015 00:53:03 +0600
Subject: [PATCH 409/415] [udemy] Make lecture downloading fatal
---
youtube_dl/extractor/udemy.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index 2a54f3764..59832b1ec 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -57,7 +57,7 @@ class UdemyIE(InfoExtractor):
'fields[asset]': 'asset_type,stream_url,thumbnail_url,download_urls,data',
'instructorPreviewMode': 'False',
})),
- lecture_id, 'Downloading lecture JSON', fatal=False)
+ lecture_id, 'Downloading lecture JSON')
def _handle_error(self, response):
if not isinstance(response, dict):
@@ -70,7 +70,7 @@ class UdemyIE(InfoExtractor):
error_str += ' - %s' % error_data.get('formErrors')
raise ExtractorError(error_str, expected=True)
- def _download_json(self, url_or_request, video_id, note='Downloading JSON metadata', *args, **kwargs):
+ def _download_json(self, url_or_request, video_id, note='Downloading JSON metadata'):
headers = {
'X-Udemy-Snail-Case': 'true',
'X-Requested-With': 'XMLHttpRequest',
@@ -88,7 +88,7 @@ class UdemyIE(InfoExtractor):
else:
url_or_request = sanitized_Request(url_or_request, headers=headers)
- response = super(UdemyIE, self)._download_json(url_or_request, video_id, note, *args, **kwargs)
+ response = super(UdemyIE, self)._download_json(url_or_request, video_id, note)
self._handle_error(response)
return response
From 22d7368dfb384e7698faad6d2891b4aaceab3d7c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 2 Dec 2015 02:34:31 +0600
Subject: [PATCH 410/415] [bbc] Extract _ID_REGEX and ad one more video id
pattern (Closes #7724)
---
youtube_dl/extractor/bbc.py | 12 +++++++-----
1 file changed, 7 insertions(+), 5 deletions(-)
diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 33b296eaf..5f7265a06 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -22,7 +22,8 @@ from ..compat import (
class BBCCoUkIE(InfoExtractor):
IE_NAME = 'bbc.co.uk'
IE_DESC = 'BBC iPlayer'
- _VALID_URL = r'https?://(?:www\.)?bbc\.co\.uk/(?:(?:programmes/(?!articles/)|iplayer(?:/[^/]+)?/(?:episode/|playlist/))|music/clips[/#])(?P<id>[\da-z]{8})'
+ _ID_REGEX = r'[pb][\da-z]{7}'
+ _VALID_URL = r'https?://(?:(?:www\.)?bbc\.co\.uk/(?:(?:programmes/(?!articles/)|iplayer(?:/[^/]+)?/(?:episode/|playlist/))|music/clips[/#])|)(?P<id>%s)' % _ID_REGEX
_MEDIASELECTOR_URLS = [
# Provides HQ HLS streams with even better quality that pc mediaset but fails
@@ -465,7 +466,7 @@ class BBCCoUkIE(InfoExtractor):
if not programme_id:
programme_id = self._search_regex(
- r'"vpid"\s*:\s*"([\da-z]{8})"', webpage, 'vpid', fatal=False, default=None)
+ r'"vpid"\s*:\s*"(%s)"' % self._ID_REGEX, webpage, 'vpid', fatal=False, default=None)
if programme_id:
formats, subtitles = self._download_media_selector(programme_id)
@@ -780,8 +781,9 @@ class BBCIE(BBCCoUkIE):
# single video story (e.g. http://www.bbc.com/travel/story/20150625-sri-lankas-spicy-secret)
programme_id = self._search_regex(
- [r'data-video-player-vpid="([\da-z]{8})"',
- r'<param[^>]+name="externalIdentifier"[^>]+value="([\da-z]{8})"'],
+ [r'data-video-player-vpid="(%s)"' % self._ID_REGEX,
+ r'<param[^>]+name="externalIdentifier"[^>]+value="(%s)"' % self._ID_REGEX,
+ r'videoId\s*:\s*["\'](%s)["\']' % self._ID_REGEX],
webpage, 'vpid', default=None)
if programme_id:
@@ -816,7 +818,7 @@ class BBCIE(BBCCoUkIE):
# Multiple video article (e.g.
# http://www.bbc.co.uk/blogs/adamcurtis/entries/3662a707-0af9-3149-963f-47bea720b460)
- EMBED_URL = r'https?://(?:www\.)?bbc\.co\.uk/(?:[^/]+/)+[\da-z]{8}(?:\b[^"]+)?'
+ EMBED_URL = r'https?://(?:www\.)?bbc\.co\.uk/(?:[^/]+/)+%s(?:\b[^"]+)?' % self._ID_REGEX
entries = []
for match in extract_all(r'new\s+SMP\(({.+?})\)'):
embed_url = match.get('playerSettings', {}).get('externalEmbedUrl')
From 7b1e379ca9316433ee911f2f36c7808522fff13d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
<jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 3 Dec 2015 13:47:21 +0100
Subject: [PATCH 411/415] [gametrailers] Fix extraction (fixes #7722)
They have stopped using the MTV system.
---
youtube_dl/extractor/gametrailers.py | 61 ++++++++++++++++++++++++----
1 file changed, 52 insertions(+), 9 deletions(-)
diff --git a/youtube_dl/extractor/gametrailers.py b/youtube_dl/extractor/gametrailers.py
index a6ab795ae..c3f031d9c 100644
--- a/youtube_dl/extractor/gametrailers.py
+++ b/youtube_dl/extractor/gametrailers.py
@@ -1,19 +1,62 @@
from __future__ import unicode_literals
-from .mtv import MTVServicesInfoExtractor
+from .common import InfoExtractor
+from ..utils import (
+ int_or_none,
+ parse_age_limit,
+ url_basename,
+)
-class GametrailersIE(MTVServicesInfoExtractor):
- _VALID_URL = r'http://www\.gametrailers\.com/(?P<type>videos|reviews|full-episodes)/(?P<id>.*?)/(?P<title>.*)'
+class GametrailersIE(InfoExtractor):
+ _VALID_URL = r'http://www\.gametrailers\.com/videos/view/[^/]+/(?P<id>.+)'
+
_TEST = {
- 'url': 'http://www.gametrailers.com/videos/zbvr8i/mirror-s-edge-2-e3-2013--debut-trailer',
- 'md5': '4c8e67681a0ea7ec241e8c09b3ea8cf7',
+ 'url': 'http://www.gametrailers.com/videos/view/gametrailers-com/116437-Just-Cause-3-Review',
+ 'md5': 'f28c4efa0bdfaf9b760f6507955b6a6a',
'info_dict': {
- 'id': '70e9a5d7-cf25-4a10-9104-6f3e7342ae0d',
+ 'id': '2983958',
'ext': 'mp4',
- 'title': 'E3 2013: Debut Trailer',
- 'description': 'Faith is back! Check out the World Premiere trailer for Mirror\'s Edge 2 straight from the EA Press Conference at E3 2013!',
+ 'display_id': '116437-Just-Cause-3-Review',
+ 'title': 'Just Cause 3 - Review',
+ 'description': 'It\'s a lot of fun to shoot at things and then watch them explode in Just Cause 3, but should there be more to the experience than that?',
},
}
- _FEED_URL = 'http://www.gametrailers.com/feeds/mrss'
+ def _real_extract(self, url):
+ display_id = self._match_id(url)
+ webpage = self._download_webpage(url, display_id)
+ title = self._html_search_regex(
+ r'<title>(.+?)\|', webpage, 'title').strip()
+ embed_url = self._proto_relative_url(
+ self._search_regex(
+ r'src=\'(//embed.gametrailers.com/embed/[^\']+)\'', webpage,
+ 'embed url'),
+ scheme='http:')
+ video_id = url_basename(embed_url)
+ embed_page = self._download_webpage(embed_url, video_id)
+ embed_vars_json = self._search_regex(
+ r'(?s)var embedVars = (\{.*?\})\s*</script>', embed_page,
+ 'embed vars')
+ info = self._parse_json(embed_vars_json, video_id)
+
+ formats = []
+ for media in info['media']:
+ if media['mediaPurpose'] == 'play':
+ formats.append({
+ 'url': media['uri'],
+ 'height': media['height'],
+ 'width:': media['width'],
+ })
+ self._sort_formats(formats)
+
+ return {
+ 'id': video_id,
+ 'display_id': display_id,
+ 'title': title,
+ 'formats': formats,
+ 'thumbnail': info.get('thumbUri'),
+ 'description': self._og_search_description(webpage),
+ 'duration': int_or_none(info.get('videoLengthInSeconds')),
+ 'age_limit': parse_age_limit(info.get('audienceRating')),
+ }
From 49358274d7145788f2e99471dbb7c7c4301a3d3d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 3 Dec 2015 20:49:14 +0600
Subject: [PATCH 412/415] [bbc] Fix _VALID_URL
---
youtube_dl/extractor/bbc.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 5f7265a06..7fb80aa38 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -23,7 +23,7 @@ class BBCCoUkIE(InfoExtractor):
IE_NAME = 'bbc.co.uk'
IE_DESC = 'BBC iPlayer'
_ID_REGEX = r'[pb][\da-z]{7}'
- _VALID_URL = r'https?://(?:(?:www\.)?bbc\.co\.uk/(?:(?:programmes/(?!articles/)|iplayer(?:/[^/]+)?/(?:episode/|playlist/))|music/clips[/#])|)(?P<id>%s)' % _ID_REGEX
+ _VALID_URL = r'https?://(?:www\.)?bbc\.co\.uk/(?:(?:programmes/(?!articles/)|iplayer(?:/[^/]+)?/(?:episode/|playlist/))|music/clips[/#])(?P<id>%s)' % _ID_REGEX
_MEDIASELECTOR_URLS = [
# Provides HQ HLS streams with even better quality that pc mediaset but fails
From 62d231c00493c329a78a30b15a336f131fc392f9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 3 Dec 2015 20:55:02 +0600
Subject: [PATCH 413/415] [extractor/common] Clarify duration can be float
---
youtube_dl/extractor/common.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index eb9bfa3d1..6ab2d68d6 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -167,7 +167,7 @@ class InfoExtractor(object):
"ext" will be calculated from URL if missing
automatic_captions: Like 'subtitles', used by the YoutubeIE for
automatically generated captions
- duration: Length of the video in seconds, as an integer.
+ duration: Length of the video in seconds, as an integer or float.
view_count: How many users have watched the video on the platform.
like_count: Number of positive ratings of the video
dislike_count: Number of negative ratings of the video
From af93fcfa0549bdfe26c39adecbb1bcc98a9345af Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 3 Dec 2015 23:23:36 +0600
Subject: [PATCH 414/415] [beeg] Update API URL (Closes #7736)
---
youtube_dl/extractor/beeg.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/beeg.py b/youtube_dl/extractor/beeg.py
index 61bc2f744..1ee4a8b05 100644
--- a/youtube_dl/extractor/beeg.py
+++ b/youtube_dl/extractor/beeg.py
@@ -29,7 +29,7 @@ class BeegIE(InfoExtractor):
video_id = self._match_id(url)
video = self._download_json(
- 'http://beeg.com/api/v1/video/%s' % video_id, video_id)
+ 'http://beeg.com/api/v3/video/%s' % video_id, video_id)
formats = []
for format_id, video_url in video.items():
From e0977d7686e5df524b1a024484e7a4bb9cfa261d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 4 Dec 2015 00:59:32 +0600
Subject: [PATCH 415/415] [beeg] Decrypt URL (Closes #7736)
---
youtube_dl/extractor/beeg.py | 24 +++++++++++++++++++++++-
1 file changed, 23 insertions(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/beeg.py b/youtube_dl/extractor/beeg.py
index 1ee4a8b05..e63c2ac00 100644
--- a/youtube_dl/extractor/beeg.py
+++ b/youtube_dl/extractor/beeg.py
@@ -1,6 +1,11 @@
from __future__ import unicode_literals
from .common import InfoExtractor
+from ..compat import (
+ compat_chr,
+ compat_ord,
+ compat_urllib_parse_unquote,
+)
from ..utils import (
int_or_none,
parse_iso8601,
@@ -31,6 +36,23 @@ class BeegIE(InfoExtractor):
video = self._download_json(
'http://beeg.com/api/v3/video/%s' % video_id, video_id)
+ def decrypt_key(key):
+ # Reverse engineered from http://static.beeg.com/cpl/1067.js
+ a = '8RPUUCS35ZWp3ADnKcSmpH71ZusrROo'
+ e = compat_urllib_parse_unquote(key)
+ return ''.join([
+ compat_chr(compat_ord(e[n]) - compat_ord(a[n % len(a)]) % 25)
+ for n in range(len(e))])
+
+ def decrypt_url(encrypted_url):
+ encrypted_url = self._proto_relative_url(
+ encrypted_url.replace('{DATA_MARKERS}', ''), 'http:')
+ key = self._search_regex(
+ r'/key=(.*?)%2Cend=', encrypted_url, 'key', default=None)
+ if not key:
+ return encrypted_url
+ return encrypted_url.replace(key, decrypt_key(key))
+
formats = []
for format_id, video_url in video.items():
if not video_url:
@@ -40,7 +62,7 @@ class BeegIE(InfoExtractor):
if not height:
continue
formats.append({
- 'url': self._proto_relative_url(video_url.replace('{DATA_MARKERS}', ''), 'http:'),
+ 'url': decrypt_url(video_url),
'format_id': format_id,
'height': int(height),
})