release 2013.06.32

devscripts/release.sh

@@ -69,7 +69,9 @@ git checkout HEAD -- youtube-dl youtube-dl.exe
 
 /bin/echo -e "\n### Signing and uploading the new binaries to youtube-dl.org..."
 for f in $RELEASE_FILES; do gpg --detach-sig "build/$version/$f"; done
-scp -r "build/$version" ytdl@youtube-dl.org:html/downloads/
+scp -r "build/$version" ytdl@yt-dl.org:html/tmp/
+ssh ytdl@yt-dl.org "mv html/tmp/$version html/downloads/"
+ssh ytdl@yt-dl.org "sh html/update_latest.sh $version"
 
 /bin/echo -e "\n### Now switching to gh-pages..."
 git clone --branch gh-pages --single-branch . build/gh-pages

test/test_download.py

@@ -153,9 +153,11 @@ def generator(test_case):
     return test_template
 
 ### And add them to TestDownload
-for test_case in defs:
+for n, test_case in enumerate(defs):
     test_method = generator(test_case)
     test_method.__name__ = "test_{0}".format(test_case["name"])
+    if getattr(TestDownload, test_method.__name__, False):
+        test_method.__name__ = "test_{0}_{1}".format(test_case["name"], n)
     setattr(TestDownload, test_method.__name__, test_method)
     del test_method
 

test/tests.json

@@ -11,6 +11,32 @@
       "description": "test chars:  \"'/\\ä↭𝕐\n\nThis is a test video for youtube-dl.\n\nFor more information, contact phihag@phihag.de ."
     }
   },
+  {
+    "name": "Youtube",
+    "url":  "http://www.youtube.com/watch?v=1ltcDfZMA3U",
+    "file":  "1ltcDfZMA3U.flv",
+    "note": "Test VEVO video (#897)",
+    "info_dict": {
+      "upload_date": "20070518",
+      "title": "Maps - It Will Find You",
+      "description": "Music video by Maps performing It Will Find You.",
+      "uploader": "MuteUSA",
+      "uploader_id": "MuteUSA"
+    }
+  },
+  {
+    "name": "Youtube",
+    "url":  "http://www.youtube.com/watch?v=UxxajLWwzqY",
+    "file":  "UxxajLWwzqY.mp4",
+    "note": "Test generic use_cipher_signature video (#897)",
+    "info_dict": {
+      "upload_date": "20120506",
+      "title": "Icona Pop - I Love It (feat. Charli XCX) [OFFICIAL VIDEO]",
+      "description": "md5:b085c9804f5ab69f4adea963a2dceb3c",
+      "uploader": "IconaPop",
+      "uploader_id": "IconaPop"
+    }
+  },
   {
     "name": "Dailymotion",
     "md5":  "392c4b85a60a90dc4792da41ce3144eb",
@@ -649,5 +675,16 @@
     "info_dict": {
         "title": "When Girls Act Like D-Bags"
     }
+  },
+  {
+    "name": "Vevo",
+    "url": "http://www.vevo.com/watch/hurts/somebody-to-die-for/GB1101300280",
+    "file": "GB1101300280.mp4",
+    "md5": "06bea460acb744eab74a9d7dcb4bfd61",
+    "info_dict": {
+        "title": "Somebody To Die For",
+        "upload_date": "20130624",
+        "uploader": "Hurts"
+    }
   }
 ]

youtube_dl/__init__.py

@@ -25,6 +25,7 @@ __authors__  = (
     'M. Yasoob Ullah Khalid',
     'Julien Fraichard',
     'Johny Mo Swag',
+    'Axel Noack',
     )
 
 __license__ = 'Public Domain'
@@ -319,7 +320,7 @@ def parseOpts(overrideArguments=None):
     if overrideArguments is not None:
         opts, args = parser.parse_args(overrideArguments)
         if opts.verbose:
-            print(u'[debug] Override config: ' + repr(overrideArguments))
+            sys.stderr.write(u'[debug] Override config: ' + repr(overrideArguments) + '\n')
     else:
         xdg_config_home = os.environ.get('XDG_CONFIG_HOME')
         if xdg_config_home:
@@ -332,9 +333,9 @@ def parseOpts(overrideArguments=None):
         argv = systemConf + userConf + commandLineConf
         opts, args = parser.parse_args(argv)
         if opts.verbose:
-            print(u'[debug] System config: ' + repr(systemConf))
-            print(u'[debug] User config: ' + repr(userConf))
-            print(u'[debug] Command-line args: ' + repr(commandLineConf))
+            sys.stderr.write(u'[debug] System config: ' + repr(systemConf) + '\n')
+            sys.stderr.write(u'[debug] User config: ' + repr(userConf) + '\n')
+            sys.stderr.write(u'[debug] Command-line args: ' + repr(commandLineConf) + '\n')
 
     return parser, opts, args
 
@@ -369,7 +370,7 @@ def _real_main(argv=None):
 
     # Dump user agent
     if opts.dump_user_agent:
-        print(std_headers['User-Agent'])
+        compat_print(std_headers['User-Agent'])
         sys.exit(0)
 
     # Batch file verification
@@ -410,18 +411,18 @@ def _real_main(argv=None):
 
     if opts.list_extractors:
         for ie in extractors:
-            print(ie.IE_NAME + (' (CURRENTLY BROKEN)' if not ie._WORKING else ''))
+            compat_print(ie.IE_NAME + (' (CURRENTLY BROKEN)' if not ie._WORKING else ''))
             matchedUrls = [url for url in all_urls if ie.suitable(url)]
             all_urls = [url for url in all_urls if url not in matchedUrls]
             for mu in matchedUrls:
-                print(u'  ' + mu)
+                compat_print(u'  ' + mu)
         sys.exit(0)
 
     # Conflicting, missing and erroneous options
     if opts.usenetrc and (opts.username is not None or opts.password is not None):
         parser.error(u'using .netrc conflicts with giving username/password')
     if opts.password is not None and opts.username is None:
-        print(u'WARNING: account username missing')
+        sys.stderr.write(u'WARNING: account username missing\n')
     if opts.outtmpl is not None and (opts.usetitle or opts.autonumber or opts.useid):
         parser.error(u'using output template conflicts with using title, video ID or auto number')
     if opts.usetitle and opts.useid:

youtube_dl/extractor/__init__.py

@@ -22,6 +22,7 @@ from .hypem import HypemIE
 from .ina import InaIE
 from .infoq import InfoQIE
 from .justintv import JustinTVIE
+from .jukebox import JukeboxIE
 from .keek import KeekIE
 from .liveleak import LiveLeakIE
 from .metacafe import MetacafeIE
@@ -44,6 +45,7 @@ from .ted import TEDIE
 from .tumblr import TumblrIE
 from .ustream import UstreamIE
 from .vbox7 import Vbox7IE
+from .vevo import VevoIE
 from .vimeo import VimeoIE
 from .vine import VineIE
 from .worldstarhiphop import WorldStarHipHopIE
@@ -125,6 +127,8 @@ def gen_extractors():
         GametrailersIE(),
         StatigramIE(),
         BreakIE(),
+        VevoIE(),
+        JukeboxIE(),
         GenericIE()
     ]
 

youtube_dl/extractor/common.py

@@ -211,7 +211,7 @@ class InfoExtractor(object):
             raise ExtractorError(u'Unable to extract %s' % _name)
         else:
             self._downloader.report_warning(u'unable to extract %s; '
-                u'please report this issue on GitHub.' % _name)
+                u'please report this issue on http://yt-dl.org/bug' % _name)
             return None
 
     def _html_search_regex(self, pattern, string, name, default=None, fatal=True, flags=0):

youtube_dl/extractor/googleplus.py

@@ -46,14 +46,18 @@ class GooglePlusIE(InfoExtractor):
         video_title = self._html_search_regex(r'<meta name\=\"Description\" content\=\"(.*?)[\n<"]',
             webpage, 'title', default=u'NA')
 
-        # Step 2, Stimulate clicking the image box to launch video
-        video_page = self._search_regex('"(https\://plus\.google\.com/photos/.*?)",,"image/jpeg","video"\]',
+        # Step 2, Simulate clicking the image box to launch video
+        DOMAIN = 'https://plus.google.com'
+        video_page = self._search_regex(r'<a href="((?:%s)?/photos/.*?)"' % re.escape(DOMAIN),
             webpage, u'video page URL')
+        if not video_page.startswith(DOMAIN):
+            video_page = DOMAIN + video_page
+
         webpage = self._download_webpage(video_page, video_id, u'Downloading video page')
 
         # Extract video links on video page
         """Extract video links of all sizes"""
-        pattern = '\d+,\d+,(\d+),"(http\://redirector\.googlevideo\.com.*?)"'
+        pattern = r'\d+,\d+,(\d+),"(http\://redirector\.googlevideo\.com.*?)"'
         mobj = re.findall(pattern, webpage)
         if len(mobj) == 0:
             raise ExtractorError(u'Unable to extract video links')

youtube_dl/extractor/jukebox.py

@@ -0,0 +1,57 @@
+# coding: utf-8
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    unescapeHTML,
+)
+
+class JukeboxIE(InfoExtractor):
+    _VALID_URL = r'^http://www\.jukebox\.es\/.+[,](?P<video_id>[a-z0-9]+).html'
+    _IFRAME = r'<iframe .*src="(?P<iframe>[^"]*)".*>'
+    _VIDEO_URL = r'"config":{"file":"(?P<video_url>http:[^"]+[.](?P<video_ext>[^.?]+)[?]mdtk=[0-9]+)"'
+    _TITLE = r'<h1 class="inline">(?P<title>[^<]+)</h1>.*<span id="infos_article_artist">(?P<artist>[^<]+)</span>'
+    _NOT_AVAILABLE = r'<span>Este video no está disponible por el momento [!]</span>'
+    _IS_YOUTUBE = r'config":{"file":"(?P<youtube_url>http:[\\][/][\\][/]www[.]youtube[.]com[\\][/]watch[?]v=[^"]+)"'
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('video_id')
+
+        html = self._download_webpage(url, video_id)
+
+        mobj = re.search(self._IFRAME, html)
+        if mobj is None:
+            raise ExtractorError(u'Cannot extract iframe url')
+        iframe_url = unescapeHTML(mobj.group('iframe'))
+
+        iframe_html = self._download_webpage(iframe_url, video_id, 'Downloading iframe')
+        mobj = re.search(self._NOT_AVAILABLE, iframe_html)
+        if mobj is not None:
+            raise ExtractorError(u'Video is not available(in your country?)!')
+
+        self.report_extraction(video_id)
+
+        mobj = re.search(self._VIDEO_URL, iframe_html)
+        if mobj is None:
+            mobj = re.search(self._IS_YOUTUBE, iframe_html)
+            if mobj is None:
+                raise ExtractorError(u'Cannot extract video url')
+            youtube_url = unescapeHTML(mobj.group('youtube_url')).replace('\/','/')
+            self.to_screen(u'Youtube video detected')
+            return self.url_result(youtube_url,ie='Youtube')
+        video_url = unescapeHTML(mobj.group('video_url')).replace('\/','/')
+        video_ext = unescapeHTML(mobj.group('video_ext'))
+
+        mobj = re.search(self._TITLE, html)
+        if mobj is None:
+            raise ExtractorError(u'Cannot extract title')
+        title = unescapeHTML(mobj.group('title'))
+        artist = unescapeHTML(mobj.group('artist'))
+
+        return [{'id': video_id,
+                 'url': video_url,
+                 'title': artist + '-' + title,
+                 'ext': video_ext
+                 }]

youtube_dl/extractor/mtv.py

@@ -27,6 +27,14 @@ class MTVIE(InfoExtractor):
 
         webpage = self._download_webpage(url, video_id)
 
+        # Some videos come from Vevo.com
+        m_vevo = re.search(r'isVevoVideo = true;.*?vevoVideoId = "(.*?)";',
+                           webpage, re.DOTALL)
+        if m_vevo:
+            vevo_id = m_vevo.group(1);
+            self.to_screen(u'Vevo video detected: %s' % vevo_id)
+            return self.url_result('vevo:%s' % vevo_id, ie='Vevo')
+
         #song_name = self._html_search_regex(r'<meta name="mtv_vt" content="([^"]+)"/>',
         #    webpage, u'song name', fatal=False)
 

youtube_dl/extractor/vevo.py

@@ -0,0 +1,44 @@
+import re
+import json
+
+from .common import InfoExtractor
+from ..utils import (
+    unified_strdate,
+    ExtractorError,
+)
+
+class VevoIE(InfoExtractor):
+    """
+    Accecps urls from vevo.com or in the format 'vevo:{id}'
+    (currently used by MTVIE)
+    """
+    _VALID_URL = r'((http://www.vevo.com/watch/.*?/.*?/)|(vevo:))(?P<id>.*)$'
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+
+        json_url = 'http://www.vevo.com/data/video/%s' % video_id
+        base_url = 'http://smil.lvl3.vevo.com'
+        videos_url = '%s/Video/V2/VFILE/%s/%sr.smil' % (base_url, video_id, video_id.lower())
+        info_json = self._download_webpage(json_url, video_id, u'Downloading json info')
+        links_webpage = self._download_webpage(videos_url, video_id, u'Downloading videos urls')
+
+        self.report_extraction(video_id)
+        video_info = json.loads(info_json)
+        m_urls = list(re.finditer(r'<video src="(?P<ext>.*?):(?P<url>.*?)"', links_webpage))
+        if m_urls is None or len(m_urls) == 0:
+            raise ExtractorError(u'Unable to extract video url')
+        # They are sorted from worst to best quality
+        m_url = m_urls[-1]
+        video_url = base_url + m_url.group('url')
+        ext = m_url.group('ext')
+
+        return {'url': video_url,
+                'ext': ext,
+                'id': video_id,
+                'title': video_info['title'],
+                'thumbnail': video_info['img'],
+                'upload_date': video_info['launchDate'].replace('/',''),
+                'uploader': video_info['Artists'][0]['title'],
+                }

youtube_dl/extractor/youtube.py

@@ -129,12 +129,13 @@ class YoutubeIE(InfoExtractor):
         """Indicate the download will use the RTMP protocol."""
         self.to_screen(u'RTMP download detected')
 
-    @staticmethod
-    def _decrypt_signature(s):
+    def _decrypt_signature(self, s):
         """Decrypt the key the two subkeys must have a length of 43"""
         (a,b) = s.split('.')
         if len(a) != 43 or len(b) != 43:
-            raise ExtractorError(u'Unable to decrypt signature, subkeys lengths not valid')
+            raise ExtractorError(u'Unable to decrypt signature, subkeys lengths %d.%d not supported; retrying might work' % (len(a), len(b)))
+        if self._downloader.params.get('verbose'):
+            self.to_screen('encrypted signature length %d.%d' % (len(a), len(b)))
         b = ''.join([b[:8],a[0],b[9:18],b[-4],b[19:39], b[18]])[0:40]
         a = a[-40:]
         s_dec = '.'.join((a,b))[::-1]
@@ -484,11 +485,15 @@ class YoutubeIE(InfoExtractor):
 
         try:
             mobj = re.search(r';ytplayer.config = ({.*?});', video_webpage)
+            if not mobj:
+                raise ValueError('Could not find vevo ID')
             info = json.loads(mobj.group(1))
             args = info['args']
-            if args.get('ptk','') == 'vevo' or 'dashmpd' in args:
-                # Vevo videos with encrypted signatures
-                self.to_screen(u'%s: Vevo video detected.' % video_id)
+            # Easy way to know if the 's' value is in url_encoded_fmt_stream_map
+            # this signatures are encrypted
+            m_s = re.search(r'[&,]s=', args['url_encoded_fmt_stream_map'])
+            if m_s is not None:
+                self.to_screen(u'%s: Encrypted signatures detected.' % video_id)
                 video_info['url_encoded_fmt_stream_map'] = [args['url_encoded_fmt_stream_map']]
         except ValueError:
             pass

youtube_dl/utils.py

@@ -474,7 +474,7 @@ class ExtractorError(Exception):
         """ tb, if given, is the original traceback (so that it can be printed out). """
 
         if not sys.exc_info()[0] in (compat_urllib_error.URLError, socket.timeout, UnavailableVideoError):
-            msg = msg + u'; please report this issue on GitHub.'
+            msg = msg + u'; please report this issue on http://yt-dl.org/bug'
         super(ExtractorError, self).__init__(msg)
 
         self.traceback = tb

youtube_dl/version.py

@@ -1,2 +1,2 @@
 
-__version__ = '2013.06.28'
+__version__ = '2013.06.32'