[extractor/safari] Partially Fix Extractor

youtube_dl/extractor/safari.py

@@ -1,15 +1,8 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
 import json
 import re
 
 from .common import InfoExtractor
 
-from ..compat import (
-    compat_parse_qs,
-    compat_urlparse,
-)
 from ..utils import (
     ExtractorError,
     update_url_query,
@@ -25,61 +18,48 @@ class SafariBaseIE(InfoExtractor):
 
     LOGGED_IN = False
 
-    def _real_initialize(self):
+    def is_logged_in(self, fatal=True):
-        self._login()
+        auth, urlh = self._download_json_handle(
+            'https://api.oreilly.com/api/v2/me/', None, note='Checking if logged in', fatal=False,
+            headers={'Accept': 'application/json'}, expected_status=401)
+        if urlh.status == 401:
+            msg = 'Unable to login: %s' % auth['detail']
+            if fatal:
+                raise ExtractorError(msg)
+            self.write_debug(msg)
+            self.LOGGED_IN = False
+            return self.LOGGED_IN
+        self.LOGGED_IN = True
+        return self.LOGGED_IN
 
-    def _login(self):
+    def _initialize_pre_login(self):
-        username, password = self._get_login_info()
+        self.is_logged_in(fatal=False)
-        if username is None:
+
+    def _perform_login(self, username, password):
+
+        if self.LOGGED_IN:
             return
 
-        _, urlh = self._download_webpage_handle(
-            'https://learning.oreilly.com/accounts/login-check/', None,
-            'Downloading login page')
-
-        def is_logged(urlh):
-            return 'learning.oreilly.com/home/' in urlh.geturl()
-
-        if is_logged(urlh):
-            self.LOGGED_IN = True
-            return
-
-        redirect_url = urlh.geturl()
-        parsed_url = compat_urlparse.urlparse(redirect_url)
-        qs = compat_parse_qs(parsed_url.query)
-        next_uri = compat_urlparse.urljoin(
-            'https://api.oreilly.com', qs['next'][0])
-
         auth, urlh = self._download_json_handle(
             'https://www.oreilly.com/member/auth/login/', None, 'Logging in',
             data=json.dumps({
                 'email': username,
-                'password': password,
+                'password': password
-                'redirect_uri': next_uri,
             }).encode(), headers={
-                'Content-Type': 'application/json',
+                'Content-Type': 'application/json'
-                'Referer': redirect_url,
             }, expected_status=400)
 
         credentials = auth.get('credentials')
         if (not auth.get('logged_in') and not auth.get('redirect_uri')
                 and credentials):
-            raise ExtractorError(
+            raise ExtractorError('Unable to login: %s' % credentials, expected=True)
-                'Unable to login: %s' % credentials, expected=True)
 
         # oreilly serves two same instances of the following cookies
         # in Set-Cookie header and expects first one to be actually set
         for cookie in ('groot_sessionid', 'orm-jwt', 'orm-rt'):
             self._apply_first_set_cookie_header(urlh, cookie)
 
-        _, urlh = self._download_webpage_handle(
+        self.is_logged_in()
-            auth.get('redirect_uri') or next_uri, None, 'Completing login',)
-
-        if is_logged(urlh):
-            self.LOGGED_IN = True
-            return
-
-        raise ExtractorError('Unable to log in')
 
 
 class SafariIE(SafariBaseIE):
@@ -96,7 +76,7 @@ class SafariIE(SafariBaseIE):
 
     _TESTS = [{
         'url': 'https://www.safaribooksonline.com/library/view/hadoop-fundamentals-livelessons/9780133392838/part00.html',
-        'md5': 'dcc5a425e79f2564148652616af1f2a3',
+        'md5': '592d0e6d0b03d9b981b3f7306ffc5ca9',
         'info_dict': {
             'id': '0_qbqx90ic',
             'ext': 'mp4',
@@ -104,6 +84,9 @@ class SafariIE(SafariBaseIE):
             'timestamp': 1437758058,
             'upload_date': '20150724',
             'uploader_id': 'stork',
+            'duration': 149,
+            'view_count': int,
+            'thumbnail': 'http://cfvod.kaltura.com/p/1926081/sp/192608100/thumbnail/entry_id/0_qbqx90ic/version/100000'
         },
     }, {
         # non-digits in course id
@@ -127,7 +110,8 @@ class SafariIE(SafariBaseIE):
     _UICONF_ID = '29375172'
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
+        entry_id = ''
+        mobj = self._match_valid_url(url)
 
         reference_id = mobj.group('reference_id')
         if reference_id:
@@ -135,7 +119,7 @@ class SafariIE(SafariBaseIE):
             partner_id = self._PARTNER_ID
             ui_id = self._UICONF_ID
         else:
-            video_id = '%s-%s' % (mobj.group('course_id'), mobj.group('part'))
+            video_id = '{}-{}'.format(mobj.group('course_id'), mobj.group('part'))
 
             webpage, urlh = self._download_webpage_handle(url, video_id)
 
@@ -144,7 +128,12 @@ class SafariIE(SafariBaseIE):
             if not reference_id:
                 reference_id = self._search_regex(
                     r'data-reference-id=(["\'])(?P<id>(?:(?!\1).)+)\1',
-                    webpage, 'kaltura reference id', group='id')
+                    webpage, 'kaltura reference id', default='', group='id')
+                entry_id = self._search_regex(
+                    r'data-entry-id=(["\'])(?P<id>(?:(?!\1).)+)\1',
+                    webpage, 'kaltura entry id', default='', group='id')
+                if not any((entry_id, reference_id)):
+                    raise ExtractorError('Unable to find kaltura "reference id" or "entry id"')
             partner_id = self._search_regex(
                 r'data-partner-id=(["\'])(?P<id>(?:(?!\1).)+)\1',
                 webpage, 'kaltura widget id', default=self._PARTNER_ID,
@@ -155,14 +144,15 @@ class SafariIE(SafariBaseIE):
                 group='id')
 
         query = {
-            'wid': '_%s' % partner_id,
+            'wid': f'_{partner_id}',
             'uiconf_id': ui_id,
-            'flashvars[referenceId]': reference_id,
+            'entry_id': entry_id,
+            'flashvars[referenceId]': reference_id
         }
 
         if self.LOGGED_IN:
             kaltura_session = self._download_json(
-                '%s/player/kaltura_session/?reference_id=%s' % (self._API_BASE, reference_id),
+                f'{self._API_BASE}/player/kaltura_session/?reference_id={reference_id}',
                 video_id, 'Downloading kaltura session JSON',
                 'Unable to download kaltura session JSON', fatal=False,
                 headers={'Accept': 'application/json'})
@@ -189,11 +179,16 @@ class SafariApiIE(SafariBaseIE):
     }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
+        mobj = self._match_valid_url(url)
         part = self._download_json(
-            url, '%s/%s' % (mobj.group('course_id'), mobj.group('part')),
+            url, '{}/{}'.format(mobj.group('course_id'), mobj.group('part')),
             'Downloading part JSON')
-        return self.url_result(part['web_url'], SafariIE.ie_key())
+        web_url = part['web_url']
+        if 'library/view' in web_url:
+            web_url = web_url.replace('library/view', 'videos')
+            natural_keys = part['natural_key']
+            web_url = f'{web_url.rsplit("/", 1)[0]}/{natural_keys[0]}-{natural_keys[1][:-5]}'
+        return self.url_result(web_url, SafariIE.ie_key())
 
 
 class SafariCourseIE(SafariBaseIE):
@@ -242,13 +237,13 @@ class SafariCourseIE(SafariBaseIE):
     @classmethod
     def suitable(cls, url):
         return (False if SafariIE.suitable(url) or SafariApiIE.suitable(url)
-                else super(SafariCourseIE, cls).suitable(url))
+                else super().suitable(url))
 
     def _real_extract(self, url):
         course_id = self._match_id(url)
 
         course_json = self._download_json(
-            '%s/book/%s/?override_format=%s' % (self._API_BASE, course_id, self._API_FORMAT),
+            f'{self._API_BASE}/book/{course_id}/?override_format={self._API_FORMAT}',
             course_id, 'Downloading course JSON')
 
         if 'chapters' not in course_json: