From 17dca698cf8032e1a41bdc5d42cdabf5736a3266 Mon Sep 17 00:00:00 2001 From: pgaig Date: Thu, 22 Jul 2021 09:58:39 +0200 Subject: [PATCH 1/4] add XSRF Token logon to vrtnu --- youtube_dl/extractor/canvas.py | 63 +++++++++++++++++++++------------- 1 file changed, 39 insertions(+), 24 deletions(-) diff --git a/youtube_dl/extractor/canvas.py b/youtube_dl/extractor/canvas.py index eefbab241..7164ead8f 100644 --- a/youtube_dl/extractor/canvas.py +++ b/youtube_dl/extractor/canvas.py @@ -17,6 +17,7 @@ from ..utils import ( str_or_none, strip_or_none, url_or_none, + urlencode_postdata ) @@ -259,7 +260,7 @@ class VrtNUIE(GigyaBaseIE): 'expected_warnings': ['Unable to download asset JSON', 'is not a supported codec', 'Unknown MIME type'], }] _NETRC_MACHINE = 'vrtnu' - _APIKEY = '3_0Z2HujMtiWq_pkAjgnS2Md2E11a1AwZjYiBETtwNE-EoEHDINgtnvcAOpNgmrVGy' + _APIKEY = '3_qhEcPa5JGFROVwu5SWKqJ4mVOIkwlFNMSKwzPDAh8QZOtHqu6L4nD5Q7lk0eXOOG' _CONTEXT_ID = 'R3595707040' def _real_initialize(self): @@ -270,35 +271,49 @@ class VrtNUIE(GigyaBaseIE): if username is None: return - auth_data = { - 'APIKey': self._APIKEY, - 'targetEnv': 'jssdk', - 'loginID': username, - 'password': password, - 'authMode': 'cookie', - } - - auth_info = self._gigya_login(auth_data) + # 1. Get Login Data + try: + auth_info = self._download_json( + 'https://accounts.vrt.be/accounts.login', None, + note='Login data', errnote='Could not get Login data', + headers={}, data=urlencode_postdata({ + 'loginID':username, + 'password':password, + 'sessionExpiration':'-2', + 'APIKey':self._APIKEY, + 'targetEnv':'jssdk', + })) + except ExtractorError as e: + self.report_warning("Cookie request HTTP code: %s" % e.cause.code) + raise e # Sometimes authentication fails for no good reason, retry login_attempt = 1 while login_attempt <= 3: try: - # When requesting a token, no actual token is returned, but the - # necessary cookies are set. - self._request_webpage( - 'https://token.vrt.be', + # get XSRF Token + xsrf_req = self._request_webpage('https://token.vrt.be/vrtnuinitlogin', + None, note='Requesting XSRF Token', errnote='Could not get XSRF Token', + query={'provider':'site', 'destination':'https://www.vrt.be/vrtnu/'}) + + post_data = { + 'UID': auth_info['UID'], + 'UIDSignature': auth_info['UIDSignature'], + 'signatureTimestamp': auth_info['signatureTimestamp'], + 'client_id': 'vrtnu-site', + } + for cookie in self._downloader.cookiejar: + if cookie.name == 'OIDCXSRF': + post_data['_csrf'] = cookie.value + break + + # do login + vrt_token = self._request_webpage( + 'https://login.vrt.be/perform_login', None, note='Requesting a token', errnote='Could not get a token', - headers={ - 'Content-Type': 'application/json', - 'Referer': 'https://www.vrt.be/vrtnu/', - }, - data=json.dumps({ - 'uid': auth_info['UID'], - 'uidsig': auth_info['UIDSignature'], - 'ts': auth_info['signatureTimestamp'], - 'email': auth_info['profile']['email'], - }).encode('utf-8')) + headers={}, data=urlencode_postdata(post_data) + ) + except ExtractorError as e: if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401: login_attempt += 1 From 03d89133f32a77d424b61279b4714778d06d9c15 Mon Sep 17 00:00:00 2001 From: pgaig Date: Thu, 22 Jul 2021 10:10:16 +0200 Subject: [PATCH 2/4] flake8 checks --- youtube_dl/extractor/canvas.py | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/youtube_dl/extractor/canvas.py b/youtube_dl/extractor/canvas.py index 7164ead8f..477cedabd 100644 --- a/youtube_dl/extractor/canvas.py +++ b/youtube_dl/extractor/canvas.py @@ -1,7 +1,6 @@ from __future__ import unicode_literals import re -import json from .common import InfoExtractor from .gigya import GigyaBaseIE @@ -277,11 +276,11 @@ class VrtNUIE(GigyaBaseIE): 'https://accounts.vrt.be/accounts.login', None, note='Login data', errnote='Could not get Login data', headers={}, data=urlencode_postdata({ - 'loginID':username, - 'password':password, - 'sessionExpiration':'-2', - 'APIKey':self._APIKEY, - 'targetEnv':'jssdk', + 'loginID': username, + 'password': password, + 'sessionExpiration': '-2', + 'APIKey': self._APIKEY, + 'targetEnv': 'jssdk', })) except ExtractorError as e: self.report_warning("Cookie request HTTP code: %s" % e.cause.code) @@ -292,10 +291,10 @@ class VrtNUIE(GigyaBaseIE): while login_attempt <= 3: try: # get XSRF Token - xsrf_req = self._request_webpage('https://token.vrt.be/vrtnuinitlogin', - None, note='Requesting XSRF Token', errnote='Could not get XSRF Token', - query={'provider':'site', 'destination':'https://www.vrt.be/vrtnu/'}) - + self._request_webpage('https://token.vrt.be/vrtnuinitlogin', + None, note='Requesting XSRF Token', errnote='Could not get XSRF Token', + query={'provider': 'site', 'destination': 'https://www.vrt.be/vrtnu/'}) + post_data = { 'UID': auth_info['UID'], 'UIDSignature': auth_info['UIDSignature'], @@ -308,12 +307,12 @@ class VrtNUIE(GigyaBaseIE): break # do login - vrt_token = self._request_webpage( + self._request_webpage( 'https://login.vrt.be/perform_login', None, note='Requesting a token', errnote='Could not get a token', headers={}, data=urlencode_postdata(post_data) ) - + except ExtractorError as e: if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401: login_attempt += 1 From 944a8f2b0b1278ca7eb13523dcd76e91fe121d7f Mon Sep 17 00:00:00 2001 From: pgaig Date: Sat, 31 Jul 2021 22:22:07 +0200 Subject: [PATCH 3/4] fix styling remarks from yt-dlp project --- youtube_dl/extractor/canvas.py | 30 +++++++++++------------------- 1 file changed, 11 insertions(+), 19 deletions(-) diff --git a/youtube_dl/extractor/canvas.py b/youtube_dl/extractor/canvas.py index 477cedabd..f04aaf007 100644 --- a/youtube_dl/extractor/canvas.py +++ b/youtube_dl/extractor/canvas.py @@ -270,27 +270,21 @@ class VrtNUIE(GigyaBaseIE): if username is None: return - # 1. Get Login Data - try: - auth_info = self._download_json( - 'https://accounts.vrt.be/accounts.login', None, - note='Login data', errnote='Could not get Login data', - headers={}, data=urlencode_postdata({ - 'loginID': username, - 'password': password, - 'sessionExpiration': '-2', - 'APIKey': self._APIKEY, - 'targetEnv': 'jssdk', - })) - except ExtractorError as e: - self.report_warning("Cookie request HTTP code: %s" % e.cause.code) - raise e + auth_info = self._download_json( + 'https://accounts.vrt.be/accounts.login', None, + note='Login data', errnote='Could not get Login data', + headers={}, data=urlencode_postdata({ + 'loginID': username, + 'password': password, + 'sessionExpiration': '-2', + 'APIKey': self._APIKEY, + 'targetEnv': 'jssdk', + })) # Sometimes authentication fails for no good reason, retry login_attempt = 1 while login_attempt <= 3: try: - # get XSRF Token self._request_webpage('https://token.vrt.be/vrtnuinitlogin', None, note='Requesting XSRF Token', errnote='Could not get XSRF Token', query={'provider': 'site', 'destination': 'https://www.vrt.be/vrtnu/'}) @@ -306,12 +300,10 @@ class VrtNUIE(GigyaBaseIE): post_data['_csrf'] = cookie.value break - # do login self._request_webpage( 'https://login.vrt.be/perform_login', None, note='Requesting a token', errnote='Could not get a token', - headers={}, data=urlencode_postdata(post_data) - ) + headers={}, data=urlencode_postdata(post_data)) except ExtractorError as e: if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401: From e0556cd956b62b9ea10e2a9478c1e1fad856c6e6 Mon Sep 17 00:00:00 2001 From: pgaig Date: Tue, 3 Aug 2021 08:48:41 +0200 Subject: [PATCH 4/4] cleaner access to the OIDCXSRF cookie --- youtube_dl/extractor/canvas.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/youtube_dl/extractor/canvas.py b/youtube_dl/extractor/canvas.py index f04aaf007..13d3ea3c2 100644 --- a/youtube_dl/extractor/canvas.py +++ b/youtube_dl/extractor/canvas.py @@ -294,11 +294,8 @@ class VrtNUIE(GigyaBaseIE): 'UIDSignature': auth_info['UIDSignature'], 'signatureTimestamp': auth_info['signatureTimestamp'], 'client_id': 'vrtnu-site', + '_csrf': self._get_cookies('https://login.vrt.be/').get('OIDCXSRF').value } - for cookie in self._downloader.cookiejar: - if cookie.name == 'OIDCXSRF': - post_data['_csrf'] = cookie.value - break self._request_webpage( 'https://login.vrt.be/perform_login',