From 8fb38341ae13fb0de040e63ceeef682726585e1e Mon Sep 17 00:00:00 2001 From: Unknown Date: Wed, 26 Sep 2018 00:43:52 +0200 Subject: [PATCH 1/9] oload.cloud (cloud extension support) --- youtube_dl/extractor/openload.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/openload.py b/youtube_dl/extractor/openload.py index d264fe206..d51d3d2b4 100644 --- a/youtube_dl/extractor/openload.py +++ b/youtube_dl/extractor/openload.py @@ -243,7 +243,7 @@ class PhantomJSwrapper(object): class OpenloadIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?(?:openload\.(?:co|io|link)|oload\.(?:tv|stream|site|xyz|win|download))/(?:f|embed)/(?P[a-zA-Z0-9-_]+)' + _VALID_URL = r'https?://(?:www\.)?(?:openload\.(?:co|io|link)|oload\.(?:tv|stream|site|xyz|win|download|cloud))/(?:f|embed)/(?P[a-zA-Z0-9-_]+)' _TESTS = [{ 'url': 'https://openload.co/f/kUEfGclsU9o', From ea4b8b6747d5d8a72ba5d4943fdaabbc4669740d Mon Sep 17 00:00:00 2001 From: Unknown Date: Tue, 4 Aug 2020 23:19:10 +0200 Subject: [PATCH 2/9] Revert "Merge remote-tracking branch 'origin/master'" This reverts commit 2e6682d80e170e715adc68d5e79b6e946d16e622. --- youtube_dl/extractor/openload.py | 140 ------------------------------- 1 file changed, 140 deletions(-) diff --git a/youtube_dl/extractor/openload.py b/youtube_dl/extractor/openload.py index 1d6dd175b..0c20d0177 100644 --- a/youtube_dl/extractor/openload.py +++ b/youtube_dl/extractor/openload.py @@ -236,143 +236,3 @@ class PhantomJSwrapper(object): self._load_cookies() return (html, encodeArgument(out)) -<<<<<<< HEAD -======= - - -class OpenloadIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?(?:openload\.(?:co|io|link)|oload\.(?:tv|stream|site|xyz|win|download|cloud))/(?:f|embed)/(?P[a-zA-Z0-9-_]+)' - - _TESTS = [{ - 'url': 'https://openload.co/f/kUEfGclsU9o', - 'md5': 'bf1c059b004ebc7a256f89408e65c36e', - 'info_dict': { - 'id': 'kUEfGclsU9o', - 'ext': 'mp4', - 'title': 'skyrim_no-audio_1080.mp4', - 'thumbnail': r're:^https?://.*\.jpg$', - }, - }, { - 'url': 'https://openload.co/embed/rjC09fkPLYs', - 'info_dict': { - 'id': 'rjC09fkPLYs', - 'ext': 'mp4', - 'title': 'movie.mp4', - 'thumbnail': r're:^https?://.*\.jpg$', - 'subtitles': { - 'en': [{ - 'ext': 'vtt', - }], - }, - }, - 'params': { - 'skip_download': True, # test subtitles only - }, - }, { - 'url': 'https://openload.co/embed/kUEfGclsU9o/skyrim_no-audio_1080.mp4', - 'only_matching': True, - }, { - 'url': 'https://openload.io/f/ZAn6oz-VZGE/', - 'only_matching': True, - }, { - 'url': 'https://openload.co/f/_-ztPaZtMhM/', - 'only_matching': True, - }, { - # unavailable via https://openload.co/f/Sxz5sADo82g/, different layout - # for title and ext - 'url': 'https://openload.co/embed/Sxz5sADo82g/', - 'only_matching': True, - }, { - # unavailable via https://openload.co/embed/e-Ixz9ZR5L0/ but available - # via https://openload.co/f/e-Ixz9ZR5L0/ - 'url': 'https://openload.co/f/e-Ixz9ZR5L0/', - 'only_matching': True, - }, { - 'url': 'https://oload.tv/embed/KnG-kKZdcfY/', - 'only_matching': True, - }, { - 'url': 'http://www.openload.link/f/KnG-kKZdcfY', - 'only_matching': True, - }, { - 'url': 'https://oload.stream/f/KnG-kKZdcfY', - 'only_matching': True, - }, { - 'url': 'https://oload.xyz/f/WwRBpzW8Wtk', - 'only_matching': True, - }, { - 'url': 'https://oload.win/f/kUEfGclsU9o', - 'only_matching': True, - }, { - 'url': 'https://oload.download/f/kUEfGclsU9o', - 'only_matching': True, - }, { - # Its title has not got its extension but url has it - 'url': 'https://oload.download/f/N4Otkw39VCw/Tomb.Raider.2018.HDRip.XviD.AC3-EVO.avi.mp4', - 'only_matching': True, - }] - - _USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36' - - @staticmethod - def _extract_urls(webpage): - return re.findall( - r']+src=["\']((?:https?://)?(?:openload\.(?:co|io)|oload\.tv)/embed/[a-zA-Z0-9-_]+)', - webpage) - - def _real_extract(self, url): - video_id = self._match_id(url) - url_pattern = 'https://openload.co/%%s/%s/' % video_id - headers = { - 'User-Agent': self._USER_AGENT, - } - - for path in ('embed', 'f'): - page_url = url_pattern % path - last = path == 'f' - webpage = self._download_webpage( - page_url, video_id, 'Downloading %s webpage' % path, - headers=headers, fatal=last) - if not webpage: - continue - if 'File not found' in webpage or 'deleted by the owner' in webpage: - if not last: - continue - raise ExtractorError('File not found', expected=True, video_id=video_id) - break - - phantom = PhantomJSwrapper(self, required_version='2.0') - webpage, _ = phantom.get(page_url, html=webpage, video_id=video_id, headers=headers) - - decoded_id = (get_element_by_id('streamurl', webpage) or - get_element_by_id('streamuri', webpage) or - get_element_by_id('streamurj', webpage) or - self._search_regex( - (r'>\s*([\w-]+~\d{10,}~\d+\.\d+\.0\.0~[\w-]+)\s*<', - r'>\s*([\w~-]+~\d+\.\d+\.\d+\.\d+~[\w~-]+)', - r'>\s*([\w-]+~\d{10,}~(?:[a-f\d]+:){2}:~[\w-]+)\s*<', - r'>\s*([\w~-]+~[a-f0-9:]+~[\w~-]+)\s*<', - r'>\s*([\w~-]+~[a-f0-9:]+~[\w~-]+)'), webpage, - 'stream URL')) - - video_url = 'https://openload.co/stream/%s?mime=true' % decoded_id - - title = self._og_search_title(webpage, default=None) or self._search_regex( - r']+class=["\']title["\'][^>]*>([^<]+)', webpage, - 'title', default=None) or self._html_search_meta( - 'description', webpage, 'title', fatal=True) - - entries = self._parse_html5_media_entries(page_url, webpage, video_id) - entry = entries[0] if entries else {} - subtitles = entry.get('subtitles') - - info_dict = { - 'id': video_id, - 'title': title, - 'thumbnail': entry.get('thumbnail') or self._og_search_thumbnail(webpage, default=None), - 'url': video_url, - 'ext': determine_ext(title, None) or determine_ext(url, 'mp4'), - 'subtitles': subtitles, - 'http_headers': headers, - } - return info_dict ->>>>>>> master From 66f48768b0740ac2c00bb980a2fe8518fcc22c80 Mon Sep 17 00:00:00 2001 From: Unknown Date: Wed, 5 Aug 2020 19:10:25 +0200 Subject: [PATCH 3/9] sc web-auth api --- youtube_dl/extractor/soundcloud.py | 245 ++++++++++++++++++++++++++++- youtube_dl/extractor/test.py | 52 ++++++ 2 files changed, 296 insertions(+), 1 deletion(-) create mode 100644 youtube_dl/extractor/test.py diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py index d37c52543..6d124c66e 100644 --- a/youtube_dl/extractor/soundcloud.py +++ b/youtube_dl/extractor/soundcloud.py @@ -3,6 +3,8 @@ from __future__ import unicode_literals import itertools import re +import time +import random from .common import ( InfoExtractor, @@ -308,8 +310,249 @@ class SoundcloudIE(InfoExtractor): return False raise + _CLIENT_ID = 'EXLwg5lHTO2dslU5EePe3xkw0m1h86Cd' + _USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36 Edg/84.0.522.50' #MS EDGE CHROME-BASED + _USERNAME = '' + _PASSWORD = '' + _KEY = '0763ed7314c69015fd4a0dc16bbf4b90' + def _real_initialize(self): - self._CLIENT_ID = self._downloader.cache.load('soundcloud', 'client_id') or 'YUKXoArFcqrlQn9tfNHvvyfnDISj04zk' + self._CLIENT_ID, _CLIENT_ID = self._downloader.cache.load('soundcloud', 'client_id') or 'EXLwg5lHTO2dslU5EePe3xkw0m1h86Cd' #'YUKXoArFcqrlQn9tfNHvvyfnDISj04zk' + self._USERNAME, self._PASSWORD = self._get_login_info() + _USERNAME = self._USERNAME + _PASSWORD = self._PASSWORD + self._login() + + def _login(self): + username, password = self._get_login_info() + if username is None: + return + + ''' + EXAMPLE + {"client_id":"EXLwg5lHTO2dslU5EePe3xkw0m1h86Cd","recaptcha_pubkey":"6Ld72JcUAAAAAItDloUGqg6H38KK5j08VuQlegV1","recaptcha_response":null,"credentials":{"identifier":user,"password":password},"signature":"8:33-1-53791-767-2073600-1028-25-25:3b9024:4","device_id":"71542-609622-728970-275967","user_agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36 Edg/84.0.522.50"} + + + call_payload = { + client_id: u.clientId, + recaptcha_pubkey: p, + recaptcha_response: 0 < n.length ? n : null, + credentials: { + identifier: e, + password: t + }, + signature: l.sign(e, u.clientId, c), + device_id: u.deviceId, + user_agent: h + } + ''' + + ''' + var t, n, o = ["touchmove", "webdriver", "__webdriver_script_fn", "Chromium PDF Viewer", "$cdc_asdjflasutopfhvcZLmcfl_", "keydown", "typeof process", "encodeURIComponent", "getAttribute", "document", "unescape", "256", "call", "Chrome PDF Viewer", "_Selenium_IDE_Recorder", "width", "TestUA", "location", "name", "forEach", "length", "Chrome PDF Plugin", "512", "mozInnerScreenY", "Date", "addEventListener", "Native Client", "424242", "exports", "height", "Widevine Content Decryption Module", "mousemove", "keyup", "1024", "420", "https:", "128", "plugins", "charCodeAt", "4096", "typeof global", "undefined", "now", "msLaunchUri", "screen", "eval", "safari", "navigator", "WebKit built-in PDF", "isTrusted"]; + t = o, + n = 219, + function(e) { + for (; --e; ) + t.push(t.shift()) + }(++n); + var g = function(e, t) { + return o[e -= 0] + }; + e[g("0x9")] = { + initialize: function() { + v = window[g("0x5")].now(); + var e = function(e) { + try { + if (!1 === e[g("0x1e")]) + return + } catch (e) {} + w += 1 + }; + window[g("0x6")](g("0xc"), e), + window[g("0x6")](g("0x1f"), e), + window[g("0x6")](g("0x24"), function() { + b += 1 + }), + window.addEventListener(g("0xd"), function() { + k += 1 + }), + window[g("0x6")]("click", function() { + _ += 1 + }) + } + ''' + + ''' + sign: function(e, t, n, o) {e = "tom_heidel@web.de", t = "EXLwg5lHTO2dslU5EePe3xkw0m1h86Cd", n = "0763ed7314c69015fd4a0dc16bbf4b90", o = undefined + r = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36" + } + + ## ignore + #var g = function(e, t) { + # return o[e -= 0] + #}; + #Return value: "$cdc_asdjflasutopfhvcZLmcfl_" + + var s = window[g("0x5")][g("0x17")]() - v #v was created earlier in initialize + g("0x5") = Date + g("0x17") = now + v = Timestamp in milliseconds e.g. 1596580552126 + Return value s: 725321 + + u = 0; + try { + u = window.screen[g("0x2e")] * window[g("0x19")][g("0xa")] + g("0x2e") = width + g("0x19") = screen + g("0xa") = height + Return value u: 2073600 + + var l = +g("0xe"); + l = 1024 + + try { + [][g("0x0")][g("0x2b")](window[g("0x1c")][g("0x12")], function(e) { + var t = e[g("0x31")]; + t === g("0x22") && (l += 1), + t === g("0x2c") && (l += 2), + t === g("0x7") && (l += 4), + t === g("0xb") && (l += 8), + t === g("0x2") && (l += 16), + "Java Appvar Plug-in" === t && (l += 32), + "Shockwave Flash" === t && (l += 64), + "Edge PDF Viewer" === t && (l += +g("0x11")), + t === g("0x1d") && (l += +g("0x2a")) + }) + } + g("0x0") = forEach + g("0x2b") = call + g("0x1c") = navigator + g("0x12") = plugins + + } catch (e) {} + o && (r = g("0x2f")); + o && (i = +g("0x14")); + o && (a = +g("0x14")); + o && (w = 42); + o && (s = +g("0x8")); + o && (u = 420 * +g("0xf")); + o && (l = +g("0xe")); + o && (b = 7); + o && (k = 9); + o && (_ = 0); + + e = "tom_heidel@web.de" + o = undefined, r = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36" + a = 33 + i = 1 + a = 33 + s = 725321 + u = 2073600 + l = 1046 + b = 2 + k = 2 + _ = 5 + w = 1049 + y = "8" + n = "0763ed7314c69015fd4a0dc16bbf4b90" #same as c in l.sign + + for (var d = [a, i, s, w, u, l, b, k].join("-"), c = _, p = n + y + d + r + e + t + d + n, h = window[g("0x29")](window[g("0x26")](p)), m = 8011470, f = 0; f < h[g("0x1")]; f += 1) + m = (m >> 1) + ((1 & m) << 23), + m += h[g("0x13")](f), + m &= 16777215; + return y + ":" + d + ":" + m.toString(16) + ":" + c + } + + d = "33-1-725321-1049-2073600-1046-2-2" + c = 5 + p = "0763ed7314c69015fd4a0dc16bbf4b90833-1-725321-1049-2073600-1046-2-2Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36tom_heidel@web.deEXLwg5lHTO2dslU5EePe3xkw0m1h86Cd33-1-725321-1049-2073600-1046-2-20763ed7314c69015fd4a0dc16bbf4b90" + + g("0x29") = unescape + g("0x26") = encodeURIComponent + + h = "0763ed7314c69015fd4a0dc16bbf4b90833-1-725321-1049-2073600-1046-2-2Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36tom_heidel@web.deEXLwg5lHTO2dslU5EePe3xkw0m1h86Cd33-1-725321-1049-2073600-1046-2-20763ed7314c69015fd4a0dc16bbf4b90" + m = 8011470 + f = 0 + + g("0x1") = length + m = 4005735 #calculated m = (m >> 1) + ((1 & m) << 23) + g("0x13") = charCodeAt + m = 4005783 #calculated m += h[g("0x13")](f) + .-.-.-. after all h.length iterations + m = 245670 + + Return value: "8:33-1-725321-1049-2073600-1046-2-2:3bfa6:5" #signature + + # user-agent string is processed in signature calculating + # this probably means if a signature with user-agent A is given you cannot change it to B when logging in + + ''' + + login_form = { + 'client_id': self._CLIENT_ID, + 'recaptcha_pubkey': 'null', + 'recaptcha_response': 'null', + 'credentials': { + 'identifier': username, + 'password': password + }, + 'signature': '',#l.sign(username, self._CLIENT_ID, c='0763ed7314c69015fd4a0dc16bbf4b90'), #c is extracted from web_auth js around (reading upwards from) function api.callEndpoint("verifyIdentifier" + 'device_id': '00000-000000-000000-000000', + 'user_agent': self._USER_AGENT + } + + ''' + login = self._call_api( + 'sessions.json', None, + 'Logging in', post_data=login_form) + ''' + + # signature generation + def sign(self):#, username=_USERNAME, client_id=_CLIENT_ID, key=_KEY): + zero = 0 + i = a = 1 + u = 0 #u is actually screenWidth * screenHeight for 1920*1080 = 2073600 + mU = 2073600 + l = 1024 #1046? + timestamp = millis = int(round(time.time() * 1000)) + mTimestamp = timestamp - (timestamp - random.randint(50000, 850000)) #hacky timestamp difference + uTimestamp = random.randint(50000, 850000) + w = 42 #1049? + b = k = 2 #25? + underscore = 0 #4, 5? + + + #d = '-'.join([str(mInt) for mInt in [a, i, s, w, u, l, b, k]]) + d = '-'.join([str(mInt) for mInt in [a, i, zero, w, u, l, b, k]]) + print(d) + + c = underscore + + n = _KEY = '0763ed7314c69015fd4a0dc16bbf4b90' + y = '8' #some kind of version?? + r = _USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36" + e = _USERNAME = "tom_heidel@web.de" + t = _CLIENT_ID = 'EXLwg5lHTO2dslU5EePe3xkw0m1h86Cd' + + + p = n + y + d + r + e + t + d + n + + h = p + + m = 8011470 + f = 0 + + for f in range(f, len(h)): + m = (m >> 1) + ((1 & m) << 23) + m += ord(h[f]) + m &= 16777215 + + # c is not even needed + out = str(y) + ':' + str(d) + ':' + format(m, 'x') + ':' + str(c) + + return out + + ### @classmethod def _resolv_url(cls, url): diff --git a/youtube_dl/extractor/test.py b/youtube_dl/extractor/test.py new file mode 100644 index 000000000..3ec71871d --- /dev/null +++ b/youtube_dl/extractor/test.py @@ -0,0 +1,52 @@ +import time +import random + +def sign():#, username=_USERNAME, client_id=_CLIENT_ID, key=_KEY): + zero = 0 + i = a = 1 + mA = 33 + u = 0 #u is actually screenWidth * screenHeight for 1920*1080 = 2073600 + mU = 2073600 + l = 1024 #1046? + mL = 1046 + timestamp = millis = int(round(time.time() * 1000)) + mTimestamp = timestamp - (timestamp - random.randint(50000, 850000)) #hacky timestamp difference + uTimestamp = random.randint(50000, 850000) + w = 42 #1049? + mW = 1049 + b = k = 2 #25? + mB = mK = 25 + underscore = 0 #4, 5? + mUnderscore = 5 + + + #d = '-'.join([str(mInt) for mInt in [a, i, s, w, u, l, b, k]]) + d = '-'.join([str(mInt) for mInt in [mA, i, uTimestamp, mW, mU, mL, mB, mK]]) + print(d) + + c = mUnderscore + + n = _KEY = '0763ed7314c69015fd4a0dc16bbf4b90' + y = '8' #some kind of version?? + r = _USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36" + e = _USERNAME = "tom_heidel@web.de" + t = _CLIENT_ID = 'EXLwg5lHTO2dslU5EePe3xkw0m1h86Cd' + + + p = n + y + d + r + e + t + d + n + + h = p + + m = 8011470 + f = 0 + + for f in range(f, len(h)): + m = (m >> 1) + ((1 & m) << 23) + m += ord(h[f]) + m &= 16777215 + + out = str(y) + ':' + str(d) + ':' + format(m, 'x') + ':' + str(c) + + return out + +print(sign()) \ No newline at end of file From 09cace68c240f0f9ea75f7790484eb17e0cbb0cd Mon Sep 17 00:00:00 2001 From: Unknown Date: Thu, 6 Aug 2020 00:47:37 +0200 Subject: [PATCH 4/9] hard web auth. --- youtube_dl/extractor/soundcloud.py | 4 +- youtube_dl/extractor/test.py | 70 ++++++++++++++++++++++++++++-- 2 files changed, 69 insertions(+), 5 deletions(-) diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py index 6d124c66e..1b9313fa5 100644 --- a/youtube_dl/extractor/soundcloud.py +++ b/youtube_dl/extractor/soundcloud.py @@ -358,7 +358,7 @@ class SoundcloudIE(InfoExtractor): var g = function(e, t) { return o[e -= 0] }; - e[g("0x9")] = { + e[g("0x9")] = { #e = function 162 initialize: function() { v = window[g("0x5")].now(); var e = function(e) { @@ -381,6 +381,8 @@ class SoundcloudIE(InfoExtractor): }) } ''' + #after shifting the list + #["forEach", "length", "Chrome PDF Plugin", "512", "mozInnerScreenY", "Date", "addEventListener", "Native Client", "424242", "exports", "height", "Widevine Content Decryption Module", "mousemove", "keyup", "1024", "420", "https:", "128", "plugins", "charCodeAt", "4096", "typeof global", "undefined", "now", "msLaunchUri", "screen", "eval", "safari", "navigator", "WebKit built-in PDF", "isTrusted", "touchmove", "webdriver", "__webdriver_script_fn", "Chromium PDF Viewer", "$cdc_asdjflasutopfhvcZLmcfl_", "keydown", "typeof process", "encodeURIComponent", "getAttribute", "document", "unescape", "256", "call", "Chrome PDF Viewer", "_Selenium_IDE_Recorder", "width", "TestUA", "location", "name"] ''' sign: function(e, t, n, o) {e = "tom_heidel@web.de", t = "EXLwg5lHTO2dslU5EePe3xkw0m1h86Cd", n = "0763ed7314c69015fd4a0dc16bbf4b90", o = undefined diff --git a/youtube_dl/extractor/test.py b/youtube_dl/extractor/test.py index 3ec71871d..ae8456892 100644 --- a/youtube_dl/extractor/test.py +++ b/youtube_dl/extractor/test.py @@ -8,10 +8,10 @@ def sign():#, username=_USERNAME, client_id=_CLIENT_ID, key=_KEY): u = 0 #u is actually screenWidth * screenHeight for 1920*1080 = 2073600 mU = 2073600 l = 1024 #1046? - mL = 1046 + mL = 1028 timestamp = millis = int(round(time.time() * 1000)) mTimestamp = timestamp - (timestamp - random.randint(50000, 850000)) #hacky timestamp difference - uTimestamp = random.randint(50000, 850000) + uTimestamp = random.randint(50000, 85000) w = 42 #1049? mW = 1049 b = k = 2 #25? @@ -30,13 +30,15 @@ def sign():#, username=_USERNAME, client_id=_CLIENT_ID, key=_KEY): y = '8' #some kind of version?? r = _USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36" e = _USERNAME = "tom_heidel@web.de" - t = _CLIENT_ID = 'EXLwg5lHTO2dslU5EePe3xkw0m1h86Cd' + t = _CLIENT_ID = 'EXLwg5lHTO2dslU5EePe3xkw0m1h86Cd' #'T5R4kgWS2PRf6lzLyIravUMnKlbIxQag' #'EXLwg5lHTO2dslU5EePe3xkw0m1h86Cd' p = n + y + d + r + e + t + d + n h = p + print(h) + m = 8011470 f = 0 @@ -49,4 +51,64 @@ def sign():#, username=_USERNAME, client_id=_CLIENT_ID, key=_KEY): return out -print(sign()) \ No newline at end of file +#print(sign()) + +def signp(a, i, s, w, u, l, b, k, c, n, r, e, t): + ''' + zero = 0 + i = a = 1 + mA = 33 + u = 0 #u is actually screenWidth * screenHeight for 1920*1080 = 2073600 + mU = 2073600 + l = 1024 #1046? + mL = 1028 + timestamp = millis = int(round(time.time() * 1000)) + mTimestamp = timestamp - (timestamp - random.randint(50000, 850000)) #hacky timestamp difference + uTimestamp = random.randint(50000, 85000) + w = 42 #1049? + mW = 1049 + b = k = 2 #25? + mB = mK = 25 + underscore = 0 #4, 5? + mUnderscore = 5 + ''' + + d = '-'.join([str(mInt) for mInt in [a, i, s, w, u, l, b, k]]) + #d = '-'.join([str(mInt) for mInt in [mA, i, uTimestamp, mW, mU, mL, mB, mK]]) + print(d) + + #c = mUnderscore + + #n = _KEY = '0763ed7314c69015fd4a0dc16bbf4b90' + y = '8' #some kind of version?? + rr = _USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36" + ee = _USERNAME = "tom_heidel@web.de" + tt = _CLIENT_ID = 'EXLwg5lHTO2dslU5EePe3xkw0m1h86Cd' #'T5R4kgWS2PRf6lzLyIravUMnKlbIxQag' #'EXLwg5lHTO2dslU5EePe3xkw0m1h86Cd' + + + p = n + y + d + r + e + t + d + n + + h = p + + print(h) + + m = 8011470 + f = 0 + + for f in range(f, len(h)): + m = (m >> 1) + ((1 & m) << 23) + m += ord(h[f]) + m &= 16777215 + + print(m) + + out = str(y) + ':' + str(d) + ':' + format(m, 'x') + ':' + str(c) + + return out + +print(signp(33, 1, 193702, 748, 2073600, 1046, 2, 2, 4, "0763ed7314c69015fd4a0dc16bbf4b90", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36", "tom_heidel@web.de", "T5R4kgWS2PRf6lzLyIravUMnKlbIxQag")) + +#d = "33-1-193702-748-2073600-1046-2-2" +#p = "0763ed7314c69015fd4a0dc16bbf4b90833-1-193702-748-2073600-1046-2-2Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36tom_heidel@web.deT5R4kgWS2PRf6lzLyIravUMnKlbIxQag33-1-193702-748-2073600-1046-2-20763ed7314c69015fd4a0dc16bbf4b90" + +#sig = "8:33-1-193702-748-2073600-1046-2-2:3bfb60:4" \ No newline at end of file From 2ab47fa3a8fb9ce0520649ed264f56be0c825a89 Mon Sep 17 00:00:00 2001 From: Unknown Date: Thu, 6 Aug 2020 15:41:41 +0200 Subject: [PATCH 5/9] updated test. sign in sc.py --- youtube_dl/extractor/soundcloud.py | 239 ++++------------------------- youtube_dl/extractor/test.py | 49 +++++- 2 files changed, 75 insertions(+), 213 deletions(-) diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py index 1b9313fa5..2bc519728 100644 --- a/youtube_dl/extractor/soundcloud.py +++ b/youtube_dl/extractor/soundcloud.py @@ -3,8 +3,7 @@ from __future__ import unicode_literals import itertools import re -import time -import random +import json from .common import ( InfoExtractor, @@ -30,6 +29,7 @@ from ..utils import ( update_url_query, url_or_none, urlhandle_detect_ext, + sanitized_Request, ) @@ -310,187 +310,21 @@ class SoundcloudIE(InfoExtractor): return False raise - _CLIENT_ID = 'EXLwg5lHTO2dslU5EePe3xkw0m1h86Cd' - _USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36 Edg/84.0.522.50' #MS EDGE CHROME-BASED - _USERNAME = '' - _PASSWORD = '' - _KEY = '0763ed7314c69015fd4a0dc16bbf4b90' - def _real_initialize(self): - self._CLIENT_ID, _CLIENT_ID = self._downloader.cache.load('soundcloud', 'client_id') or 'EXLwg5lHTO2dslU5EePe3xkw0m1h86Cd' #'YUKXoArFcqrlQn9tfNHvvyfnDISj04zk' - self._USERNAME, self._PASSWORD = self._get_login_info() - _USERNAME = self._USERNAME - _PASSWORD = self._PASSWORD + self._CLIENT_ID = self._downloader.cache.load('soundcloud', 'client_id') or "T5R4kgWS2PRf6lzLyIravUMnKlbIxQag" #'EXLwg5lHTO2dslU5EePe3xkw0m1h86Cd' #'YUKXoArFcqrlQn9tfNHvvyfnDISj04zk' + self._USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36" self._login() + _API_AUTH_QUERY_TEMPLATE = '?client_id=%s' + _API_AUTH_URL_PW = 'https://api-auth.soundcloud.com/web-auth/sign-in/password%s' + _access_token = None + def _login(self): username, password = self._get_login_info() if username is None: return - ''' - EXAMPLE - {"client_id":"EXLwg5lHTO2dslU5EePe3xkw0m1h86Cd","recaptcha_pubkey":"6Ld72JcUAAAAAItDloUGqg6H38KK5j08VuQlegV1","recaptcha_response":null,"credentials":{"identifier":user,"password":password},"signature":"8:33-1-53791-767-2073600-1028-25-25:3b9024:4","device_id":"71542-609622-728970-275967","user_agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36 Edg/84.0.522.50"} - - - call_payload = { - client_id: u.clientId, - recaptcha_pubkey: p, - recaptcha_response: 0 < n.length ? n : null, - credentials: { - identifier: e, - password: t - }, - signature: l.sign(e, u.clientId, c), - device_id: u.deviceId, - user_agent: h - } - ''' - - ''' - var t, n, o = ["touchmove", "webdriver", "__webdriver_script_fn", "Chromium PDF Viewer", "$cdc_asdjflasutopfhvcZLmcfl_", "keydown", "typeof process", "encodeURIComponent", "getAttribute", "document", "unescape", "256", "call", "Chrome PDF Viewer", "_Selenium_IDE_Recorder", "width", "TestUA", "location", "name", "forEach", "length", "Chrome PDF Plugin", "512", "mozInnerScreenY", "Date", "addEventListener", "Native Client", "424242", "exports", "height", "Widevine Content Decryption Module", "mousemove", "keyup", "1024", "420", "https:", "128", "plugins", "charCodeAt", "4096", "typeof global", "undefined", "now", "msLaunchUri", "screen", "eval", "safari", "navigator", "WebKit built-in PDF", "isTrusted"]; - t = o, - n = 219, - function(e) { - for (; --e; ) - t.push(t.shift()) - }(++n); - var g = function(e, t) { - return o[e -= 0] - }; - e[g("0x9")] = { #e = function 162 - initialize: function() { - v = window[g("0x5")].now(); - var e = function(e) { - try { - if (!1 === e[g("0x1e")]) - return - } catch (e) {} - w += 1 - }; - window[g("0x6")](g("0xc"), e), - window[g("0x6")](g("0x1f"), e), - window[g("0x6")](g("0x24"), function() { - b += 1 - }), - window.addEventListener(g("0xd"), function() { - k += 1 - }), - window[g("0x6")]("click", function() { - _ += 1 - }) - } - ''' - #after shifting the list - #["forEach", "length", "Chrome PDF Plugin", "512", "mozInnerScreenY", "Date", "addEventListener", "Native Client", "424242", "exports", "height", "Widevine Content Decryption Module", "mousemove", "keyup", "1024", "420", "https:", "128", "plugins", "charCodeAt", "4096", "typeof global", "undefined", "now", "msLaunchUri", "screen", "eval", "safari", "navigator", "WebKit built-in PDF", "isTrusted", "touchmove", "webdriver", "__webdriver_script_fn", "Chromium PDF Viewer", "$cdc_asdjflasutopfhvcZLmcfl_", "keydown", "typeof process", "encodeURIComponent", "getAttribute", "document", "unescape", "256", "call", "Chrome PDF Viewer", "_Selenium_IDE_Recorder", "width", "TestUA", "location", "name"] - - ''' - sign: function(e, t, n, o) {e = "tom_heidel@web.de", t = "EXLwg5lHTO2dslU5EePe3xkw0m1h86Cd", n = "0763ed7314c69015fd4a0dc16bbf4b90", o = undefined - r = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36" - } - - ## ignore - #var g = function(e, t) { - # return o[e -= 0] - #}; - #Return value: "$cdc_asdjflasutopfhvcZLmcfl_" - - var s = window[g("0x5")][g("0x17")]() - v #v was created earlier in initialize - g("0x5") = Date - g("0x17") = now - v = Timestamp in milliseconds e.g. 1596580552126 - Return value s: 725321 - - u = 0; - try { - u = window.screen[g("0x2e")] * window[g("0x19")][g("0xa")] - g("0x2e") = width - g("0x19") = screen - g("0xa") = height - Return value u: 2073600 - - var l = +g("0xe"); - l = 1024 - - try { - [][g("0x0")][g("0x2b")](window[g("0x1c")][g("0x12")], function(e) { - var t = e[g("0x31")]; - t === g("0x22") && (l += 1), - t === g("0x2c") && (l += 2), - t === g("0x7") && (l += 4), - t === g("0xb") && (l += 8), - t === g("0x2") && (l += 16), - "Java Appvar Plug-in" === t && (l += 32), - "Shockwave Flash" === t && (l += 64), - "Edge PDF Viewer" === t && (l += +g("0x11")), - t === g("0x1d") && (l += +g("0x2a")) - }) - } - g("0x0") = forEach - g("0x2b") = call - g("0x1c") = navigator - g("0x12") = plugins - - } catch (e) {} - o && (r = g("0x2f")); - o && (i = +g("0x14")); - o && (a = +g("0x14")); - o && (w = 42); - o && (s = +g("0x8")); - o && (u = 420 * +g("0xf")); - o && (l = +g("0xe")); - o && (b = 7); - o && (k = 9); - o && (_ = 0); - - e = "tom_heidel@web.de" - o = undefined, r = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36" - a = 33 - i = 1 - a = 33 - s = 725321 - u = 2073600 - l = 1046 - b = 2 - k = 2 - _ = 5 - w = 1049 - y = "8" - n = "0763ed7314c69015fd4a0dc16bbf4b90" #same as c in l.sign - - for (var d = [a, i, s, w, u, l, b, k].join("-"), c = _, p = n + y + d + r + e + t + d + n, h = window[g("0x29")](window[g("0x26")](p)), m = 8011470, f = 0; f < h[g("0x1")]; f += 1) - m = (m >> 1) + ((1 & m) << 23), - m += h[g("0x13")](f), - m &= 16777215; - return y + ":" + d + ":" + m.toString(16) + ":" + c - } - - d = "33-1-725321-1049-2073600-1046-2-2" - c = 5 - p = "0763ed7314c69015fd4a0dc16bbf4b90833-1-725321-1049-2073600-1046-2-2Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36tom_heidel@web.deEXLwg5lHTO2dslU5EePe3xkw0m1h86Cd33-1-725321-1049-2073600-1046-2-20763ed7314c69015fd4a0dc16bbf4b90" - - g("0x29") = unescape - g("0x26") = encodeURIComponent - - h = "0763ed7314c69015fd4a0dc16bbf4b90833-1-725321-1049-2073600-1046-2-2Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36tom_heidel@web.deEXLwg5lHTO2dslU5EePe3xkw0m1h86Cd33-1-725321-1049-2073600-1046-2-20763ed7314c69015fd4a0dc16bbf4b90" - m = 8011470 - f = 0 - - g("0x1") = length - m = 4005735 #calculated m = (m >> 1) + ((1 & m) << 23) - g("0x13") = charCodeAt - m = 4005783 #calculated m += h[g("0x13")](f) - .-.-.-. after all h.length iterations - m = 245670 - - Return value: "8:33-1-725321-1049-2073600-1046-2-2:3bfa6:5" #signature - - # user-agent string is processed in signature calculating - # this probably means if a signature with user-agent A is given you cannot change it to B when logging in - - ''' - - login_form = { + payload = { 'client_id': self._CLIENT_ID, 'recaptcha_pubkey': 'null', 'recaptcha_response': 'null', @@ -498,47 +332,37 @@ class SoundcloudIE(InfoExtractor): 'identifier': username, 'password': password }, - 'signature': '',#l.sign(username, self._CLIENT_ID, c='0763ed7314c69015fd4a0dc16bbf4b90'), #c is extracted from web_auth js around (reading upwards from) function api.callEndpoint("verifyIdentifier" + 'signature': self.sign(username, password, self._CLIENT_ID), 'device_id': '00000-000000-000000-000000', 'user_agent': self._USER_AGENT } - ''' - login = self._call_api( - 'sessions.json', None, - 'Logging in', post_data=login_form) - ''' + query = self._API_AUTH_QUERY_TEMPLATE % self._CLIENT_ID + login = sanitized_Request(self._API_AUTH_URL_PW % query, json.dumps(payload)) + response = self._download_json(login) + print(response) + return 0 + # signature generation - def sign(self):#, username=_USERNAME, client_id=_CLIENT_ID, key=_KEY): - zero = 0 - i = a = 1 - u = 0 #u is actually screenWidth * screenHeight for 1920*1080 = 2073600 - mU = 2073600 - l = 1024 #1046? - timestamp = millis = int(round(time.time() * 1000)) - mTimestamp = timestamp - (timestamp - random.randint(50000, 850000)) #hacky timestamp difference - uTimestamp = random.randint(50000, 850000) - w = 42 #1049? - b = k = 2 #25? - underscore = 0 #4, 5? - - - #d = '-'.join([str(mInt) for mInt in [a, i, s, w, u, l, b, k]]) - d = '-'.join([str(mInt) for mInt in [a, i, zero, w, u, l, b, k]]) - print(d) - - c = underscore - - n = _KEY = '0763ed7314c69015fd4a0dc16bbf4b90' - y = '8' #some kind of version?? + def sign(self, user, pw, clid="T5R4kgWS2PRf6lzLyIravUMnKlbIxQag"): + a = 33 + i = 1 + s = 440123 + w = 117 + u = 1800000 + l = 1042 + b = 37 + k = 37 + c = 5 + n = _KEY = "0763ed7314c69015fd4a0dc16bbf4b90" + y = _REV = "8" r = _USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36" - e = _USERNAME = "tom_heidel@web.de" - t = _CLIENT_ID = 'EXLwg5lHTO2dslU5EePe3xkw0m1h86Cd' - + e = _USERNAME = user + t = _CLIENT_ID = clid + d = '-'.join([str(mInt) for mInt in [a, i, s, w, u, l, b, k]]) p = n + y + d + r + e + t + d + n - h = p m = 8011470 @@ -554,7 +378,6 @@ class SoundcloudIE(InfoExtractor): return out - ### @classmethod def _resolv_url(cls, url): diff --git a/youtube_dl/extractor/test.py b/youtube_dl/extractor/test.py index ae8456892..36e4fad62 100644 --- a/youtube_dl/extractor/test.py +++ b/youtube_dl/extractor/test.py @@ -1,5 +1,8 @@ import time import random +import requests +import sys +import getpass def sign():#, username=_USERNAME, client_id=_CLIENT_ID, key=_KEY): zero = 0 @@ -75,7 +78,7 @@ def signp(a, i, s, w, u, l, b, k, c, n, r, e, t): d = '-'.join([str(mInt) for mInt in [a, i, s, w, u, l, b, k]]) #d = '-'.join([str(mInt) for mInt in [mA, i, uTimestamp, mW, mU, mL, mB, mK]]) - print(d) + #print(d) #c = mUnderscore @@ -90,7 +93,7 @@ def signp(a, i, s, w, u, l, b, k, c, n, r, e, t): h = p - print(h) + #print(h) m = 8011470 f = 0 @@ -100,15 +103,51 @@ def signp(a, i, s, w, u, l, b, k, c, n, r, e, t): m += ord(h[f]) m &= 16777215 - print(m) + #print(m) out = str(y) + ':' + str(d) + ':' + format(m, 'x') + ':' + str(c) return out -print(signp(33, 1, 193702, 748, 2073600, 1046, 2, 2, 4, "0763ed7314c69015fd4a0dc16bbf4b90", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36", "tom_heidel@web.de", "T5R4kgWS2PRf6lzLyIravUMnKlbIxQag")) +#sig = signp(33, 1, 193702, 748, 2073600, 1046, 2, 2, 4, "0763ed7314c69015fd4a0dc16bbf4b90", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36", "tom_heidel@web.de", "T5R4kgWS2PRf6lzLyIravUMnKlbIxQag") +#print(sig) +#print(signp(33, 1, -1, 748, 2073600, 1046, 2, 2, 4, "0763ed7314c69015fd4a0dc16bbf4b90", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36", "tom_heidel@web.de", "T5R4kgWS2PRf6lzLyIravUMnKlbIxQag")) +#print(signp(33, 1, 440123, 117, 1800000, 1042, 37, 37, 5, "0763ed7314c69015fd4a0dc16bbf4b90", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36", "tom_heidel@web.de", "T5R4kgWS2PRf6lzLyIravUMnKlbIxQag")) #d = "33-1-193702-748-2073600-1046-2-2" #p = "0763ed7314c69015fd4a0dc16bbf4b90833-1-193702-748-2073600-1046-2-2Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36tom_heidel@web.deT5R4kgWS2PRf6lzLyIravUMnKlbIxQag33-1-193702-748-2073600-1046-2-20763ed7314c69015fd4a0dc16bbf4b90" -#sig = "8:33-1-193702-748-2073600-1046-2-2:3bfb60:4" \ No newline at end of file +#sig = "8:33-1-193702-748-2073600-1046-2-2:3bfb60:4" + +login_form_hardcoded = { + 'client_id': "T5R4kgWS2PRf6lzLyIravUMnKlbIxQag", + 'recaptcha_pubkey': 'null', + 'recaptcha_response': 'null', + 'credentials': { + 'identifier': "tom_heidel@web.de", + 'password': ''#getpass.getpass() + }, + 'signature': '',#sig, + 'device_id': '00000-000000-000000-000000', + 'user_agent': "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36" + } + +user = input("User: ") +password = getpass.getpass() +sig_soft = signp(33, 1, 193702, 748, 2073600, 1046, 2, 2, 4, "0763ed7314c69015fd4a0dc16bbf4b90", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36", user, "T5R4kgWS2PRf6lzLyIravUMnKlbIxQag") + +login_form_soft = { + 'client_id': "T5R4kgWS2PRf6lzLyIravUMnKlbIxQag", + 'recaptcha_pubkey': 'null', + 'recaptcha_response': 'null', + 'credentials': { + 'identifier': user, + 'password': password + }, + 'signature': sig_soft, + 'device_id': '00000-000000-000000-000000', + 'user_agent': "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36" + } + +r = requests.post("https://api-auth.soundcloud.com/web-auth/sign-in/password?client_id=T5R4kgWS2PRf6lzLyIravUMnKlbIxQag", json=login_form_soft) +print(r.text) \ No newline at end of file From fb4126a18ca23076e5b3cefcd7ea800d2c19b806 Mon Sep 17 00:00:00 2001 From: Unknown Date: Thu, 6 Aug 2020 23:31:12 +0200 Subject: [PATCH 6/9] working login. aac hq is working if available --- youtube_dl/extractor/soundcloud.py | 32 +++--- youtube_dl/extractor/test.py | 153 ----------------------------- 2 files changed, 19 insertions(+), 166 deletions(-) delete mode 100644 youtube_dl/extractor/test.py diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py index 2bc519728..2d60bfebf 100644 --- a/youtube_dl/extractor/soundcloud.py +++ b/youtube_dl/extractor/soundcloud.py @@ -318,6 +318,8 @@ class SoundcloudIE(InfoExtractor): _API_AUTH_QUERY_TEMPLATE = '?client_id=%s' _API_AUTH_URL_PW = 'https://api-auth.soundcloud.com/web-auth/sign-in/password%s' _access_token = None + _HEADERS = {} + _NETRC_MACHINE = 'soundcloud' def _login(self): username, password = self._get_login_info() @@ -338,10 +340,14 @@ class SoundcloudIE(InfoExtractor): } query = self._API_AUTH_QUERY_TEMPLATE % self._CLIENT_ID - login = sanitized_Request(self._API_AUTH_URL_PW % query, json.dumps(payload)) - response = self._download_json(login) - print(response) - return 0 + login = sanitized_Request(self._API_AUTH_URL_PW % query, json.dumps(payload).encode('utf-8')) + response = self._download_json(login, None) + self._access_token = response.get('session').get('access_token') + if not self._access_token: + self.report_warning('Unable to get access token, login may has failed') + else: + self._HEADERS = {'Authorization': 'OAuth ' + self._access_token} + # signature generation @@ -457,7 +463,7 @@ class SoundcloudIE(InfoExtractor): if not format_url: continue stream = self._download_json( - format_url, track_id, query=query, fatal=False) + format_url, track_id, query=query, fatal=False, headers=self._HEADERS) if not isinstance(stream, dict): continue stream_url = url_or_none(stream.get('url')) @@ -555,7 +561,7 @@ class SoundcloudIE(InfoExtractor): info_json_url = self._resolv_url(self._BASE_URL + resolve_title) info = self._download_json( - info_json_url, full_title, 'Downloading info JSON', query=query) + info_json_url, full_title, 'Downloading info JSON', query=query, headers=self._HEADERS) return self._extract_info_dict(info, full_title, token) @@ -571,7 +577,7 @@ class SoundcloudPlaylistBaseIE(SoundcloudIE): 'ids': ','.join([compat_str(t['id']) for t in tracks]), 'playlistId': playlist_id, 'playlistSecretToken': token, - }) + }, headers=self._HEADERS) entries = [] for track in tracks: track_id = str_or_none(track.get('id')) @@ -615,7 +621,7 @@ class SoundcloudSetIE(SoundcloudPlaylistBaseIE): full_title += '/' + token info = self._download_json(self._resolv_url( - self._BASE_URL + full_title), full_title) + self._BASE_URL + full_title), full_title, headers=self._HEADERS) if 'errors' in info: msgs = (compat_str(err['error_message']) for err in info['errors']) @@ -640,7 +646,7 @@ class SoundcloudPagedPlaylistBaseIE(SoundcloudIE): for i in itertools.count(): response = self._download_json( next_href, playlist_id, - 'Downloading track page %s' % (i + 1), query=query) + 'Downloading track page %s' % (i + 1), query=query, headers=self._HEADERS) collection = response['collection'] @@ -762,7 +768,7 @@ class SoundcloudUserIE(SoundcloudPagedPlaylistBaseIE): user = self._download_json( self._resolv_url(self._BASE_URL + uploader), - uploader, 'Downloading user info') + uploader, 'Downloading user info', headers=self._HEADERS) resource = mobj.group('rsrc') or 'all' @@ -787,7 +793,7 @@ class SoundcloudTrackStationIE(SoundcloudPagedPlaylistBaseIE): def _real_extract(self, url): track_name = self._match_id(url) - track = self._download_json(self._resolv_url(url), track_name) + track = self._download_json(self._resolv_url(url), track_name, headers=self._HEADERS) track_id = self._search_regex( r'soundcloud:track-stations:(\d+)', track['id'], 'track id') @@ -820,7 +826,7 @@ class SoundcloudPlaylistIE(SoundcloudPlaylistBaseIE): data = self._download_json( self._API_V2_BASE + 'playlists/' + playlist_id, - playlist_id, 'Downloading playlist', query=query) + playlist_id, 'Downloading playlist', query=query, headers=self._HEADERS) return self._extract_set(data, token) @@ -857,7 +863,7 @@ class SoundcloudSearchIE(SearchInfoExtractor, SoundcloudIE): for i in itertools.count(1): response = self._download_json( next_url, collection_id, 'Downloading page {0}'.format(i), - 'Unable to download API page') + 'Unable to download API page', headers=self._HEADERS) collection = response.get('collection', []) if not collection: diff --git a/youtube_dl/extractor/test.py b/youtube_dl/extractor/test.py deleted file mode 100644 index 36e4fad62..000000000 --- a/youtube_dl/extractor/test.py +++ /dev/null @@ -1,153 +0,0 @@ -import time -import random -import requests -import sys -import getpass - -def sign():#, username=_USERNAME, client_id=_CLIENT_ID, key=_KEY): - zero = 0 - i = a = 1 - mA = 33 - u = 0 #u is actually screenWidth * screenHeight for 1920*1080 = 2073600 - mU = 2073600 - l = 1024 #1046? - mL = 1028 - timestamp = millis = int(round(time.time() * 1000)) - mTimestamp = timestamp - (timestamp - random.randint(50000, 850000)) #hacky timestamp difference - uTimestamp = random.randint(50000, 85000) - w = 42 #1049? - mW = 1049 - b = k = 2 #25? - mB = mK = 25 - underscore = 0 #4, 5? - mUnderscore = 5 - - - #d = '-'.join([str(mInt) for mInt in [a, i, s, w, u, l, b, k]]) - d = '-'.join([str(mInt) for mInt in [mA, i, uTimestamp, mW, mU, mL, mB, mK]]) - print(d) - - c = mUnderscore - - n = _KEY = '0763ed7314c69015fd4a0dc16bbf4b90' - y = '8' #some kind of version?? - r = _USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36" - e = _USERNAME = "tom_heidel@web.de" - t = _CLIENT_ID = 'EXLwg5lHTO2dslU5EePe3xkw0m1h86Cd' #'T5R4kgWS2PRf6lzLyIravUMnKlbIxQag' #'EXLwg5lHTO2dslU5EePe3xkw0m1h86Cd' - - - p = n + y + d + r + e + t + d + n - - h = p - - print(h) - - m = 8011470 - f = 0 - - for f in range(f, len(h)): - m = (m >> 1) + ((1 & m) << 23) - m += ord(h[f]) - m &= 16777215 - - out = str(y) + ':' + str(d) + ':' + format(m, 'x') + ':' + str(c) - - return out - -#print(sign()) - -def signp(a, i, s, w, u, l, b, k, c, n, r, e, t): - ''' - zero = 0 - i = a = 1 - mA = 33 - u = 0 #u is actually screenWidth * screenHeight for 1920*1080 = 2073600 - mU = 2073600 - l = 1024 #1046? - mL = 1028 - timestamp = millis = int(round(time.time() * 1000)) - mTimestamp = timestamp - (timestamp - random.randint(50000, 850000)) #hacky timestamp difference - uTimestamp = random.randint(50000, 85000) - w = 42 #1049? - mW = 1049 - b = k = 2 #25? - mB = mK = 25 - underscore = 0 #4, 5? - mUnderscore = 5 - ''' - - d = '-'.join([str(mInt) for mInt in [a, i, s, w, u, l, b, k]]) - #d = '-'.join([str(mInt) for mInt in [mA, i, uTimestamp, mW, mU, mL, mB, mK]]) - #print(d) - - #c = mUnderscore - - #n = _KEY = '0763ed7314c69015fd4a0dc16bbf4b90' - y = '8' #some kind of version?? - rr = _USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36" - ee = _USERNAME = "tom_heidel@web.de" - tt = _CLIENT_ID = 'EXLwg5lHTO2dslU5EePe3xkw0m1h86Cd' #'T5R4kgWS2PRf6lzLyIravUMnKlbIxQag' #'EXLwg5lHTO2dslU5EePe3xkw0m1h86Cd' - - - p = n + y + d + r + e + t + d + n - - h = p - - #print(h) - - m = 8011470 - f = 0 - - for f in range(f, len(h)): - m = (m >> 1) + ((1 & m) << 23) - m += ord(h[f]) - m &= 16777215 - - #print(m) - - out = str(y) + ':' + str(d) + ':' + format(m, 'x') + ':' + str(c) - - return out - -#sig = signp(33, 1, 193702, 748, 2073600, 1046, 2, 2, 4, "0763ed7314c69015fd4a0dc16bbf4b90", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36", "tom_heidel@web.de", "T5R4kgWS2PRf6lzLyIravUMnKlbIxQag") -#print(sig) -#print(signp(33, 1, -1, 748, 2073600, 1046, 2, 2, 4, "0763ed7314c69015fd4a0dc16bbf4b90", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36", "tom_heidel@web.de", "T5R4kgWS2PRf6lzLyIravUMnKlbIxQag")) -#print(signp(33, 1, 440123, 117, 1800000, 1042, 37, 37, 5, "0763ed7314c69015fd4a0dc16bbf4b90", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36", "tom_heidel@web.de", "T5R4kgWS2PRf6lzLyIravUMnKlbIxQag")) - -#d = "33-1-193702-748-2073600-1046-2-2" -#p = "0763ed7314c69015fd4a0dc16bbf4b90833-1-193702-748-2073600-1046-2-2Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36tom_heidel@web.deT5R4kgWS2PRf6lzLyIravUMnKlbIxQag33-1-193702-748-2073600-1046-2-20763ed7314c69015fd4a0dc16bbf4b90" - -#sig = "8:33-1-193702-748-2073600-1046-2-2:3bfb60:4" - -login_form_hardcoded = { - 'client_id': "T5R4kgWS2PRf6lzLyIravUMnKlbIxQag", - 'recaptcha_pubkey': 'null', - 'recaptcha_response': 'null', - 'credentials': { - 'identifier': "tom_heidel@web.de", - 'password': ''#getpass.getpass() - }, - 'signature': '',#sig, - 'device_id': '00000-000000-000000-000000', - 'user_agent': "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36" - } - -user = input("User: ") -password = getpass.getpass() -sig_soft = signp(33, 1, 193702, 748, 2073600, 1046, 2, 2, 4, "0763ed7314c69015fd4a0dc16bbf4b90", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36", user, "T5R4kgWS2PRf6lzLyIravUMnKlbIxQag") - -login_form_soft = { - 'client_id': "T5R4kgWS2PRf6lzLyIravUMnKlbIxQag", - 'recaptcha_pubkey': 'null', - 'recaptcha_response': 'null', - 'credentials': { - 'identifier': user, - 'password': password - }, - 'signature': sig_soft, - 'device_id': '00000-000000-000000-000000', - 'user_agent': "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36" - } - -r = requests.post("https://api-auth.soundcloud.com/web-auth/sign-in/password?client_id=T5R4kgWS2PRf6lzLyIravUMnKlbIxQag", json=login_form_soft) -print(r.text) \ No newline at end of file From a58f3e77772aa58513d87e79ec7fd2b0a170a42e Mon Sep 17 00:00:00 2001 From: Unknown Date: Thu, 6 Aug 2020 23:58:43 +0200 Subject: [PATCH 7/9] flake8 --- youtube_dl/extractor/soundcloud.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py index 2d60bfebf..1397547fd 100644 --- a/youtube_dl/extractor/soundcloud.py +++ b/youtube_dl/extractor/soundcloud.py @@ -311,7 +311,7 @@ class SoundcloudIE(InfoExtractor): raise def _real_initialize(self): - self._CLIENT_ID = self._downloader.cache.load('soundcloud', 'client_id') or "T5R4kgWS2PRf6lzLyIravUMnKlbIxQag" #'EXLwg5lHTO2dslU5EePe3xkw0m1h86Cd' #'YUKXoArFcqrlQn9tfNHvvyfnDISj04zk' + self._CLIENT_ID = self._downloader.cache.load('soundcloud', 'client_id') or "T5R4kgWS2PRf6lzLyIravUMnKlbIxQag" # 'EXLwg5lHTO2dslU5EePe3xkw0m1h86Cd' # 'YUKXoArFcqrlQn9tfNHvvyfnDISj04zk' self._USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36" self._login() @@ -331,9 +331,9 @@ class SoundcloudIE(InfoExtractor): 'recaptcha_pubkey': 'null', 'recaptcha_response': 'null', 'credentials': { - 'identifier': username, - 'password': password - }, + 'identifier': username, + 'password': password + }, 'signature': self.sign(username, password, self._CLIENT_ID), 'device_id': '00000-000000-000000-000000', 'user_agent': self._USER_AGENT @@ -348,12 +348,10 @@ class SoundcloudIE(InfoExtractor): else: self._HEADERS = {'Authorization': 'OAuth ' + self._access_token} - - # signature generation def sign(self, user, pw, clid="T5R4kgWS2PRf6lzLyIravUMnKlbIxQag"): a = 33 - i = 1 + i = 1 s = 440123 w = 117 u = 1800000 @@ -372,7 +370,7 @@ class SoundcloudIE(InfoExtractor): h = p m = 8011470 - f = 0 + f = 0 for f in range(f, len(h)): m = (m >> 1) + ((1 & m) << 23) @@ -384,7 +382,6 @@ class SoundcloudIE(InfoExtractor): return out - @classmethod def _resolv_url(cls, url): return SoundcloudIE._API_V2_BASE + 'resolve?url=' + url From c68a4ae6796a4778083ad112647e89901d12bf75 Mon Sep 17 00:00:00 2001 From: Unknown Date: Fri, 7 Aug 2020 01:28:23 +0200 Subject: [PATCH 8/9] auth fix. --- youtube_dl/extractor/soundcloud.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py index 1397547fd..0043ba012 100644 --- a/youtube_dl/extractor/soundcloud.py +++ b/youtube_dl/extractor/soundcloud.py @@ -4,6 +4,7 @@ from __future__ import unicode_literals import itertools import re import json +import random from .common import ( InfoExtractor, @@ -312,9 +313,9 @@ class SoundcloudIE(InfoExtractor): def _real_initialize(self): self._CLIENT_ID = self._downloader.cache.load('soundcloud', 'client_id') or "T5R4kgWS2PRf6lzLyIravUMnKlbIxQag" # 'EXLwg5lHTO2dslU5EePe3xkw0m1h86Cd' # 'YUKXoArFcqrlQn9tfNHvvyfnDISj04zk' - self._USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36" self._login() + _USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36" _API_AUTH_QUERY_TEMPLATE = '?client_id=%s' _API_AUTH_URL_PW = 'https://api-auth.soundcloud.com/web-auth/sign-in/password%s' _access_token = None @@ -326,6 +327,11 @@ class SoundcloudIE(InfoExtractor): if username is None: return + def genDevId(): + def genNumBlock(): + return ''.join([str(random.randrange(10)) for i in range(6)]) + return '-'.join([genNumBlock() for i in range(4)]) + payload = { 'client_id': self._CLIENT_ID, 'recaptcha_pubkey': 'null', @@ -335,7 +341,7 @@ class SoundcloudIE(InfoExtractor): 'password': password }, 'signature': self.sign(username, password, self._CLIENT_ID), - 'device_id': '00000-000000-000000-000000', + 'device_id': genDevId(), 'user_agent': self._USER_AGENT } @@ -349,7 +355,7 @@ class SoundcloudIE(InfoExtractor): self._HEADERS = {'Authorization': 'OAuth ' + self._access_token} # signature generation - def sign(self, user, pw, clid="T5R4kgWS2PRf6lzLyIravUMnKlbIxQag"): + def sign(self, user, pw, clid): a = 33 i = 1 s = 440123 From 1a57b3c03f6c4b0ad25767e9c646fb67ba0b773b Mon Sep 17 00:00:00 2001 From: Unknown Date: Fri, 7 Aug 2020 01:50:00 +0200 Subject: [PATCH 9/9] flake8 fix. --- youtube_dl/extractor/soundcloud.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py index 0043ba012..ac09cb5e6 100644 --- a/youtube_dl/extractor/soundcloud.py +++ b/youtube_dl/extractor/soundcloud.py @@ -365,11 +365,11 @@ class SoundcloudIE(InfoExtractor): b = 37 k = 37 c = 5 - n = _KEY = "0763ed7314c69015fd4a0dc16bbf4b90" - y = _REV = "8" - r = _USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36" - e = _USERNAME = user - t = _CLIENT_ID = clid + n = "0763ed7314c69015fd4a0dc16bbf4b90" # _KEY + y = "8" # _REV + r = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36" # _USER_AGENT + e = user # _USERNAME + t = clid # _CLIENT_ID d = '-'.join([str(mInt) for mInt in [a, i, s, w, u, l, b, k]]) p = n + y + d + r + e + t + d + n