From 05698ebf9b7264ada5b1a3a29ca5b508f87262b9 Mon Sep 17 00:00:00 2001 From: Nick Lai Date: Mon, 6 Apr 2020 16:45:20 +0800 Subject: [PATCH 1/5] [MicrosoftStream] Add new extractor --- youtube_dl/extractor/extractors.py | 1 + youtube_dl/extractor/microsoftstream.py | 133 ++++++++++++++++++++++++ 2 files changed, 134 insertions(+) create mode 100644 youtube_dl/extractor/microsoftstream.py diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py index e407ab3d9..847a58f16 100644 --- a/youtube_dl/extractor/extractors.py +++ b/youtube_dl/extractor/extractors.py @@ -619,6 +619,7 @@ from .metacritic import MetacriticIE from .mgoon import MgoonIE from .mgtv import MGTVIE from .miaopai import MiaoPaiIE +from .microsoftstream import MicrosoftStreamIE from .microsoftvirtualacademy import ( MicrosoftVirtualAcademyIE, MicrosoftVirtualAcademyCourseIE, diff --git a/youtube_dl/extractor/microsoftstream.py b/youtube_dl/extractor/microsoftstream.py new file mode 100644 index 000000000..11a0b2202 --- /dev/null +++ b/youtube_dl/extractor/microsoftstream.py @@ -0,0 +1,133 @@ +# coding: utf-8 +from __future__ import unicode_literals +from .common import InfoExtractor +from ..utils import ExtractorError + + +class MicrosoftStreamBaseIE(InfoExtractor): + _LOGIN_URL = 'https://web.microsoftstream.com/?noSignUpCheck=1' # expect redirection + _EXPECTED_TITLE = 'Microsoft Stream' + + def is_logged_in(self, webpage): + return self._EXPECTED_TITLE in webpage + + def _real_initialize(self): + username, password = self._get_login_info() + + if username is not None or password is not None: + raise ExtractorError('MicrosoftStream Extractor does not support username/password log-in at the moment. Please use cookies log-in instead. See https://github.com/ytdl-org/youtube-dl/blob/master/README.md#how-do-i-pass-cookies-to-youtube-dl for more information') + + +class MicrosoftStreamIE(MicrosoftStreamBaseIE): + IE_NAME = 'microsoftstream' + _VALID_URL = r'https?://(?:(?:web|www)\.)?microsoftstream\.com/video/(?P[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})' # https://regex101.com/r/K1mlgK/1/ + _NETRC_MACHINE = 'microsoftstream' + + _TEST = { + 'url': 'https://web.microsoftstream.com/video/c883c6a5-9895-4900-9a35-62f4b5d506c9', + 'info_dict': { + 'id': 'c883c6a5-9895-4900-9a35-62f4b5d506c9', + 'ext': 'mp4', + 'title': 'Webinar for Researchers: Use of GitLab', + 'thumbnail': r're:^https?://.*$', + } + } + + def _remap_thumbnails(self, thumbnail_dict_list): + output = [] + preference_index = ['extraSmall', 'small', 'medium', 'large'] + + for _, key in enumerate(thumbnail_dict_list): + output.append({ + 'preference': preference_index.index(key), + 'url': thumbnail_dict_list[key]['url'] + }) + return output + + def _remap_playback(self, master_playlist_urls, video_id, http_headers={}): + """ + A parser for the HLS and MPD playlists from the API endpoint. + """ + output = [] + + for master_playlist_url in master_playlist_urls: + # Handle HLS Master playlist + if self._determine_protocol(master_playlist_url['mimeType']) == 'm3u8': + varient_playlists = self._extract_m3u8_formats(master_playlist_url['playbackUrl'], video_id, headers=http_headers) + + # For MPEG-DASH Master playlists + elif self._determine_protocol(master_playlist_url['mimeType']) == 'http_dash_segments': + varient_playlists = self._extract_mpd_formats(master_playlist_url['playbackUrl'], video_id, headers=http_headers) + + else: + self.to_screen('Found unresolvable stream with format %s' % master_playlist_url['mimeType']) + continue + + # Patching the "Authorization" header + for varient_playlist in varient_playlists: + varient_playlist['http_headers'] = http_headers + output.append(varient_playlist) + + return output + + def _determine_protocol(self, mime): + """ + A switch board for the MIME type provided from the API endpoint. + """ + if mime in ['application/dash+xml']: + return 'http_dash_segments' + elif mime in ['application/vnd.apple.mpegurl']: + return 'm3u8' + else: + return None + + def _remap_texttracks(self, tracks): + """ + A parser for the texttracks response. + """ + subtitle = {} + automatic_captions = {} + for track in tracks: + if track['autoGenerated'] is True: + if track['language'] not in automatic_captions: + automatic_captions[track['language']] = [] + automatic_captions[track['language']].append({'url': track['url']}) + else: + if track['language'] not in subtitle: + subtitle[track['language']] = [] + subtitle[track['language']].append({'url': track['url']}) + return (subtitle, automatic_captions) + + def _real_extract(self, url): + video_id = self._match_id(url) + webpage = self._download_webpage(url, video_id) + + if not self.is_logged_in(webpage): + return self.raise_login_required() + + # Extract access token from webpage + accessToken = self._html_search_regex(r"\"AccessToken\":\"(?P.+?)\"", webpage, 'AccessToken') + apiGateway = self._html_search_regex(r"\"ApiGatewayUri\":\"(?P.+?)\"", webpage, 'APIGateway') + headers = {'Authorization': 'Bearer %s' % accessToken} + + # "GET" api for video information + apiUri = "%s/videos/%s?$expand=creator,tokens,status,liveEvent,extensions&api-version=1.3-private" % (apiGateway, video_id) + apiCall = self._download_json(apiUri, video_id, headers=headers) + + # "GET" api for subtitles and auto-captions + texttracksUri = "%s/videos/%s/texttracks?api-version=1.3-private" % (apiGateway, video_id) + texttracksCall = self._download_json(texttracksUri, video_id, headers=headers)['value'] + subtitles, automatic_captions = self._remap_texttracks(texttracksCall) + + return { + 'id': video_id, + 'title': apiCall['name'], + 'description': apiCall['description'], + 'uploader': apiCall['creator']['name'], + 'thumbnails': self._remap_thumbnails(apiCall['posterImage']), + 'formats': self._remap_playback(apiCall['playbackUrls'], video_id, http_headers=headers), + 'subtitles': subtitles, + 'automatic_captions': automatic_captions, + 'is_live': False, + # 'duration': apiCall['media']['duration'], + } From 5416301787f048579f30fda25dc234dc3d52f722 Mon Sep 17 00:00:00 2001 From: Nick Lai Date: Fri, 10 Apr 2020 17:57:10 +0800 Subject: [PATCH 2/5] [MicrosoftStream] Fixed code style and refactor --- youtube_dl/extractor/microsoftstream.py | 205 ++++++++++++++++-------- 1 file changed, 142 insertions(+), 63 deletions(-) diff --git a/youtube_dl/extractor/microsoftstream.py b/youtube_dl/extractor/microsoftstream.py index 11a0b2202..c152e24a9 100644 --- a/youtube_dl/extractor/microsoftstream.py +++ b/youtube_dl/extractor/microsoftstream.py @@ -17,11 +17,36 @@ class MicrosoftStreamBaseIE(InfoExtractor): if username is not None or password is not None: raise ExtractorError('MicrosoftStream Extractor does not support username/password log-in at the moment. Please use cookies log-in instead. See https://github.com/ytdl-org/youtube-dl/blob/master/README.md#how-do-i-pass-cookies-to-youtube-dl for more information') + """ + Extraction Helper + """ + + def _extract_access_token(self, webpage): + """ + Extract the JWT access token with Regex + """ + self._ACCESS_TOKEN = self._html_search_regex(r"\"AccessToken\":\"(?P.+?)\"", webpage, 'AccessToken') + return self._ACCESS_TOKEN + + def _extract_api_gateway(self, webpage): + """ + Extract the API gateway with Regex + """ + self._API_GATEWAY = self._html_search_regex(r"\"ApiGatewayUri\":\"(?P.+?)\"", webpage, 'APIGateway') + return self._API_GATEWAY + class MicrosoftStreamIE(MicrosoftStreamBaseIE): + """ + Extract of single Microsoft Stream video + """ IE_NAME = 'microsoftstream' _VALID_URL = r'https?://(?:(?:web|www)\.)?microsoftstream\.com/video/(?P[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})' # https://regex101.com/r/K1mlgK/1/ _NETRC_MACHINE = 'microsoftstream' + _ACCESS_TOKEN = None # A JWT token + _API_GATEWAY = None + _TEXTTRACKS_RESPONSE = None + _VIDEO_ID = None _TEST = { 'url': 'https://web.microsoftstream.com/video/c883c6a5-9895-4900-9a35-62f4b5d506c9', @@ -33,42 +58,49 @@ class MicrosoftStreamIE(MicrosoftStreamBaseIE): } } - def _remap_thumbnails(self, thumbnail_dict_list): - output = [] - preference_index = ['extraSmall', 'small', 'medium', 'large'] + """ + Getters - for _, key in enumerate(thumbnail_dict_list): - output.append({ - 'preference': preference_index.index(key), - 'url': thumbnail_dict_list[key]['url'] - }) - return output + The following getters include helpful message to prompt developers for potential errors. + """ + @property + def api_gateway(self): + if self._API_GATEWAY is None: + raise ExtractorError('API gateway is None. Did you forget to call "_extract_api_gateway"?') + return self._API_GATEWAY - def _remap_playback(self, master_playlist_urls, video_id, http_headers={}): + @property + def access_token(self): + if self._ACCESS_TOKEN is None: + raise ExtractorError('Access token is None. Did you forget to call "_extract_access_token"?') + + return self._ACCESS_TOKEN + + @property + def video_id(self): + if self._VIDEO_ID is None: + raise('Variable "_VIDEO_ID" is not defined. Did you make the main extraction call?') + return self._VIDEO_ID + + @property + def headers(self): + return {'Authorization': 'Bearer %s' % self.access_token} + + @property + def texttrack_info_endpoint(self): + return "%s/videos/%s/texttracks?api-version=1.3-private" % (self.api_gateway, self.video_id) + + @property + def media_info_endpoint(self): + return "%s/videos/%s?$expand=creator,tokens,status,liveEvent,extensions&api-version=1.3-private" % (self.api_gateway, self.video_id) + + def _request_texttracks(self): """ - A parser for the HLS and MPD playlists from the API endpoint. + Make an additional request to Microsoft Stream for the subtitle and auto-caption """ - output = [] - - for master_playlist_url in master_playlist_urls: - # Handle HLS Master playlist - if self._determine_protocol(master_playlist_url['mimeType']) == 'm3u8': - varient_playlists = self._extract_m3u8_formats(master_playlist_url['playbackUrl'], video_id, headers=http_headers) - - # For MPEG-DASH Master playlists - elif self._determine_protocol(master_playlist_url['mimeType']) == 'http_dash_segments': - varient_playlists = self._extract_mpd_formats(master_playlist_url['playbackUrl'], video_id, headers=http_headers) - - else: - self.to_screen('Found unresolvable stream with format %s' % master_playlist_url['mimeType']) - continue - - # Patching the "Authorization" header - for varient_playlist in varient_playlists: - varient_playlist['http_headers'] = http_headers - output.append(varient_playlist) - - return output + # Map default variable + self._TEXTTRACKS_RESPONSE = self._download_json(self.texttrack_info_endpoint, self.video_id, headers=self.headers)['value'] + return self._TEXTTRACKS_RESPONSE def _determine_protocol(self, mime): """ @@ -81,53 +113,100 @@ class MicrosoftStreamIE(MicrosoftStreamBaseIE): else: return None - def _remap_texttracks(self, tracks): + def _remap_thumbnails(self, thumbnail_dict_list): + output = [] + preference_index = ['extraSmall', 'small', 'medium', 'large'] + + for _, key in enumerate(thumbnail_dict_list): + output.append({ + 'preference': preference_index.index(key), + 'url': thumbnail_dict_list.get(key).get('url') + }) + return output + + def _remap_playback(self, master_playlist_urls): """ - A parser for the texttracks response. + A parser for the HLS and MPD playlists from the API endpoint. """ - subtitle = {} - automatic_captions = {} - for track in tracks: - if track['autoGenerated'] is True: - if track['language'] not in automatic_captions: - automatic_captions[track['language']] = [] - automatic_captions[track['language']].append({'url': track['url']}) + output = [] + + for master_playlist_url in master_playlist_urls: + protocol = self._determine_protocol(master_playlist_url['mimeType']) + # Handle HLS Master playlist + if protocol == 'm3u8': + varient_playlists = self._extract_m3u8_formats(master_playlist_url['playbackUrl'], video_id=self.video_id, headers=self.headers) + + # For MPEG-DASH Master playlists + elif protocol == 'http_dash_segments': + varient_playlists = self._extract_mpd_formats(master_playlist_url['playbackUrl'], video_id=self.video_id, headers=self.headers) + + # For other Master playlists (like Microsoft Smooth Streaming) else: - if track['language'] not in subtitle: - subtitle[track['language']] = [] - subtitle[track['language']].append({'url': track['url']}) - return (subtitle, automatic_captions) + self.to_screen('Found unresolvable stream with format %s' % master_playlist_url['mimeType']) + continue + + # Patching the "Authorization" header + for varient_playlist in varient_playlists: + varient_playlist['http_headers'] = self.headers + output.append(varient_playlist) + return output + + def _extract_subtitle(self, tracks, is_auto_generated): + """ + An internal method for filtering and remapping text tracks + """ + if type(is_auto_generated) is not bool: + raise ExtractorError('Unexpected variable "is_auto_generated" type: must be a Boolean') + + subtitle_subset = {} + + for track in tracks: + track_language = track.get('language') # The track language must have a language code. + + if track.get('autoGenerated') is is_auto_generated: + if track_language not in subtitle_subset: + subtitle_subset[track_language] = [] # Scaffold an empty list for the object to insert into + + # Since the subtitle is token protected, a get request will fire here. + data = self._download_webpage(url_or_request=track.get('url'), video_id=self.video_id, headers=self.headers) + subtitle_subset[track_language].append({'data': data, "ext": "vtt"}) + + return subtitle_subset + + def _get_subtitles(self, tracks=None): # Fulfill abstract method + tracks = self._TEXTTRACKS_RESPONSE if tracks is None else tracks + return self._extract_subtitle(tracks, False) + + def _get_automatic_captions(self, tracks=None): # Fulfill abstract method + tracks = self._TEXTTRACKS_RESPONSE if tracks is None else tracks + return self._extract_subtitle(tracks, True) def _real_extract(self, url): - video_id = self._match_id(url) - webpage = self._download_webpage(url, video_id) + self._VIDEO_ID = self._match_id(url) + webpage = self._download_webpage(url, self.video_id) if not self.is_logged_in(webpage): return self.raise_login_required() # Extract access token from webpage - accessToken = self._html_search_regex(r"\"AccessToken\":\"(?P.+?)\"", webpage, 'AccessToken') - apiGateway = self._html_search_regex(r"\"ApiGatewayUri\":\"(?P.+?)\"", webpage, 'APIGateway') - headers = {'Authorization': 'Bearer %s' % accessToken} + self._extract_access_token(webpage) + self._extract_api_gateway(webpage) # "GET" api for video information - apiUri = "%s/videos/%s?$expand=creator,tokens,status,liveEvent,extensions&api-version=1.3-private" % (apiGateway, video_id) - apiCall = self._download_json(apiUri, video_id, headers=headers) + apiUri = self.media_info_endpoint + apiCall = self._download_json(apiUri, self.video_id, headers=self.headers) - # "GET" api for subtitles and auto-captions - texttracksUri = "%s/videos/%s/texttracks?api-version=1.3-private" % (apiGateway, video_id) - texttracksCall = self._download_json(texttracksUri, video_id, headers=headers)['value'] - subtitles, automatic_captions = self._remap_texttracks(texttracksCall) + texttracks = self._request_texttracks() return { - 'id': video_id, + 'id': self.video_id, 'title': apiCall['name'], - 'description': apiCall['description'], - 'uploader': apiCall['creator']['name'], - 'thumbnails': self._remap_thumbnails(apiCall['posterImage']), - 'formats': self._remap_playback(apiCall['playbackUrls'], video_id, http_headers=headers), - 'subtitles': subtitles, - 'automatic_captions': automatic_captions, + 'description': apiCall.get('description'), + 'uploader': apiCall.get('creator').get('name'), + 'thumbnails': self._remap_thumbnails(apiCall.get('posterImage')), + 'formats': self._remap_playback(apiCall['playbackUrls']), + 'subtitles': self._get_subtitles(texttracks), + 'automatic_captions': self._get_automatic_captions(texttracks), 'is_live': False, # 'duration': apiCall['media']['duration'], } From 292be92987999fe5bcd28dd782bb41e0a08cec4f Mon Sep 17 00:00:00 2001 From: Nick Lai Date: Fri, 10 Apr 2020 18:15:55 +0800 Subject: [PATCH 3/5] [MicrosoftStream] Fix code style, add test --- youtube_dl/extractor/microsoftstream.py | 46 +++++++++++++++---------- 1 file changed, 27 insertions(+), 19 deletions(-) diff --git a/youtube_dl/extractor/microsoftstream.py b/youtube_dl/extractor/microsoftstream.py index c152e24a9..57841c5ef 100644 --- a/youtube_dl/extractor/microsoftstream.py +++ b/youtube_dl/extractor/microsoftstream.py @@ -6,10 +6,12 @@ from ..utils import ExtractorError class MicrosoftStreamBaseIE(InfoExtractor): _LOGIN_URL = 'https://web.microsoftstream.com/?noSignUpCheck=1' # expect redirection - _EXPECTED_TITLE = 'Microsoft Stream' def is_logged_in(self, webpage): - return self._EXPECTED_TITLE in webpage + """ + This test is based on the fact that Microsoft Stream will redirect unauthenticated users + """ + return 'Microsoft Stream' in webpage def _real_initialize(self): username, password = self._get_login_info() @@ -38,7 +40,7 @@ class MicrosoftStreamBaseIE(InfoExtractor): class MicrosoftStreamIE(MicrosoftStreamBaseIE): """ - Extract of single Microsoft Stream video + Extractor for single Microsoft Stream video """ IE_NAME = 'microsoftstream' _VALID_URL = r'https?://(?:(?:web|www)\.)?microsoftstream\.com/video/(?P[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})' # https://regex101.com/r/K1mlgK/1/ @@ -48,15 +50,22 @@ class MicrosoftStreamIE(MicrosoftStreamBaseIE): _TEXTTRACKS_RESPONSE = None _VIDEO_ID = None - _TEST = { + _TEST = [{ 'url': 'https://web.microsoftstream.com/video/c883c6a5-9895-4900-9a35-62f4b5d506c9', 'info_dict': { 'id': 'c883c6a5-9895-4900-9a35-62f4b5d506c9', 'ext': 'mp4', 'title': 'Webinar for Researchers: Use of GitLab', - 'thumbnail': r're:^https?://.*$', - } - } + 'thumbnail': r're:^https?://.*$' + }, + 'skip': 'Requires Microsoft 365 account credentials', + }, { + 'url': 'https://web.microsoftstream.com/video/c883c6a5-9895-4900-9a35-62f4b5d506c9', + 'only_matching': True, + }, { + 'url': 'https://www.microsoftstream.com/video/1541f3f9-7fed-4901-ae70-0f7cb775679f', + 'only_matching': True, + }] """ Getters @@ -99,7 +108,7 @@ class MicrosoftStreamIE(MicrosoftStreamBaseIE): Make an additional request to Microsoft Stream for the subtitle and auto-caption """ # Map default variable - self._TEXTTRACKS_RESPONSE = self._download_json(self.texttrack_info_endpoint, self.video_id, headers=self.headers)['value'] + self._TEXTTRACKS_RESPONSE = self._download_json(self.texttrack_info_endpoint, self.video_id, headers=self.headers).get('value') return self._TEXTTRACKS_RESPONSE def _determine_protocol(self, mime): @@ -142,7 +151,7 @@ class MicrosoftStreamIE(MicrosoftStreamBaseIE): # For other Master playlists (like Microsoft Smooth Streaming) else: - self.to_screen('Found unresolvable stream with format %s' % master_playlist_url['mimeType']) + self.to_screen('Found unresolvable stream with format: %s' % master_playlist_url['mimeType']) continue # Patching the "Authorization" header @@ -183,8 +192,8 @@ class MicrosoftStreamIE(MicrosoftStreamBaseIE): def _real_extract(self, url): self._VIDEO_ID = self._match_id(url) - webpage = self._download_webpage(url, self.video_id) + webpage = self._download_webpage(url, self.video_id) if not self.is_logged_in(webpage): return self.raise_login_required() @@ -193,20 +202,19 @@ class MicrosoftStreamIE(MicrosoftStreamBaseIE): self._extract_api_gateway(webpage) # "GET" api for video information - apiUri = self.media_info_endpoint - apiCall = self._download_json(apiUri, self.video_id, headers=self.headers) + apiResponse = self._download_json(self.media_info_endpoint, self.video_id, headers=self.headers) texttracks = self._request_texttracks() return { 'id': self.video_id, - 'title': apiCall['name'], - 'description': apiCall.get('description'), - 'uploader': apiCall.get('creator').get('name'), - 'thumbnails': self._remap_thumbnails(apiCall.get('posterImage')), - 'formats': self._remap_playback(apiCall['playbackUrls']), + 'title': apiResponse['name'], + 'description': apiResponse.get('description'), + 'uploader': apiResponse.get('creator').get('name'), + 'thumbnails': self._remap_thumbnails(apiResponse.get('posterImage')), + 'formats': self._remap_playback(apiResponse['playbackUrls']), 'subtitles': self._get_subtitles(texttracks), 'automatic_captions': self._get_automatic_captions(texttracks), - 'is_live': False, - # 'duration': apiCall['media']['duration'], + 'is_live': False + # 'duration': apiResponse['media']['duration'], } From 67db51cdfe08bcaf9cf2500851ba2d0f04163757 Mon Sep 17 00:00:00 2001 From: Nick Lai Date: Fri, 10 Apr 2020 21:34:20 +0800 Subject: [PATCH 4/5] [MicrosoftStream] Add extractor for channels and refactor single video extractor --- youtube_dl/extractor/extractors.py | 5 +- youtube_dl/extractor/microsoftstream.py | 247 +++++++++++++++++------- 2 files changed, 176 insertions(+), 76 deletions(-) diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py index 847a58f16..6086437fa 100644 --- a/youtube_dl/extractor/extractors.py +++ b/youtube_dl/extractor/extractors.py @@ -619,7 +619,10 @@ from .metacritic import MetacriticIE from .mgoon import MgoonIE from .mgtv import MGTVIE from .miaopai import MiaoPaiIE -from .microsoftstream import MicrosoftStreamIE +from .microsoftstream import ( + MicrosoftStreamIE, + MicrosoftStreamChannelIE +) from .microsoftvirtualacademy import ( MicrosoftVirtualAcademyIE, MicrosoftVirtualAcademyCourseIE, diff --git a/youtube_dl/extractor/microsoftstream.py b/youtube_dl/extractor/microsoftstream.py index 57841c5ef..99515479a 100644 --- a/youtube_dl/extractor/microsoftstream.py +++ b/youtube_dl/extractor/microsoftstream.py @@ -1,11 +1,13 @@ # coding: utf-8 from __future__ import unicode_literals from .common import InfoExtractor -from ..utils import ExtractorError +from ..utils import (ExtractorError, update_url_query, merge_dicts) class MicrosoftStreamBaseIE(InfoExtractor): _LOGIN_URL = 'https://web.microsoftstream.com/?noSignUpCheck=1' # expect redirection + _ACCESS_TOKEN = None # A JWT token + _API_GATEWAY = None def is_logged_in(self, webpage): """ @@ -37,43 +39,14 @@ class MicrosoftStreamBaseIE(InfoExtractor): self._API_GATEWAY = self._html_search_regex(r"\"ApiGatewayUri\":\"(?P.+?)\"", webpage, 'APIGateway') return self._API_GATEWAY - -class MicrosoftStreamIE(MicrosoftStreamBaseIE): """ - Extractor for single Microsoft Stream video - """ - IE_NAME = 'microsoftstream' - _VALID_URL = r'https?://(?:(?:web|www)\.)?microsoftstream\.com/video/(?P[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})' # https://regex101.com/r/K1mlgK/1/ - _NETRC_MACHINE = 'microsoftstream' - _ACCESS_TOKEN = None # A JWT token - _API_GATEWAY = None - _TEXTTRACKS_RESPONSE = None - _VIDEO_ID = None - - _TEST = [{ - 'url': 'https://web.microsoftstream.com/video/c883c6a5-9895-4900-9a35-62f4b5d506c9', - 'info_dict': { - 'id': 'c883c6a5-9895-4900-9a35-62f4b5d506c9', - 'ext': 'mp4', - 'title': 'Webinar for Researchers: Use of GitLab', - 'thumbnail': r're:^https?://.*$' - }, - 'skip': 'Requires Microsoft 365 account credentials', - }, { - 'url': 'https://web.microsoftstream.com/video/c883c6a5-9895-4900-9a35-62f4b5d506c9', - 'only_matching': True, - }, { - 'url': 'https://www.microsoftstream.com/video/1541f3f9-7fed-4901-ae70-0f7cb775679f', - 'only_matching': True, - }] - - """ - Getters - - The following getters include helpful message to prompt developers for potential errors. + Common getters """ @property def api_gateway(self): + """ + Return the start of an API endoint, like "https://aaea-1.api.microsoftstream.com/api/" + """ if self._API_GATEWAY is None: raise ExtractorError('API gateway is None. Did you forget to call "_extract_api_gateway"?') return self._API_GATEWAY @@ -82,34 +55,15 @@ class MicrosoftStreamIE(MicrosoftStreamBaseIE): def access_token(self): if self._ACCESS_TOKEN is None: raise ExtractorError('Access token is None. Did you forget to call "_extract_access_token"?') - return self._ACCESS_TOKEN - @property - def video_id(self): - if self._VIDEO_ID is None: - raise('Variable "_VIDEO_ID" is not defined. Did you make the main extraction call?') - return self._VIDEO_ID - @property def headers(self): return {'Authorization': 'Bearer %s' % self.access_token} - @property - def texttrack_info_endpoint(self): - return "%s/videos/%s/texttracks?api-version=1.3-private" % (self.api_gateway, self.video_id) - - @property - def media_info_endpoint(self): - return "%s/videos/%s?$expand=creator,tokens,status,liveEvent,extensions&api-version=1.3-private" % (self.api_gateway, self.video_id) - - def _request_texttracks(self): - """ - Make an additional request to Microsoft Stream for the subtitle and auto-caption - """ - # Map default variable - self._TEXTTRACKS_RESPONSE = self._download_json(self.texttrack_info_endpoint, self.video_id, headers=self.headers).get('value') - return self._TEXTTRACKS_RESPONSE + """ + Utils function + """ def _determine_protocol(self, mime): """ @@ -122,6 +76,12 @@ class MicrosoftStreamIE(MicrosoftStreamBaseIE): else: return None + """ + Remapper + + Remap data into correct field + """ + def _remap_thumbnails(self, thumbnail_dict_list): output = [] preference_index = ['extraSmall', 'small', 'medium', 'large'] @@ -133,7 +93,7 @@ class MicrosoftStreamIE(MicrosoftStreamBaseIE): }) return output - def _remap_playback(self, master_playlist_urls): + def _remap_playback(self, master_playlist_urls, video_id=None): """ A parser for the HLS and MPD playlists from the API endpoint. """ @@ -143,11 +103,11 @@ class MicrosoftStreamIE(MicrosoftStreamBaseIE): protocol = self._determine_protocol(master_playlist_url['mimeType']) # Handle HLS Master playlist if protocol == 'm3u8': - varient_playlists = self._extract_m3u8_formats(master_playlist_url['playbackUrl'], video_id=self.video_id, headers=self.headers) + varient_playlists = self._extract_m3u8_formats(master_playlist_url['playbackUrl'], video_id=video_id, headers=self.headers) # For MPEG-DASH Master playlists elif protocol == 'http_dash_segments': - varient_playlists = self._extract_mpd_formats(master_playlist_url['playbackUrl'], video_id=self.video_id, headers=self.headers) + varient_playlists = self._extract_mpd_formats(master_playlist_url['playbackUrl'], video_id=video_id, headers=self.headers) # For other Master playlists (like Microsoft Smooth Streaming) else: @@ -160,7 +120,7 @@ class MicrosoftStreamIE(MicrosoftStreamBaseIE): output.append(varient_playlist) return output - def _extract_subtitle(self, tracks, is_auto_generated): + def _remap_subtitle(self, tracks, video_id, is_auto_generated): """ An internal method for filtering and remapping text tracks """ @@ -177,18 +137,83 @@ class MicrosoftStreamIE(MicrosoftStreamBaseIE): subtitle_subset[track_language] = [] # Scaffold an empty list for the object to insert into # Since the subtitle is token protected, a get request will fire here. - data = self._download_webpage(url_or_request=track.get('url'), video_id=self.video_id, headers=self.headers) + data = self._download_webpage(url_or_request=track.get('url'), video_id=video_id, headers=self.headers) subtitle_subset[track_language].append({'data': data, "ext": "vtt"}) return subtitle_subset + def _remap_video(self, video): + return { + 'id': video['id'], + 'title': video['name'], + 'description': video.get('description'), + 'uploader': video.get('creator').get('name'), + 'thumbnails': self._remap_thumbnails(video.get('posterImage') or []), + 'formats': self._remap_playback(video['playbackUrls'], video_id=video['id']), + 'is_live': False + } + """ + Formatter + """ + + def _format_texttrack_endpoint(self, video_id): + return "%s/videos/%s/texttracks?api-version=1.4-private" % (self.api_gateway, video_id) + + def _request_texttracks(self, video_id): + """ + Make an additional request to Microsoft Stream for the subtitle and auto-caption + """ + self._TEXTTRACKS_RESPONSE = self._download_json(self._format_texttrack_endpoint(video_id), video_id, headers=self.headers).get('value') + return self._TEXTTRACKS_RESPONSE + + +class MicrosoftStreamIE(MicrosoftStreamBaseIE): + """ + Extractor for single Microsoft Stream video + """ + IE_NAME = 'microsoftstream' + _VALID_URL = r'https?://(?:(?:web|www)\.)?microsoftstream\.com/video/(?P[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})' # https://regex101.com/r/K1mlgK/1/ + _NETRC_MACHINE = 'microsoftstream' + _TEXTTRACKS_RESPONSE = None + _VIDEO_ID = None + + _TEST = [{ + 'url': 'https://web.microsoftstream.com/video/c883c6a5-9895-4900-9a35-62f4b5d506c9', + 'info_dict': { + 'id': 'c883c6a5-9895-4900-9a35-62f4b5d506c9', + 'ext': 'mp4', + 'title': 'Webinar for Researchers: Use of GitLab', + 'thumbnail': r're:^https?://.*$' + }, + 'skip': 'Requires Microsoft 365 account credentials', + }, { + 'url': 'https://www.microsoftstream.com/video/1541f3f9-7fed-4901-ae70-0f7cb775679f', + 'only_matching': True, + }] + + """ + Getters + + The following getters include helpful message to prompt developers for potential errors. + """ + + @property + def video_id(self): + if self._VIDEO_ID is None: + raise('Variable "_VIDEO_ID" is not defined. Did you make the main extraction call?') + return self._VIDEO_ID + + @property + def media_info_endpoint(self): + return "%s/videos/%s?$expand=creator,tokens,status,liveEvent,extensions&api-version=1.4-private" % (self.api_gateway, self.video_id) + def _get_subtitles(self, tracks=None): # Fulfill abstract method tracks = self._TEXTTRACKS_RESPONSE if tracks is None else tracks - return self._extract_subtitle(tracks, False) + return self._remap_subtitle(tracks, is_auto_generated=False, video_id=self.video_id) def _get_automatic_captions(self, tracks=None): # Fulfill abstract method tracks = self._TEXTTRACKS_RESPONSE if tracks is None else tracks - return self._extract_subtitle(tracks, True) + return self._remap_subtitle(tracks, is_auto_generated=True, video_id=self.video_id) def _real_extract(self, url): self._VIDEO_ID = self._match_id(url) @@ -202,19 +227,91 @@ class MicrosoftStreamIE(MicrosoftStreamBaseIE): self._extract_api_gateway(webpage) # "GET" api for video information - apiResponse = self._download_json(self.media_info_endpoint, self.video_id, headers=self.headers) + video = self._download_json(self.media_info_endpoint, self.video_id, headers=self.headers) + texttracks = self._request_texttracks(self.video_id) - texttracks = self._request_texttracks() - - return { - 'id': self.video_id, - 'title': apiResponse['name'], - 'description': apiResponse.get('description'), - 'uploader': apiResponse.get('creator').get('name'), - 'thumbnails': self._remap_thumbnails(apiResponse.get('posterImage')), - 'formats': self._remap_playback(apiResponse['playbackUrls']), + return merge_dicts(self._remap_video(video), { 'subtitles': self._get_subtitles(texttracks), 'automatic_captions': self._get_automatic_captions(texttracks), - 'is_live': False - # 'duration': apiResponse['media']['duration'], + }) + + +class MicrosoftStreamChannelIE(MicrosoftStreamBaseIE): + """ + Extractor for single channel of Microsoft Stream video + """ + IE_NAME = 'microsoftstream:channel' + _VALID_URL = r'https?://(?:(?:web|www)\.)?microsoftstream\.com/channel/(?P[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})' # https://regex101.com/r/K1mlgK/1/ + _NETRC_MACHINE = 'microsoftstream' + _CHANNEL_ID = None + + _TEST = [{ + 'url': 'https://web.microsoftstream.com/channel/c883c6a5-9895-4900-9a35-62f4b5d506c9', + 'only_matching': True, + }, { + 'url': 'https://www.microsoftstream.com/channel/0ceffe58-07b1-4098-8ed9-a15f4e8231f7', + 'only_matching': True, + }] + + def _format_channel_video_endpoint(self, skip=0, top=100): + parameters = { + '$top': top, + '$skip': skip, + '$expand': 'creator,events', + '$orderby': 'created asc', + 'api-version': '1.4-private' } + + return update_url_query('%schannels/%s/videos' % (self.api_gateway, self._CHANNEL_ID), parameters) + + def _iterate_video(self, video): + subtitle_dict = {} + + if (self._downloader.params.get('writesubtitles', False) + or self._downloader.params.get('writeautomaticsub', False) + or self._downloader.params.get('listsubtitles')): + texttracks = self._request_texttracks(video['id']) + + if (self._downloader.params.get('writesubtitles', False) + or self._downloader.params.get('listsubtitles')): + subtitle_dict['subtitles'] = self._remap_subtitle(texttracks, video_id=video['id'], is_auto_generated=False) + + if (self._downloader.params.get('writeautomaticsub', False) + or self._downloader.params.get('listsubtitles')): + subtitle_dict['automatic_captions'] = self._remap_subtitle(texttracks, video_id=video['id'], is_auto_generated=False) + + return merge_dicts(self._remap_video(video), subtitle_dict) + + def _fetch_video(self): + found_all_video = False + current_skip = 0 + top = 100 + all_video = [] + + while found_all_video is False: + video_info_endpoint = self._format_channel_video_endpoint(current_skip, top) + channel_video_subset = self._download_json(video_info_endpoint, self._CHANNEL_ID, headers=self.headers)['value'] + + for video in channel_video_subset: + all_video.append(self._iterate_video(video)) # Remap the video + + found_all_video = bool(len(channel_video_subset) < top) # Break out from the iteration if all content is downloaded. + current_skip += top # Adjust starting position + + return all_video + + def _real_extract(self, url): + self._CHANNEL_ID = self._match_id(url) + + webpage = self._download_webpage(url, self._CHANNEL_ID) + if not self.is_logged_in(webpage): + return self.raise_login_required() + + # Extract access token from webpage + self._extract_access_token(webpage) + self._extract_api_gateway(webpage) + + entries = self._fetch_video() + + return {'_type': 'playlist', + 'entries': entries} From 5e7738a0df9bd6c00fea256b546c05e590efac26 Mon Sep 17 00:00:00 2001 From: Nick Lai Date: Mon, 20 Apr 2020 03:15:53 +0800 Subject: [PATCH 5/5] [MicrosoftStream] Fix typo --- youtube_dl/extractor/microsoftstream.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/microsoftstream.py b/youtube_dl/extractor/microsoftstream.py index 99515479a..78457d000 100644 --- a/youtube_dl/extractor/microsoftstream.py +++ b/youtube_dl/extractor/microsoftstream.py @@ -177,7 +177,7 @@ class MicrosoftStreamIE(MicrosoftStreamBaseIE): _TEXTTRACKS_RESPONSE = None _VIDEO_ID = None - _TEST = [{ + _TESTS = [{ 'url': 'https://web.microsoftstream.com/video/c883c6a5-9895-4900-9a35-62f4b5d506c9', 'info_dict': { 'id': 'c883c6a5-9895-4900-9a35-62f4b5d506c9', @@ -245,7 +245,7 @@ class MicrosoftStreamChannelIE(MicrosoftStreamBaseIE): _NETRC_MACHINE = 'microsoftstream' _CHANNEL_ID = None - _TEST = [{ + _TESTS = [{ 'url': 'https://web.microsoftstream.com/channel/c883c6a5-9895-4900-9a35-62f4b5d506c9', 'only_matching': True, }, {