From 8d2881a442211b5ea058e7a1f3bfe46359080fd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maciej=20Kr=C3=BCger?= Date: Thu, 22 Jul 2021 12:46:23 +0200 Subject: [PATCH 1/8] [voe] Add new extractor --- youtube_dl/extractor/extractors.py | 1 + youtube_dl/extractor/voe.py | 47 ++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) create mode 100644 youtube_dl/extractor/voe.py diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py index 6e8fc3961..5b658fb1c 100644 --- a/youtube_dl/extractor/extractors.py +++ b/youtube_dl/extractor/extractors.py @@ -1484,6 +1484,7 @@ from .voxmedia import ( VoxMediaVolumeIE, VoxMediaIE, ) +from .voe import VOEIE from .vrt import VRTIE from .vrak import VrakIE from .vrv import ( diff --git a/youtube_dl/extractor/voe.py b/youtube_dl/extractor/voe.py new file mode 100644 index 000000000..4f24e50bb --- /dev/null +++ b/youtube_dl/extractor/voe.py @@ -0,0 +1,47 @@ +# coding: utf-8 +from __future__ import unicode_literals + +from .common import InfoExtractor + + +class VOEIE(InfoExtractor): + IE_NAME = 'voe' + IE_DESC = 'VOE.SX' + _VALID_URL = r'https?://voe\.sx/(e/)?(?P[a-z0-9]+)' + _TEST = { + 'url': 'https://voe.sx/e/ng7ja5n5n2y8', + 'info_dict': { + 'id': 'ng7ja5n5n2y8', + 'title': 'md5:05ab15eb43a32f0f5918755156c5fb34', + 'thumbnail': r're:^https?://.*\.jpg$', + 'ext': 'm3u8', + }, + } + + def _real_extract(self, url): + video_id = self._match_id(url) + + webpage = self._download_webpage( + 'https://voe.sx/e/%s' % video_id, video_id) + + m3u8 = self._search_regex( + r'(https.+m3u8)', + webpage, 'm3u8') + + title = self._search_regex( + r'Watch (?P<title>.+)<\/title>', + webpage, 'title', group='title') + + thumbnail = self._search_regex( + r'VOEPlayer.poster="(?P<thumbnail>https.+)"', + webpage, 'thumbnail', group='thumbnail') + + formats = self._extract_m3u8_formats(m3u8, video_id) + self._sort_formats(formats) + + return { + 'id': video_id, + 'title': title, + 'formats': formats, + 'thumbnail': thumbnail, + } From 001d5b8395b5b1eccfb829867c2e36e2ee7241b8 Mon Sep 17 00:00:00 2001 From: dirkf <fieldhouse@gmx.net> Date: Thu, 17 Nov 2022 19:23:43 +0000 Subject: [PATCH 2/8] Fix extraction --- youtube_dl/extractor/voe.py | 37 +++++++++++++++++++++++++++---------- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/youtube_dl/extractor/voe.py b/youtube_dl/extractor/voe.py index 4f24e50bb..c4512154d 100644 --- a/youtube_dl/extractor/voe.py +++ b/youtube_dl/extractor/voe.py @@ -2,7 +2,11 @@ from __future__ import unicode_literals from .common import InfoExtractor - +from ..utils import ( + int_or_none, + js_to_json, + url_or_none, +) class VOEIE(InfoExtractor): IE_NAME = 'voe' @@ -12,7 +16,7 @@ class VOEIE(InfoExtractor): 'url': 'https://voe.sx/e/ng7ja5n5n2y8', 'info_dict': { 'id': 'ng7ja5n5n2y8', - 'title': 'md5:05ab15eb43a32f0f5918755156c5fb34', + 'title': 'md5:a86687fb962742f04652aee19ad34e06', 'thumbnail': r're:^https?://.*\.jpg$', 'ext': 'm3u8', }, @@ -24,21 +28,34 @@ class VOEIE(InfoExtractor): webpage = self._download_webpage( 'https://voe.sx/e/%s' % video_id, video_id) - m3u8 = self._search_regex( - r'(https.+m3u8)', - webpage, 'm3u8') + sources = self._parse_json( + self._search_regex(r'\bsources\s*=\s*(\{[^}]+\})', webpage, 'sources'), + video_id, transform_source=js_to_json) title = self._search_regex( - r'<title>Watch (?P<title>.+)<\/title>', + r'<title>(?:Watch\s+)?(?P<title>.+?)(?:-\s+VOE\s+\|.+)?', webpage, 'title', group='title') - thumbnail = self._search_regex( - r'VOEPlayer.poster="(?Phttps.+)"', - webpage, 'thumbnail', group='thumbnail') + formats = [] + + f_url = url_or_none(sources.get('hls')) + if f_url: + formats.extend(self._extract_m3u8_formats( + f_url, video_id, entry_protocol='m3u8_native', fatal=False)) + f_url = url_or_none(sources.get('mp4')) + if f_url: + formats.append({ + 'url': f_url, + 'ext': 'mp4', + 'height': int_or_none(sources.get('video_height')), + }) - formats = self._extract_m3u8_formats(m3u8, video_id) self._sort_formats(formats) + thumbnail = url_or_none(self._search_regex( + r'(?:VOEPlayer.|data-)poster\s*=\s*(["\'])(?P(?:(?!\1)\S)+)\1', + webpage, 'thumbnail', group='thumbnail', default=None)) + return { 'id': video_id, 'title': title, From d4d99b1b51d21c48c23c93e99565500d4ac66d05 Mon Sep 17 00:00:00 2001 From: dirkf Date: Thu, 17 Nov 2022 19:29:59 +0000 Subject: [PATCH 3/8] Linty --- youtube_dl/extractor/voe.py | 1 + 1 file changed, 1 insertion(+) diff --git a/youtube_dl/extractor/voe.py b/youtube_dl/extractor/voe.py index c4512154d..870a6b1aa 100644 --- a/youtube_dl/extractor/voe.py +++ b/youtube_dl/extractor/voe.py @@ -8,6 +8,7 @@ from ..utils import ( url_or_none, ) + class VOEIE(InfoExtractor): IE_NAME = 'voe' IE_DESC = 'VOE.SX' From 3525025a6fc58891f52a73f77f63fc96f8c54868 Mon Sep 17 00:00:00 2001 From: dirkf Date: Mon, 18 Nov 2024 02:28:15 +0000 Subject: [PATCH 4/8] Update extractor for current site behaviour --- youtube_dl/extractor/voe.py | 34 +++++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/youtube_dl/extractor/voe.py b/youtube_dl/extractor/voe.py index 870a6b1aa..cbf30f19a 100644 --- a/youtube_dl/extractor/voe.py +++ b/youtube_dl/extractor/voe.py @@ -1,6 +1,8 @@ # coding: utf-8 from __future__ import unicode_literals +import base64 + from .common import InfoExtractor from ..utils import ( int_or_none, @@ -12,8 +14,8 @@ from ..utils import ( class VOEIE(InfoExtractor): IE_NAME = 'voe' IE_DESC = 'VOE.SX' - _VALID_URL = r'https?://voe\.sx/(e/)?(?P[a-z0-9]+)' - _TEST = { + _VALID_URL = r'https?://voe\.sx(?P/(e/)?(?P[a-z0-9]+))' + _TESTS = [{ 'url': 'https://voe.sx/e/ng7ja5n5n2y8', 'info_dict': { 'id': 'ng7ja5n5n2y8', @@ -21,29 +23,39 @@ class VOEIE(InfoExtractor): 'thumbnail': r're:^https?://.*\.jpg$', 'ext': 'm3u8', }, - } + }] def _real_extract(self, url): - video_id = self._match_id(url) + video_id, video_path = self._match_valid_url(url).group('id', 'path') webpage = self._download_webpage( 'https://voe.sx/e/%s' % video_id, video_id) - sources = self._parse_json( - self._search_regex(r'\bsources\s*=\s*(\{[^}]+\})', webpage, 'sources'), - video_id, transform_source=js_to_json) + player_url = self._search_regex( + r'''("|')(?Phttps://(?!voe\.sx/)[^/]+%s)\1\s*;''' % (path,), + webpage, 'redirect', group='url', default=None) + if player_url: + webpage = self._download_webpage( + player_url, video_id, note='Redirecting to player page') + + + sources = self._search_json( + r'\bsources\s*=', webpage, 'sources', video_id, transform_source=js_to_json) title = self._search_regex( r'(?:Watch\s+)?(?P<title>.+?)(?:-\s+VOE\s+\|.+)?', webpage, 'title', group='title') formats = [] - - f_url = url_or_none(sources.get('hls')) + for fmt in ('mp4, 'hls'): + if fmt not in sources: + continue + sources[fmt] = url_or_none(sources['fmt']) or url_or_none(base64.b64decode(sources['fmt'])) + f_url = sources.get('hls') if f_url: formats.extend(self._extract_m3u8_formats( f_url, video_id, entry_protocol='m3u8_native', fatal=False)) - f_url = url_or_none(sources.get('mp4')) + f_url = sources.get('mp4') if f_url: formats.append({ 'url': f_url, @@ -54,7 +66,7 @@ class VOEIE(InfoExtractor): self._sort_formats(formats) thumbnail = url_or_none(self._search_regex( - r'(?:VOEPlayer.|data-)poster\s*=\s*(["\'])(?P(?:(?!\1)\S)+)\1', + r'''(?:VOEPlayer\s*\.\s*|data-)poster\s*=\s*("|')(?P(?:(?!\1)\S)+)\1''', webpage, 'thumbnail', group='thumbnail', default=None)) return { From ff968566a5cc3e6b2fa851bef268846a68f5fb0a Mon Sep 17 00:00:00 2001 From: dirkf Date: Mon, 18 Nov 2024 02:39:56 +0000 Subject: [PATCH 5/8] Fix quoting --- youtube_dl/extractor/voe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/voe.py b/youtube_dl/extractor/voe.py index cbf30f19a..40c2c9436 100644 --- a/youtube_dl/extractor/voe.py +++ b/youtube_dl/extractor/voe.py @@ -47,7 +47,7 @@ class VOEIE(InfoExtractor): webpage, 'title', group='title') formats = [] - for fmt in ('mp4, 'hls'): + for fmt in ('mp4', 'hls'): if fmt not in sources: continue sources[fmt] = url_or_none(sources['fmt']) or url_or_none(base64.b64decode(sources['fmt'])) From 0b817e51c210152cbd04c792db76395d5c9507fd Mon Sep 17 00:00:00 2001 From: dirkf Date: Mon, 18 Nov 2024 02:48:29 +0000 Subject: [PATCH 6/8] Linty --- youtube_dl/extractor/voe.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/youtube_dl/extractor/voe.py b/youtube_dl/extractor/voe.py index 40c2c9436..adef0ab4b 100644 --- a/youtube_dl/extractor/voe.py +++ b/youtube_dl/extractor/voe.py @@ -32,15 +32,14 @@ class VOEIE(InfoExtractor): 'https://voe.sx/e/%s' % video_id, video_id) player_url = self._search_regex( - r'''("|')(?Phttps://(?!voe\.sx/)[^/]+%s)\1\s*;''' % (path,), + r'''("|')(?Phttps://(?!voe\.sx/)[^/]+%s)\1\s*;''' % (video_path,), webpage, 'redirect', group='url', default=None) if player_url: webpage = self._download_webpage( player_url, video_id, note='Redirecting to player page') - sources = self._search_json( - r'\bsources\s*=', webpage, 'sources', video_id, transform_source=js_to_json) + r'\bsources\s*=', webpage, 'sources', video_id, transform_source=js_to_json) title = self._search_regex( r'(?:Watch\s+)?(?P<title>.+?)(?:-\s+VOE\s+\|.+)?', From 9179a0294b5e62490424f391bb634e522826cbe5 Mon Sep 17 00:00:00 2001 From: dirkf Date: Mon, 18 Nov 2024 03:30:38 +0000 Subject: [PATCH 7/8] Make test pass --- youtube_dl/extractor/voe.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/voe.py b/youtube_dl/extractor/voe.py index adef0ab4b..4f72b0d9f 100644 --- a/youtube_dl/extractor/voe.py +++ b/youtube_dl/extractor/voe.py @@ -4,6 +4,7 @@ from __future__ import unicode_literals import base64 from .common import InfoExtractor +from ..compat import compat_base64_b64decode from ..utils import ( int_or_none, js_to_json, @@ -19,7 +20,7 @@ class VOEIE(InfoExtractor): 'url': 'https://voe.sx/e/ng7ja5n5n2y8', 'info_dict': { 'id': 'ng7ja5n5n2y8', - 'title': 'md5:a86687fb962742f04652aee19ad34e06', + 'title': 'md5:8dd774de9b73851151d80ef6baaea7f1', 'thumbnail': r're:^https?://.*\.jpg$', 'ext': 'm3u8', }, @@ -49,7 +50,7 @@ class VOEIE(InfoExtractor): for fmt in ('mp4', 'hls'): if fmt not in sources: continue - sources[fmt] = url_or_none(sources['fmt']) or url_or_none(base64.b64decode(sources['fmt'])) + sources[fmt] = url_or_none(sources[fmt]) or url_or_none(compat_base64_b64decode(sources[fmt]).decode('utf-8')) f_url = sources.get('hls') if f_url: formats.extend(self._extract_m3u8_formats( From e29f8e8707b3c37eb10d9e2944d76550d43d297a Mon Sep 17 00:00:00 2001 From: dirkf Date: Mon, 18 Nov 2024 03:36:46 +0000 Subject: [PATCH 8/8] Linty --- youtube_dl/extractor/voe.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/youtube_dl/extractor/voe.py b/youtube_dl/extractor/voe.py index 4f72b0d9f..8d355d778 100644 --- a/youtube_dl/extractor/voe.py +++ b/youtube_dl/extractor/voe.py @@ -1,10 +1,8 @@ # coding: utf-8 from __future__ import unicode_literals -import base64 - from .common import InfoExtractor -from ..compat import compat_base64_b64decode +from ..compat import compat_base64_b64decode from ..utils import ( int_or_none, js_to_json,