From 81546c4781225f7a45d4883103945ef6b98f821b Mon Sep 17 00:00:00 2001 From: ryanstep Date: Mon, 9 Dec 2024 16:52:36 -0500 Subject: [PATCH 1/5] boiler plate --- youtube_dl/extractor/canalrcn.py | 38 ++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 youtube_dl/extractor/canalrcn.py diff --git a/youtube_dl/extractor/canalrcn.py b/youtube_dl/extractor/canalrcn.py new file mode 100644 index 000000000..7d1247e96 --- /dev/null +++ b/youtube_dl/extractor/canalrcn.py @@ -0,0 +1,38 @@ +# coding: utf-8 +from __future__ import unicode_literals + +from .common import InfoExtractor + + +class YourExtractorIE(InfoExtractor): + _VALID_URL = r'https?://(?:www\.)?canalrcn\.com/watch/(?P[0-9]+)' + _TEST = { + 'url': 'https://yourextractor.com/watch/42', + 'md5': 'TODO: md5 sum of the first 10241 bytes of the video file (use --test)', + 'info_dict': { + 'id': '42', + 'ext': 'mp4', + 'title': 'Video title goes here', + 'thumbnail': r're:^https?://.*\.jpg$', + # TODO more properties, either as: + # * A value + # * MD5 checksum; start the string with md5: + # * A regular expression; start the string with re: + # * Any Python type (for example int or float) + } + } + + def _real_extract(self, url): + video_id = self._match_id(url) + webpage = self._download_webpage(url, video_id) + + # TODO more code goes here, for example ... + title = self._html_search_regex(r'

(.+?)

', webpage, 'title') + + return { + 'id': video_id, + 'title': title, + 'description': self._og_search_description(webpage), + 'uploader': self._search_regex(r']+id="uploader"[^>]*>([^<]+)<', webpage, 'uploader', fatal=False), + # TODO more properties (see youtube_dl/extractor/common.py) + } \ No newline at end of file From b71f53a4ce059484bc5244319a68b46d7a03218a Mon Sep 17 00:00:00 2001 From: ryanstep Date: Mon, 9 Dec 2024 17:06:48 -0500 Subject: [PATCH 2/5] boiler plate 2 --- youtube_dl/extractor/canalrcn.py | 4 ++-- youtube_dl/extractor/extractors.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/canalrcn.py b/youtube_dl/extractor/canalrcn.py index 7d1247e96..07e4faa3e 100644 --- a/youtube_dl/extractor/canalrcn.py +++ b/youtube_dl/extractor/canalrcn.py @@ -4,10 +4,10 @@ from __future__ import unicode_literals from .common import InfoExtractor -class YourExtractorIE(InfoExtractor): +class CanalrcnIE(InfoExtractor): _VALID_URL = r'https?://(?:www\.)?canalrcn\.com/watch/(?P[0-9]+)' _TEST = { - 'url': 'https://yourextractor.com/watch/42', + 'url': 'https://canalrcn.com/watch/42', 'md5': 'TODO: md5 sum of the first 10241 bytes of the video file (use --test)', 'info_dict': { 'id': '42', diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py index 3da5f8020..b6972509e 100644 --- a/youtube_dl/extractor/extractors.py +++ b/youtube_dl/extractor/extractors.py @@ -169,6 +169,7 @@ from .cammodels import CamModelsIE from .camtube import CamTubeIE from .camwithher import CamWithHerIE from .canalplus import CanalplusIE +from .canalrcn import CanalrcnIE from .canalc2 import Canalc2IE from .canvas import ( CanvasIE, From 82523cd5739468c3825aaa2eecc5d70e0e3729fe Mon Sep 17 00:00:00 2001 From: ryanstep Date: Mon, 9 Dec 2024 17:35:59 -0500 Subject: [PATCH 3/5] initial test --- youtube_dl/extractor/canalrcn.py | 87 ++++++++++++++++++++++++-------- 1 file changed, 65 insertions(+), 22 deletions(-) diff --git a/youtube_dl/extractor/canalrcn.py b/youtube_dl/extractor/canalrcn.py index 07e4faa3e..7d4973d5c 100644 --- a/youtube_dl/extractor/canalrcn.py +++ b/youtube_dl/extractor/canalrcn.py @@ -2,37 +2,80 @@ from __future__ import unicode_literals from .common import InfoExtractor - +from ..utils import ( + ExtractorError, + extract_attributes, + int_or_none, + traverse_obj +) +import json +import re class CanalrcnIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?canalrcn\.com/watch/(?P[0-9]+)' - _TEST = { - 'url': 'https://canalrcn.com/watch/42', - 'md5': 'TODO: md5 sum of the first 10241 bytes of the video file (use --test)', + _VALID_URL = r'https?://(?:www\.)?canalrcn\.com/(?:[^/]+/)+(?P[^/?&#]+)' + _TESTS = [{ + 'url': 'https://www.canalrcn.com/la-rosa-de-guadalupe/capitulos/la-rosa-de-guadalupe-capitulo-58-los-enamorados-3619', 'info_dict': { - 'id': '42', + 'id': 'x8ecrn2', 'ext': 'mp4', - 'title': 'Video title goes here', - 'thumbnail': r're:^https?://.*\.jpg$', - # TODO more properties, either as: - # * A value - # * MD5 checksum; start the string with md5: - # * A regular expression; start the string with re: - # * Any Python type (for example int or float) - } - } + 'title': 'La rosa de Guadalupe | Capítulo 58 | Los enamorados', + 'description': 'Pamela conoce a un hombre, pero sus papás no se lo aprueban porque no tiene recursos.', + 'thumbnail': r're:^https?://.*\.(?:jpg|png|webp)', + }, + 'params': { + 'skip_download': True, + }, + 'skip': 'Geo-restricted to Colombia', + }] def _real_extract(self, url): video_id = self._match_id(url) webpage = self._download_webpage(url, video_id) + + # Extract JSON-LD data + json_ld = self._search_regex( + r']+type=(["\'])application/ld\+json\1[^>]*>(?P[^<]+)', + webpage, 'JSON-LD', group='json', default='{}') + + try: + json_data = json.loads(json_ld) + except json.JSONDecodeError: + raise ExtractorError('Could not parse JSON-LD data') + + # Find the VideoObject in the JSON-LD array + video_data = None + if isinstance(json_data, list): + for item in json_data: + if isinstance(item, dict) and item.get('@type') == 'VideoObject': + video_data = item + break + + if not video_data: + raise ExtractorError('Could not find video information in JSON-LD data') - # TODO more code goes here, for example ... - title = self._html_search_regex(r'

(.+?)

', webpage, 'title') + # Extract player information + player_match = re.search(r'dailymotion\.createPlayer\("([^"]+)",\s*{([^}]+)}', webpage) + if not player_match: + raise ExtractorError('Could not find player configuration') + + player_config = player_match.group(2) + video_url_match = re.search(r'video:\s*["\']([^"\']+)', player_config) + if not video_url_match: + raise ExtractorError('Could not find video URL') + + dailymotion_id = video_url_match.group(1).split('&&')[0] + + # Configure geo-bypass headers + self._downloader.params['geo_verification_proxy'] = 'co' return { - 'id': video_id, - 'title': title, - 'description': self._og_search_description(webpage), - 'uploader': self._search_regex(r']+id="uploader"[^>]*>([^<]+)<', webpage, 'uploader', fatal=False), - # TODO more properties (see youtube_dl/extractor/common.py) + '_type': 'url_transparent', + 'url': 'http://www.dailymotion.com/video/%s' % dailymotion_id, + 'ie_key': 'Dailymotion', + 'id': dailymotion_id, + 'title': video_data.get('name'), + 'description': video_data.get('description'), + 'thumbnail': video_data.get('thumbnailUrl'), + 'duration': video_data.get('duration'), + 'note': 'Video is geo-restricted to Colombia', } \ No newline at end of file From acea8f31d4a764410228e057939444cf54aeaf51 Mon Sep 17 00:00:00 2001 From: ryanstep Date: Mon, 9 Dec 2024 17:53:13 -0500 Subject: [PATCH 4/5] added error handling for geo-restriction --- youtube_dl/extractor/canalrcn.py | 43 ++++++++++++++++---------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/youtube_dl/extractor/canalrcn.py b/youtube_dl/extractor/canalrcn.py index 7d4973d5c..616b784d5 100644 --- a/youtube_dl/extractor/canalrcn.py +++ b/youtube_dl/extractor/canalrcn.py @@ -4,15 +4,17 @@ from __future__ import unicode_literals from .common import InfoExtractor from ..utils import ( ExtractorError, - extract_attributes, - int_or_none, - traverse_obj + clean_html, + try_get, ) import json -import re class CanalrcnIE(InfoExtractor): _VALID_URL = r'https?://(?:www\.)?canalrcn\.com/(?:[^/]+/)+(?P[^/?&#]+)' + + # Specify geo-restriction + _GEO_COUNTRIES = ['CO'] + _TESTS = [{ 'url': 'https://www.canalrcn.com/la-rosa-de-guadalupe/capitulos/la-rosa-de-guadalupe-capitulo-58-los-enamorados-3619', 'info_dict': { @@ -25,14 +27,14 @@ class CanalrcnIE(InfoExtractor): 'params': { 'skip_download': True, }, - 'skip': 'Geo-restricted to Colombia', + 'expected_warnings': ['Video is geo-restricted to Colombia'], + 'skip': 'Geo-restricted to Colombia' }] def _real_extract(self, url): video_id = self._match_id(url) webpage = self._download_webpage(url, video_id) - # Extract JSON-LD data json_ld = self._search_regex( r']+type=(["\'])application/ld\+json\1[^>]*>(?P[^<]+)', webpage, 'JSON-LD', group='json', default='{}') @@ -42,7 +44,6 @@ class CanalrcnIE(InfoExtractor): except json.JSONDecodeError: raise ExtractorError('Could not parse JSON-LD data') - # Find the VideoObject in the JSON-LD array video_data = None if isinstance(json_data, list): for item in json_data: @@ -53,20 +54,21 @@ class CanalrcnIE(InfoExtractor): if not video_data: raise ExtractorError('Could not find video information in JSON-LD data') - # Extract player information - player_match = re.search(r'dailymotion\.createPlayer\("([^"]+)",\s*{([^}]+)}', webpage) - if not player_match: - raise ExtractorError('Could not find player configuration') + embed_url = video_data.get('embedUrl') + if not embed_url: + raise ExtractorError('Could not find video embed URL') - player_config = player_match.group(2) - video_url_match = re.search(r'video:\s*["\']([^"\']+)', player_config) - if not video_url_match: - raise ExtractorError('Could not find video URL') - - dailymotion_id = video_url_match.group(1).split('&&')[0] - - # Configure geo-bypass headers - self._downloader.params['geo_verification_proxy'] = 'co' + dailymotion_id = self._search_regex( + r'dailymotion\.com/(?:embed/)?video/([a-zA-Z0-9]+)', + embed_url, + 'dailymotion id' + ) + + #geo-restriction handling + self.raise_geo_restricted( + msg='This video is only available in Colombia', + countries=self._GEO_COUNTRIES + ) return { '_type': 'url_transparent', @@ -77,5 +79,4 @@ class CanalrcnIE(InfoExtractor): 'description': video_data.get('description'), 'thumbnail': video_data.get('thumbnailUrl'), 'duration': video_data.get('duration'), - 'note': 'Video is geo-restricted to Colombia', } \ No newline at end of file From a87561248a2cf7283bead4b4f4d323d3d5ae0c94 Mon Sep 17 00:00:00 2001 From: ryanstep Date: Mon, 9 Dec 2024 21:03:52 -0500 Subject: [PATCH 5/5] style checks --- youtube_dl/extractor/canalrcn.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/youtube_dl/extractor/canalrcn.py b/youtube_dl/extractor/canalrcn.py index 616b784d5..71cdc7aed 100644 --- a/youtube_dl/extractor/canalrcn.py +++ b/youtube_dl/extractor/canalrcn.py @@ -4,17 +4,18 @@ from __future__ import unicode_literals from .common import InfoExtractor from ..utils import ( ExtractorError, - clean_html, - try_get, ) import json + class CanalrcnIE(InfoExtractor): + """Information extractor for canalrcn.com""" + _VALID_URL = r'https?://(?:www\.)?canalrcn\.com/(?:[^/]+/)+(?P[^/?&#]+)' - + # Specify geo-restriction _GEO_COUNTRIES = ['CO'] - + _TESTS = [{ 'url': 'https://www.canalrcn.com/la-rosa-de-guadalupe/capitulos/la-rosa-de-guadalupe-capitulo-58-los-enamorados-3619', 'info_dict': { @@ -34,23 +35,23 @@ class CanalrcnIE(InfoExtractor): def _real_extract(self, url): video_id = self._match_id(url) webpage = self._download_webpage(url, video_id) - + json_ld = self._search_regex( r']+type=(["\'])application/ld\+json\1[^>]*>(?P[^<]+)', webpage, 'JSON-LD', group='json', default='{}') - + try: json_data = json.loads(json_ld) except json.JSONDecodeError: raise ExtractorError('Could not parse JSON-LD data') - + video_data = None if isinstance(json_data, list): for item in json_data: if isinstance(item, dict) and item.get('@type') == 'VideoObject': video_data = item break - + if not video_data: raise ExtractorError('Could not find video information in JSON-LD data') @@ -63,8 +64,7 @@ class CanalrcnIE(InfoExtractor): embed_url, 'dailymotion id' ) - - #geo-restriction handling + # geo-restriction handling self.raise_geo_restricted( msg='This video is only available in Colombia', countries=self._GEO_COUNTRIES @@ -79,4 +79,4 @@ class CanalrcnIE(InfoExtractor): 'description': video_data.get('description'), 'thumbnail': video_data.get('thumbnailUrl'), 'duration': video_data.get('duration'), - } \ No newline at end of file + }