lopar/youtube-dl
1
0
Fork 0
mirror of https://github.com/ytdl-org/youtube-dl.git synced 2024-12-23 08:26:50 +00:00
Code Releases Activity

[pinterest] Add support for large collections(more than 25 pins)

Browse Source
This commit is contained in:
Remita Amine 2020-11-23 14:16:38 +01:00
parent 2cd43a00d1
commit 484fe78737
1 changed files with 56 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

63
youtube_dl/extractor/pinterest.py
View File

@ -1,6 +1,9 @@
# coding: utf-8 # coding: utf-8
from __future__ import unicode_literals from __future__ import unicode_literals
import json
import re
from .common import InfoExtractor from .common import InfoExtractor
from ..compat import compat_str from ..compat import compat_str
from ..utils import ( from ..utils import (
@ -16,12 +19,12 @@ from ..utils import (
class PinterestBaseIE(InfoExtractor): class PinterestBaseIE(InfoExtractor):
_VALID_URL_BASE = r'https?://(?:[^/]+\.)?pinterest\.(?:com|fr|de|ch|jp|cl|ca|it|co\.uk|nz|ru|com\.au|at|pt|co\.kr|es|com\.mx|dk|ph|th|com\.uy|co|nl|info|kr|ie|vn|com\.vn|ec|mx|in|pe|co\.at|hu|co\.in|co\.nz|id|com\.ec|com\.py|tw|be|uk|com\.bo|com\.pe)' _VALID_URL_BASE = r'https?://(?:[^/]+\.)?pinterest\.(?:com|fr|de|ch|jp|cl|ca|it|co\.uk|nz|ru|com\.au|at|pt|co\.kr|es|com\.mx|dk|ph|th|com\.uy|co|nl|info|kr|ie|vn|com\.vn|ec|mx|in|pe|co\.at|hu|co\.in|co\.nz|id|com\.ec|com\.py|tw|be|uk|com\.bo|com\.pe)'
def _extract_resource(self, webpage, video_id): def _call_api(self, resource, video_id, options):
return self._parse_json( return self._download_json(
self._search_regex( 'https://www.pinterest.com/resource/%sResource/get/' % resource,
r'<script[^>]+\bid=["\']initial-state["\'][^>]*>({.+?})</script>', video_id, 'Download %s JSON metadata' % resource, query={
webpage, 'application json'), 'data': json.dumps({'options': options})
video_id)['resourceResponses'] })['resource_response']
def _extract_video(self, data, extract_formats=True): def _extract_video(self, data, extract_formats=True):
video_id = data['id'] video_id = data['id']
@ -128,13 +131,16 @@ class PinterestIE(PinterestBaseIE):
def _real_extract(self, url): def _real_extract(self, url):
video_id = self._match_id(url) video_id = self._match_id(url)
webpage = self._download_webpage(url, video_id) data = self._call_api(
data = self._extract_resource(webpage, video_id)[0]['response']['data'] 'Pin', video_id, {
'field_set_key': 'unauth_react_main_pin',
'id': video_id,
})['data']
return self._extract_video(data) return self._extract_video(data)
class PinterestCollectionIE(PinterestBaseIE): class PinterestCollectionIE(PinterestBaseIE):
_VALID_URL = r'%s/[^/]+/(?P<id>[^/?#&]+)' % PinterestBaseIE._VALID_URL_BASE _VALID_URL = r'%s/(?P<username>[^/]+)/(?P<id>[^/?#&]+)' % PinterestBaseIE._VALID_URL_BASE
_TESTS = [{ _TESTS = [{
'url': 'https://www.pinterest.ca/mashal0407/cool-diys/', 'url': 'https://www.pinterest.ca/mashal0407/cool-diys/',
'info_dict': { 'info_dict': {
@ -142,6 +148,14 @@ class PinterestCollectionIE(PinterestBaseIE):
'title': 'cool diys', 'title': 'cool diys',
}, },
'playlist_count': 8, 'playlist_count': 8,
}, {
'url': 'https://www.pinterest.ca/fudohub/videos/',
'info_dict': {
'id': '682858430939307450',
'title': 'VIDEOS',
},
'playlist_mincount': 365,
'skip': 'Test with extract_formats=False',
}] }]
@classmethod @classmethod
@ -150,11 +164,24 @@ class PinterestCollectionIE(PinterestBaseIE):
PinterestCollectionIE, cls).suitable(url) PinterestCollectionIE, cls).suitable(url)
def _real_extract(self, url): def _real_extract(self, url):
collection_name = self._match_id(url) username, slug = re.match(self._VALID_URL, url).groups()
webpage = self._download_webpage(url, collection_name) board = self._call_api(
resource = self._extract_resource(webpage, collection_name)[1] 'Board', slug, {
'slug': slug,
'username': username
})['data']
board_id = board['id']
options = {
'board_id': board_id,
'page_size': 250,
}
bookmark = None
entries = [] entries = []
for item in resource['response']['data']: while True:
if bookmark:
options['bookmarks'] = [bookmark]
board_feed = self._call_api('BoardFeed', board_id, options)
for item in (board_feed.get('data') or []):
if not isinstance(item, dict) or item.get('type') != 'pin': if not isinstance(item, dict) or item.get('type') != 'pin':
continue continue
video_id = item.get('id') video_id = item.get('id')
@ -167,10 +194,8 @@ class PinterestCollectionIE(PinterestBaseIE):
# }) # })
# entries.append(video) # entries.append(video)
entries.append(self._extract_video(item)) entries.append(self._extract_video(item))
title = try_get( bookmark = board_feed.get('bookmark')
resource, lambda x: x['options']['board_title'], compat_str) if not bookmark:
collection_id = try_get( break
resource, lambda x: x['options']['board_id'],
compat_str) or collection_name
return self.playlist_result( return self.playlist_result(
entries, playlist_id=collection_id, playlist_title=title) entries, playlist_id=board_id, playlist_title=board.get('name'))