from __future__ import unicode_literals import io import os import subprocess import time import re import collections from string import ascii_letters import random from ..utils import expand_path # from youtube_dl import YoutubeDL # todo fail to import youtubedl class for _NUMERIC_FIELDS # http://python-notes.curiousefficiency.org/en/latest/python_concepts/import_traps.html _NUMERIC_FIELDS = set(( 'width', 'height', 'tbr', 'abr', 'asr', 'vbr', 'fps', 'filesize', 'filesize_approx', 'timestamp', 'upload_year', 'upload_month', 'upload_day', 'duration', 'view_count', 'like_count', 'dislike_count', 'repost_count', 'average_rating', 'comment_count', 'age_limit', 'start_time', 'end_time', 'chapter_number', 'season_number', 'episode_number', 'track_number', 'disc_number', 'release_year', 'playlist_index', )) from .common import AudioConversionError, PostProcessor from ..compat import ( compat_subprocess_get_DEVNULL, ) from ..utils import ( encodeArgument, encodeFilename, get_exe_version, is_outdated_version, PostProcessingError, prepend_extension, shell_quote, subtitles_filename, dfxp2srt, ISO639Utils, replace_extension, ) EXT_TO_OUT_FORMATS = { 'aac': 'adts', 'flac': 'flac', 'm4a': 'ipod', 'mka': 'matroska', 'mkv': 'matroska', 'mpg': 'mpeg', 'ogv': 'ogg', 'ts': 'mpegts', 'wma': 'asf', 'wmv': 'asf', } ACODECS = { 'mp3': 'libmp3lame', 'aac': 'aac', 'flac': 'flac', 'm4a': 'aac', 'opus': 'libopus', 'vorbis': 'libvorbis', 'wav': None, } class FFmpegPostProcessorError(PostProcessingError): pass class FFmpegPostProcessor(PostProcessor): def __init__(self, downloader=None): PostProcessor.__init__(self, downloader) self._determine_executables() def check_version(self): if not self.available: raise FFmpegPostProcessorError('ffmpeg or avconv not found. Please install one.') required_version = '10-0' if self.basename == 'avconv' else '1.0' if is_outdated_version( self._versions[self.basename], required_version): warning = 'Your copy of %s is outdated, update %s to version %s or newer if you encounter any errors.' % ( self.basename, self.basename, required_version) if self._downloader: self._downloader.report_warning(warning) @staticmethod def get_versions(downloader=None): return FFmpegPostProcessor(downloader)._versions def _determine_executables(self): programs = ['avprobe', 'avconv', 'ffmpeg', 'ffprobe'] prefer_ffmpeg = True self.basename = None self.probe_basename = None self._paths = None self._versions = None if self._downloader: prefer_ffmpeg = self._downloader.params.get('prefer_ffmpeg', True) location = self._downloader.params.get('ffmpeg_location') if location is not None: if not os.path.exists(location): self._downloader.report_warning( 'ffmpeg-location %s does not exist! ' 'Continuing without avconv/ffmpeg.' % (location)) self._versions = {} return elif not os.path.isdir(location): basename = os.path.splitext(os.path.basename(location))[0] if basename not in programs: self._downloader.report_warning( 'Cannot identify executable %s, its basename should be one of %s. ' 'Continuing without avconv/ffmpeg.' % (location, ', '.join(programs))) self._versions = {} return None location = os.path.dirname(os.path.abspath(location)) if basename in ('ffmpeg', 'ffprobe'): prefer_ffmpeg = True self._paths = dict( (p, os.path.join(location, p)) for p in programs) self._versions = dict( (p, get_exe_version(self._paths[p], args=['-version'])) for p in programs) if self._versions is None: self._versions = dict( (p, get_exe_version(p, args=['-version'])) for p in programs) self._paths = dict((p, p) for p in programs) if prefer_ffmpeg is False: prefs = ('avconv', 'ffmpeg') else: prefs = ('ffmpeg', 'avconv') for p in prefs: if self._versions[p]: self.basename = p break if prefer_ffmpeg is False: prefs = ('avprobe', 'ffprobe') else: prefs = ('ffprobe', 'avprobe') for p in prefs: if self._versions[p]: self.probe_basename = p break @property def available(self): return self.basename is not None @property def executable(self): return self._paths[self.basename] @property def probe_available(self): return self.probe_basename is not None @property def probe_executable(self): return self._paths[self.probe_basename] def get_audio_codec(self, path): if not self.probe_available: raise PostProcessingError('ffprobe or avprobe not found. Please install one.') try: cmd = [ encodeFilename(self.probe_executable, True), encodeArgument('-show_streams'), encodeFilename(self._ffmpeg_filename_argument(path), True)] if self._downloader.params.get('verbose', False): self._downloader.to_screen('[debug] %s command line: %s' % (self.basename, shell_quote(cmd))) handle = subprocess.Popen(cmd, stderr=compat_subprocess_get_DEVNULL(), stdout=subprocess.PIPE, stdin=subprocess.PIPE) output = handle.communicate()[0] if handle.wait() != 0: return None except (IOError, OSError): return None audio_codec = None for line in output.decode('ascii', 'ignore').split('\n'): if line.startswith('codec_name='): audio_codec = line.split('=')[1].strip() elif line.strip() == 'codec_type=audio' and audio_codec is not None: return audio_codec return None def run_ffmpeg_multiple_files(self, input_paths, out_path, opts, info): self.check_version() oldest_mtime = min( os.stat(encodeFilename(path)).st_mtime for path in input_paths) opts += map(lambda s: self._resolve_postprocessor_arg_var(s, info), self._configuration_args()) files_cmd = [] for path in input_paths: files_cmd.extend([ encodeArgument('-i'), encodeFilename(self._ffmpeg_filename_argument(path), True) ]) cmd = ([encodeFilename(self.executable, True), encodeArgument('-y')] + files_cmd + [encodeArgument(o) for o in opts] + [encodeFilename(self._ffmpeg_filename_argument(out_path), True)]) if self._downloader.params.get('verbose', False): self._downloader.to_screen('[debug] ffmpeg command line: %s' % shell_quote(cmd)) p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE) stdout, stderr = p.communicate() if p.returncode != 0: stderr = stderr.decode('utf-8', 'replace') msg = stderr.strip().split('\n')[-1] raise FFmpegPostProcessorError(msg) self.try_utime(out_path, oldest_mtime, oldest_mtime) def run_ffmpeg(self, path, out_path, opts, info): self.run_ffmpeg_multiple_files([path], out_path, opts, info) def _ffmpeg_filename_argument(self, fn): # Always use 'file:' because the filename may contain ':' (ffmpeg # interprets that as a protocol) or can start with '-' (-- is broken in # ffmpeg, see https://ffmpeg.org/trac/ffmpeg/ticket/2127 for details) # Also leave '-' intact in order not to break streaming to stdout. return 'file:' + fn if fn != '-' else fn def _resolve_postprocessor_arg_var(self, arg, info): # map(lambda s: s % info, args) if "%(" not in arg: return arg template_dict = dict(info) template_dict['epoch'] = int(time.time()) # autonumber_size = self.params.get('autonumber_size') # if autonumber_size is None: autonumber_size = 5 # # template_dict['autonumber'] = self.params.get('autonumber_start', 1) - 1 + self._num_downloads # if template_dict.get('resolution') is None: # if template_dict.get('width') and template_dict.get('height'): # template_dict['resolution'] = '%dx%d' % (template_dict['width'], template_dict['height']) # elif template_dict.get('height'): # template_dict['resolution'] = '%sp' % template_dict['height'] # elif template_dict.get('width'): # template_dict['resolution'] = '%dx?' % template_dict['width'] # sanitize = lambda k, v: sanitize_filename( # compat_str(v), # restricted=self.params.get('restrictfilenames'), # is_id=(k == 'id' or k.endswith('_id'))) # template_dict = dict((k, v if isinstance(v, compat_numeric_types) else sanitize(k, v)) # for k, v in template_dict.items() # if v is not None and not isinstance(v, (list, tuple, dict))) template_dict = collections.defaultdict(lambda: 'NA', template_dict) outtmpl = arg # For fields playlist_index and autonumber convert all occurrences # of %(field)s to %(field)0Nd for backward compatibility field_size_compat_map = { 'playlist_index': len(str(template_dict['n_entries'])), 'autonumber': autonumber_size, } FIELD_SIZE_COMPAT_RE = r'(?autonumber|playlist_index)\)s' mobj = re.search(FIELD_SIZE_COMPAT_RE, outtmpl) if mobj: outtmpl = re.sub( FIELD_SIZE_COMPAT_RE, r'%%(\1)0%dd' % field_size_compat_map[mobj.group('field')], outtmpl) # Missing numeric fields used together with integer presentation types # in format specification will break the argument substitution since # string 'NA' is returned for missing fields. We will patch output # template for missing fields to meet string presentation type. for numeric_field in _NUMERIC_FIELDS: if numeric_field not in template_dict: # As of [1] format syntax is: # %[mapping_key][conversion_flags][minimum_width][.precision][length_modifier]type # 1. https://docs.python.org/2/library/stdtypes.html#string-formatting FORMAT_RE = r'''(?x) (?