From ab6ade9c4c0c0ef3f3c5ceb3f3c03619d1899309 Mon Sep 17 00:00:00 2001
From: "kerry.nich" <kerry.ann.nicholson@gmail.com>
Date: Tue, 22 Mar 2022 19:19:38 -0400
Subject: [PATCH] Added progress hook updates for postprocessing
---
youtube_dl/YoutubeDL.py | 13 +++-
youtube_dl/postprocessor/common.py | 10 +++
youtube_dl/postprocessor/embedthumbnail.py | 8 +++
youtube_dl/postprocessor/execafterdownload.py | 4 ++
youtube_dl/postprocessor/ffmpeg.py | 63 +++++++++++++++++++
youtube_dl/postprocessor/metadatafromtitle.py | 4 ++
youtube_dl/postprocessor/xattrpp.py | 5 ++
7 files changed, 104 insertions(+), 3 deletions(-)
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 69736acff..3c11b88c9 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -234,11 +234,13 @@ class YoutubeDL(object):
postprocessor.
progress_hooks: A list of functions that get called on download
progress, with a dictionary with the entries
- * status: One of "downloading", "error", or "finished".
+ * status: One of "downloading", "error", "finished",
+ or "postprocessed".
Check this first and ignore unknown values.
- If status is one of "downloading", or "finished", the
- following properties may also be present:
+ If status is one of "downloading", "finished", or
+ "postprocessed", the following properties may also be
+ present:
* filename: The final filename (always present)
* tmpfilename: The filename we're currently writing to
* downloaded_bytes: Bytes on disk
@@ -253,6 +255,9 @@ class YoutubeDL(object):
downloaded video fragment.
* fragment_count: The number of fragments (= individual
files that will be merged)
+ * postprocessor: The specific postprocessor that ran.
+ See youtube_dl/postprocessor/__init__.py
+ for a list of possibilities.
Progress hooks are guaranteed to be called at least once
(with status "finished") if the download is successful.
@@ -2110,6 +2115,8 @@ class YoutubeDL(object):
pps_chain.extend(ie_info['__postprocessors'])
pps_chain.extend(self._pps)
for pp in pps_chain:
+ for ph in self._progress_hooks:
+ pp.add_progress_hook(ph)
files_to_delete = []
try:
files_to_delete, info = pp.run(info)
diff --git a/youtube_dl/postprocessor/common.py b/youtube_dl/postprocessor/common.py
index 599dd1df2..22a27127f 100644
--- a/youtube_dl/postprocessor/common.py
+++ b/youtube_dl/postprocessor/common.py
@@ -33,6 +33,7 @@ class PostProcessor(object):
def __init__(self, downloader=None):
self._downloader = downloader
+ self._progress_hooks = []
def set_downloader(self, downloader):
"""Sets the downloader for this PP."""
@@ -64,6 +65,15 @@ class PostProcessor(object):
def _configuration_args(self, default=[]):
return cli_configuration_args(self._downloader.params, 'postprocessor_args', default)
+ def _hook_progress(self, status):
+ for ph in self._progress_hooks:
+ ph(status)
+
+ def add_progress_hook(self, ph):
+ # See YoutubeDl.py (search for progress_hooks) for a description of
+ # this interface
+ self._progress_hooks.append(ph)
+
class AudioConversionError(PostProcessingError):
pass
diff --git a/youtube_dl/postprocessor/embedthumbnail.py b/youtube_dl/postprocessor/embedthumbnail.py
index 3990908b6..20371baaa 100644
--- a/youtube_dl/postprocessor/embedthumbnail.py
+++ b/youtube_dl/postprocessor/embedthumbnail.py
@@ -127,4 +127,12 @@ class EmbedThumbnailPP(FFmpegPostProcessor):
else:
raise EmbedThumbnailPPError('Only mp3 and m4a/mp4 are supported for thumbnail embedding for now.')
+ fsize = os.path.getsize(encodeFilename(filename))
+ self._hook_progress({
+ 'total_bytes': fsize,
+ 'filename': encodeFilename(filename),
+ 'status': 'postprocessed',
+ 'postprocessor': self.__class__.__name__
+ })
+
return [], info
diff --git a/youtube_dl/postprocessor/execafterdownload.py b/youtube_dl/postprocessor/execafterdownload.py
index 64dabe790..a546d51c9 100644
--- a/youtube_dl/postprocessor/execafterdownload.py
+++ b/youtube_dl/postprocessor/execafterdownload.py
@@ -27,5 +27,9 @@ class ExecAfterDownloadPP(PostProcessor):
if retCode != 0:
raise PostProcessingError(
'Command returned error code %d' % retCode)
+ self._hook_progress({
+ 'status': 'postprocessed',
+ 'postprocessor': self.__class__.__name__
+ })
return [], information
diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 9f76c9d4e..3f358f2a1 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -349,6 +349,14 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor):
new_path, time.time(), information['filetime'],
errnote='Cannot update utime of audio file')
+ fsize = os.path.getsize(new_path)
+ self._hook_progress({
+ 'total_bytes': fsize,
+ 'filename': new_path,
+ 'status': 'postprocessed',
+ 'postprocessor': self.__class__.__name__
+ })
+
return [path], information
@@ -372,6 +380,13 @@ class FFmpegVideoConvertorPP(FFmpegPostProcessor):
information['filepath'] = outpath
information['format'] = self._preferedformat
information['ext'] = self._preferedformat
+ fsize = os.path.getsize(outpath)
+ self._hook_progress({
+ 'total_bytes': fsize,
+ 'filename': outpath,
+ 'status': 'postprocessed',
+ 'postprocessor': self.__class__.__name__
+ })
return [path], information
@@ -429,6 +444,13 @@ class FFmpegEmbedSubtitlePP(FFmpegPostProcessor):
self.run_ffmpeg_multiple_files(input_files, temp_filename, opts)
os.remove(encodeFilename(filename))
os.rename(encodeFilename(temp_filename), encodeFilename(filename))
+ fsize = os.path.getsize(encodeFilename(filename))
+ self._hook_progress({
+ 'total_bytes': fsize,
+ 'filename': encodeFilename(filename),
+ 'status': 'postprocessed',
+ 'postprocessor': self.__class__.__name__
+ })
return sub_filenames, information
@@ -514,6 +536,13 @@ class FFmpegMetadataPP(FFmpegPostProcessor):
os.remove(metadata_filename)
os.remove(encodeFilename(filename))
os.rename(encodeFilename(temp_filename), encodeFilename(filename))
+ fsize = os.path.getsize(encodeFilename(filename))
+ self._hook_progress({
+ 'total_bytes': fsize,
+ 'filename': encodeFilename(filename),
+ 'status': 'postprocessed',
+ 'postprocessor': self.__class__.__name__
+ })
return [], info
@@ -525,6 +554,13 @@ class FFmpegMergerPP(FFmpegPostProcessor):
self._downloader.to_screen('[ffmpeg] Merging formats into "%s"' % filename)
self.run_ffmpeg_multiple_files(info['__files_to_merge'], temp_filename, args)
os.rename(encodeFilename(temp_filename), encodeFilename(filename))
+ fsize = os.path.getsize(encodeFilename(filename))
+ self._hook_progress({
+ 'total_bytes': fsize,
+ 'filename': encodeFilename(filename),
+ 'status': 'postprocessed',
+ 'postprocessor': self.__class__.__name__
+ })
return info['__files_to_merge'], info
def can_merge(self):
@@ -560,6 +596,13 @@ class FFmpegFixupStretchedPP(FFmpegPostProcessor):
os.remove(encodeFilename(filename))
os.rename(encodeFilename(temp_filename), encodeFilename(filename))
+ fsize = os.path.getsize(encodeFilename(filename))
+ self._hook_progress({
+ 'total_bytes': fsize,
+ 'filename': encodeFilename(filename),
+ 'status': 'postprocessed',
+ 'postprocessor': self.__class__.__name__
+ })
return [], info
@@ -578,6 +621,13 @@ class FFmpegFixupM4aPP(FFmpegPostProcessor):
os.remove(encodeFilename(filename))
os.rename(encodeFilename(temp_filename), encodeFilename(filename))
+ fsize = os.path.getsize(encodeFilename(filename))
+ self._hook_progress({
+ 'total_bytes': fsize,
+ 'filename': encodeFilename(filename),
+ 'status': 'postprocessed',
+ 'postprocessor': self.__class__.__name__
+ })
return [], info
@@ -594,6 +644,14 @@ class FFmpegFixupM3u8PP(FFmpegPostProcessor):
os.remove(encodeFilename(filename))
os.rename(encodeFilename(temp_filename), encodeFilename(filename))
+
+ fsize = os.path.getsize(encodeFilename(filename))
+ self._hook_progress({
+ 'total_bytes': fsize,
+ 'filename': encodeFilename(filename),
+ 'status': 'postprocessed',
+ 'postprocessor': self.__class__.__name__
+ })
return [], info
@@ -657,4 +715,9 @@ class FFmpegSubtitlesConvertorPP(FFmpegPostProcessor):
'data': f.read(),
}
+ self._hook_progress({
+ 'status': 'postprocessed',
+ 'postprocessor': self.__class__.__name__
+ })
+
return sub_filenames, info
diff --git a/youtube_dl/postprocessor/metadatafromtitle.py b/youtube_dl/postprocessor/metadatafromtitle.py
index f5c14d974..285664f86 100644
--- a/youtube_dl/postprocessor/metadatafromtitle.py
+++ b/youtube_dl/postprocessor/metadatafromtitle.py
@@ -44,5 +44,9 @@ class MetadataFromTitlePP(PostProcessor):
self._downloader.to_screen(
'[fromtitle] parsed %s: %s'
% (attribute, value if value is not None else 'NA'))
+ self._hook_progress({
+ 'status': 'postprocessed',
+ 'postprocessor': self.__class__.__name__
+ })
return [], info
diff --git a/youtube_dl/postprocessor/xattrpp.py b/youtube_dl/postprocessor/xattrpp.py
index 814dabecf..8ffdd1661 100644
--- a/youtube_dl/postprocessor/xattrpp.py
+++ b/youtube_dl/postprocessor/xattrpp.py
@@ -55,6 +55,11 @@ class XAttrMetadataPP(PostProcessor):
write_xattr(filename, xattrname, byte_value)
num_written += 1
+ self._hook_progress({
+ 'status': 'postprocessed',
+ 'postprocessor': self.__class__.__name__
+ })
+
return [], info
except XAttrUnavailableError as e: