From f7e1bfea7a5d53dc900c43ce84674d17d1c7d0e6 Mon Sep 17 00:00:00 2001 From: vijeth-aradhya Date: Wed, 31 May 2017 01:21:34 +0530 Subject: [PATCH 01/55] media_types/video/util.py: Add accepted resolutions Add dict consisting of all the accepted resolutions with their dimensions. This should be eventually moved to config where the instance owner can modify it. --- mediagoblin/media_types/video/util.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/mediagoblin/media_types/video/util.py b/mediagoblin/media_types/video/util.py index 8b65d839..1f5e907d 100644 --- a/mediagoblin/media_types/video/util.py +++ b/mediagoblin/media_types/video/util.py @@ -18,6 +18,15 @@ import logging from mediagoblin import mg_globals as mgg +ACCEPTED_RESOLUTIONS = { + '144p' : (256, 144), + '240p' : (352, 240), + '360p' : (480, 360), + '480p' : (858, 480), + '720p' : (1280, 720), + '1080p' : (1920, 1080), +} + _log = logging.getLogger(__name__) From 2963b0a18231aac5d5fda3ae2ecc55a0b5c08823 Mon Sep 17 00:00:00 2001 From: vijeth-aradhya Date: Fri, 2 Jun 2017 01:55:42 +0530 Subject: [PATCH 02/55] media_types/video/processing.py: Move store_metadata Move store_metadata to self.common_setup() so that it's easy to parallelize self.transcode() later. --- mediagoblin/media_types/video/processing.py | 22 ++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/mediagoblin/media_types/video/processing.py b/mediagoblin/media_types/video/processing.py index ca3087a2..e6b4a0b2 100644 --- a/mediagoblin/media_types/video/processing.py +++ b/mediagoblin/media_types/video/processing.py @@ -178,6 +178,14 @@ class CommonVideoProcessor(MediaProcessor): self.transcoder = transcoders.VideoTranscoder() self.did_transcode = False + # Extract metadata and keep a record of it + self.metadata = transcoders.discover(self.process_filename) + + # metadata's stream info here is a DiscovererContainerInfo instance, + # it gets split into DiscovererAudioInfo and DiscovererVideoInfo; + # metadata itself has container-related data in tags, like video-codec + store_metadata(self.entry, self.metadata) + def copy_original(self): # If we didn't transcode, then we need to keep the original if not self.did_transcode or \ @@ -246,20 +254,12 @@ class CommonVideoProcessor(MediaProcessor): if self._skip_processing('webm_video', **file_metadata): return - # Extract metadata and keep a record of it - metadata = transcoders.discover(self.process_filename) - - # metadata's stream info here is a DiscovererContainerInfo instance, - # it gets split into DiscovererAudioInfo and DiscovererVideoInfo; - # metadata itself has container-related data in tags, like video-codec - store_metadata(self.entry, metadata) - - orig_dst_dimensions = (metadata.get_video_streams()[0].get_width(), - metadata.get_video_streams()[0].get_height()) + orig_dst_dimensions = (self.metadata.get_video_streams()[0].get_width(), + self.metadata.get_video_streams()[0].get_height()) # Figure out whether or not we need to transcode this video or # if we can skip it - if skip_transcode(metadata, medium_size): + if skip_transcode(self.metadata, medium_size): _log.debug('Skipping transcoding') dst_dimensions = orig_dst_dimensions From d3390c4391f18f4f1ac76557a98b106452baf295 Mon Sep 17 00:00:00 2001 From: vijeth-aradhya Date: Sun, 4 Jun 2017 23:45:18 +0530 Subject: [PATCH 03/55] Add workflow method to ProcessingManager This method just raises NotImplementedError if the specific media processing manager does not have a workflow method. --- mediagoblin/processing/__init__.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/mediagoblin/processing/__init__.py b/mediagoblin/processing/__init__.py index 29345227..7e05b242 100644 --- a/mediagoblin/processing/__init__.py +++ b/mediagoblin/processing/__init__.py @@ -257,6 +257,13 @@ class ProcessingManager(object): return processor + def workflow(self): + """ + Returns the Celery command needed to proceed with media processing + *This method has to be implemented in all media types* + """ + raise NotImplementedError + def request_from_args(args, which_args): """ From 81c59ef06b1e53fa0d9b177b5a8e7978d1b01186 Mon Sep 17 00:00:00 2001 From: vijeth-aradhya Date: Mon, 5 Jun 2017 01:58:00 +0530 Subject: [PATCH 04/55] Add workflow method to VideoProcessingManager This commit makes sure the old celery call works perfectly when workflow method is introduced. --- mediagoblin/media_types/video/processing.py | 6 ++++++ mediagoblin/processing/__init__.py | 2 +- mediagoblin/submit/lib.py | 10 ++++++---- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/mediagoblin/media_types/video/processing.py b/mediagoblin/media_types/video/processing.py index e6b4a0b2..d3fa9d5f 100644 --- a/mediagoblin/media_types/video/processing.py +++ b/mediagoblin/media_types/video/processing.py @@ -28,6 +28,7 @@ from mediagoblin.processing import ( ProcessingManager, request_from_args, get_process_filename, store_public, copy_original) +from mediagoblin.processing.task import ProcessMedia from mediagoblin.tools.translate import lazy_pass_to_ugettext as _ from mediagoblin.media_types import MissingComponents @@ -503,3 +504,8 @@ class VideoProcessingManager(ProcessingManager): self.add_processor(InitialProcessor) self.add_processor(Resizer) self.add_processor(Transcoder) + + def workflow(self, entry, feed_url, reprocess_action, reprocess_info=None): + ProcessMedia().apply_async( + [entry.id, feed_url, reprocess_action, reprocess_info], {}, + task_id=entry.queued_task_id) diff --git a/mediagoblin/processing/__init__.py b/mediagoblin/processing/__init__.py index 7e05b242..4e5853c1 100644 --- a/mediagoblin/processing/__init__.py +++ b/mediagoblin/processing/__init__.py @@ -257,7 +257,7 @@ class ProcessingManager(object): return processor - def workflow(self): + def workflow(self, entry, feed_url, reprocess_action, reprocess_info=None): """ Returns the Celery command needed to proceed with media processing *This method has to be implemented in all media types* diff --git a/mediagoblin/submit/lib.py b/mediagoblin/submit/lib.py index 08a603e9..402eb851 100644 --- a/mediagoblin/submit/lib.py +++ b/mediagoblin/submit/lib.py @@ -28,7 +28,7 @@ from mediagoblin.tools.response import json_response from mediagoblin.tools.text import convert_to_tag_list_of_dicts from mediagoblin.tools.federation import create_activity, create_generator from mediagoblin.db.models import Collection, MediaEntry, ProcessingMetaData -from mediagoblin.processing import mark_entry_failed +from mediagoblin.processing import mark_entry_failed, get_entry_and_processing_manager from mediagoblin.processing.task import ProcessMedia from mediagoblin.notifications import add_comment_subscription from mediagoblin.media_types import sniff_media @@ -262,10 +262,12 @@ def run_process_media(entry, feed_url=None, :param reprocess_action: What particular action should be run. :param reprocess_info: A dict containing all of the necessary reprocessing info for the given media_type""" + + reprocess_info = reprocess_info or {} + entry, manager = get_entry_and_processing_manager(entry.id) + try: - ProcessMedia().apply_async( - [entry.id, feed_url, reprocess_action, reprocess_info], {}, - task_id=entry.queued_task_id) + manager.workflow(entry, feed_url, reprocess_action, reprocess_info) except BaseException as exc: # The purpose of this section is because when running in "lazy" # or always-eager-with-exceptions-propagated celery mode that From c62181f4bca1ae7f74d4bd8ec15615337d23ec9a Mon Sep 17 00:00:00 2001 From: vijeth-aradhya Date: Mon, 5 Jun 2017 02:11:56 +0530 Subject: [PATCH 05/55] Add workflow method to other media ProcessingManagers Add workflow method to all the media types other than video. Calls the old celery task as it is. --- mediagoblin/media_types/ascii/processing.py | 5 +++++ mediagoblin/media_types/audio/processing.py | 5 +++++ mediagoblin/media_types/image/processing.py | 6 ++++++ mediagoblin/media_types/pdf/processing.py | 5 +++++ mediagoblin/media_types/raw_image/processing.py | 5 +++++ mediagoblin/media_types/stl/processing.py | 5 +++++ 6 files changed, 31 insertions(+) diff --git a/mediagoblin/media_types/ascii/processing.py b/mediagoblin/media_types/ascii/processing.py index 00d04e63..71ccc86e 100644 --- a/mediagoblin/media_types/ascii/processing.py +++ b/mediagoblin/media_types/ascii/processing.py @@ -273,3 +273,8 @@ class AsciiProcessingManager(ProcessingManager): super(AsciiProcessingManager, self).__init__() self.add_processor(InitialProcessor) self.add_processor(Resizer) + + def workflow(self, entry, feed_url, reprocess_action, reprocess_info=None): + ProcessMedia().apply_async( + [entry.id, feed_url, reprocess_action, reprocess_info], {}, + task_id=entry.queued_task_id) diff --git a/mediagoblin/media_types/audio/processing.py b/mediagoblin/media_types/audio/processing.py index 427309de..a83d60f7 100644 --- a/mediagoblin/media_types/audio/processing.py +++ b/mediagoblin/media_types/audio/processing.py @@ -365,3 +365,8 @@ class AudioProcessingManager(ProcessingManager): self.add_processor(InitialProcessor) self.add_processor(Resizer) self.add_processor(Transcoder) + + def workflow(self, entry, feed_url, reprocess_action, reprocess_info=None): + ProcessMedia().apply_async( + [entry.id, feed_url, reprocess_action, reprocess_info], {}, + task_id=entry.queued_task_id) diff --git a/mediagoblin/media_types/image/processing.py b/mediagoblin/media_types/image/processing.py index 14091d6e..42234eff 100644 --- a/mediagoblin/media_types/image/processing.py +++ b/mediagoblin/media_types/image/processing.py @@ -431,6 +431,12 @@ class ImageProcessingManager(ProcessingManager): self.add_processor(Resizer) self.add_processor(MetadataProcessing) + def workflow(self, entry, feed_url, reprocess_action, reprocess_info=None): + ProcessMedia().apply_async( + [entry.id, feed_url, reprocess_action, reprocess_info], {}, + task_id=entry.queued_task_id) + + if __name__ == '__main__': import sys import pprint diff --git a/mediagoblin/media_types/pdf/processing.py b/mediagoblin/media_types/pdf/processing.py index ac4bab6d..d93b19bb 100644 --- a/mediagoblin/media_types/pdf/processing.py +++ b/mediagoblin/media_types/pdf/processing.py @@ -470,3 +470,8 @@ class PdfProcessingManager(ProcessingManager): super(PdfProcessingManager, self).__init__() self.add_processor(InitialProcessor) self.add_processor(Resizer) + + def workflow(self, entry, feed_url, reprocess_action, reprocess_info=None): + ProcessMedia().apply_async( + [entry.id, feed_url, reprocess_action, reprocess_info], {}, + task_id=entry.queued_task_id) diff --git a/mediagoblin/media_types/raw_image/processing.py b/mediagoblin/media_types/raw_image/processing.py index 740ba2dd..a385d563 100644 --- a/mediagoblin/media_types/raw_image/processing.py +++ b/mediagoblin/media_types/raw_image/processing.py @@ -80,3 +80,8 @@ class RawImageProcessingManager(ProcessingManager): super(RawImageProcessingManager, self).__init__() self.add_processor(InitialRawProcessor) self.add_processor(Resizer) + + def workflow(self, entry, feed_url, reprocess_action, reprocess_info=None): + ProcessMedia().apply_async( + [entry.id, feed_url, reprocess_action, reprocess_info], {}, + task_id=entry.queued_task_id) diff --git a/mediagoblin/media_types/stl/processing.py b/mediagoblin/media_types/stl/processing.py index 55764aeb..7f2f350d 100644 --- a/mediagoblin/media_types/stl/processing.py +++ b/mediagoblin/media_types/stl/processing.py @@ -368,3 +368,8 @@ class StlProcessingManager(ProcessingManager): super(StlProcessingManager, self).__init__() self.add_processor(InitialProcessor) self.add_processor(Resizer) + + def workflow(self, entry, feed_url, reprocess_action, reprocess_info=None): + ProcessMedia().apply_async( + [entry.id, feed_url, reprocess_action, reprocess_info], {}, + task_id=entry.queued_task_id) From 16ef1164c15b13c20945890cbb5cc4a8e4e73852 Mon Sep 17 00:00:00 2001 From: vijeth-aradhya Date: Wed, 7 Jun 2017 20:39:26 +0530 Subject: [PATCH 06/55] Modify CommonVideoPrcoessor to accomodate mulitple qualities A few trivial modifciations like replacing store_metadata, etc needed to accomodate multiple qualities. --- mediagoblin/media_types/video/processing.py | 84 +++++++++------------ 1 file changed, 37 insertions(+), 47 deletions(-) diff --git a/mediagoblin/media_types/video/processing.py b/mediagoblin/media_types/video/processing.py index d3fa9d5f..a7e2d20c 100644 --- a/mediagoblin/media_types/video/processing.py +++ b/mediagoblin/media_types/video/processing.py @@ -165,9 +165,10 @@ class CommonVideoProcessor(MediaProcessor): """ Provides a base for various video processing steps """ - acceptable_files = ['original', 'best_quality', 'webm_video'] + acceptable_files = ['original, best_quality', 'webm_144p', 'webm_360p', + 'webm_480p', 'webm_720p', 'webm_1080p', 'webm_video'] - def common_setup(self): + def common_setup(self, resolution=None): self.video_config = mgg \ .global_config['plugins'][MEDIA_TYPE] @@ -179,32 +180,23 @@ class CommonVideoProcessor(MediaProcessor): self.transcoder = transcoders.VideoTranscoder() self.did_transcode = False - # Extract metadata and keep a record of it - self.metadata = transcoders.discover(self.process_filename) - - # metadata's stream info here is a DiscovererContainerInfo instance, - # it gets split into DiscovererAudioInfo and DiscovererVideoInfo; - # metadata itself has container-related data in tags, like video-codec - store_metadata(self.entry, self.metadata) + if resolution: + self.curr_file = 'webm_' + str(resolution) + self.part_filename = (self.name_builder.fill('{basename}.' + + str(resolution) + '.webm')) + else: + self.curr_file = 'webm_video' + self.part_filename = self.name_builder.fill('{basename}.medium.webm') def copy_original(self): # If we didn't transcode, then we need to keep the original - if not self.did_transcode or \ - (self.video_config['keep_original'] and self.did_transcode): - copy_original( - self.entry, self.process_filename, - self.name_builder.fill('{basename}{ext}')) + raise NotImplementedError def _keep_best(self): """ If there is no original, keep the best file that we have """ - if not self.entry.media_files.get('best_quality'): - # Save the best quality file if no original? - if not self.entry.media_files.get('original') and \ - self.entry.media_files.get('webm_video'): - self.entry.media_files['best_quality'] = self.entry \ - .media_files['webm_video'] + raise NotImplementedError def _skip_processing(self, keyname, **kwargs): file_metadata = self.entry.get_file_metadata(keyname) @@ -213,7 +205,7 @@ class CommonVideoProcessor(MediaProcessor): return False skip = True - if keyname == 'webm_video': + if 'webm' in keyname: if kwargs.get('medium_size') != file_metadata.get('medium_size'): skip = False elif kwargs.get('vp8_quality') != file_metadata.get('vp8_quality'): @@ -233,8 +225,7 @@ class CommonVideoProcessor(MediaProcessor): def transcode(self, medium_size=None, vp8_quality=None, vp8_threads=None, vorbis_quality=None): progress_callback = ProgressCallback(self.entry) - tmp_dst = os.path.join(self.workbench.dir, - self.name_builder.fill('{basename}.medium.webm')) + tmp_dst = os.path.join(self.workbench.dir, self.part_filename) if not medium_size: medium_size = ( @@ -252,24 +243,23 @@ class CommonVideoProcessor(MediaProcessor): 'vp8_quality': vp8_quality, 'vorbis_quality': vorbis_quality} - if self._skip_processing('webm_video', **file_metadata): + if self._skip_processing(self.curr_file, **file_metadata): return - orig_dst_dimensions = (self.metadata.get_video_streams()[0].get_width(), - self.metadata.get_video_streams()[0].get_height()) + metadata = transcoders.discover(self.process_filename) + orig_dst_dimensions = (metadata.get_video_streams()[0].get_width(), + metadata.get_video_streams()[0].get_height()) # Figure out whether or not we need to transcode this video or # if we can skip it - if skip_transcode(self.metadata, medium_size): + if skip_transcode(metadata, medium_size): _log.debug('Skipping transcoding') - dst_dimensions = orig_dst_dimensions - # If there is an original and transcoded, delete the transcoded # since it must be of lower quality then the original if self.entry.media_files.get('original') and \ - self.entry.media_files.get('webm_video'): - self.entry.media_files['webm_video'].delete() + self.entry.media_files.get(self.curr_file): + self.entry.media_files[self.curr_file].delete() else: self.transcoder.transcode(self.process_filename, tmp_dst, @@ -279,27 +269,16 @@ class CommonVideoProcessor(MediaProcessor): progress_callback=progress_callback, dimensions=tuple(medium_size)) if self.transcoder.dst_data: - video_info = self.transcoder.dst_data.get_video_streams()[0] - dst_dimensions = (video_info.get_width(), - video_info.get_height()) - self._keep_best() - # Push transcoded video to public storage _log.debug('Saving medium...') store_public(self.entry, 'webm_video', tmp_dst, self.name_builder.fill('{basename}.medium.webm')) _log.debug('Saved medium') - self.entry.set_file_metadata('webm_video', **file_metadata) + # Is this the file_metadata that paroneayea was talking about? + self.entry.set_file_metadata(self.curr_file, **file_metadata) self.did_transcode = True - else: - dst_dimensions = orig_dst_dimensions - - # Save the width and height of the transcoded video - self.entry.media_data_init( - width=dst_dimensions[0], - height=dst_dimensions[1]) def generate_thumb(self, thumb_size=None): # Temporary file for the video thumbnail (cleaned up with workbench) @@ -331,6 +310,17 @@ class CommonVideoProcessor(MediaProcessor): self.entry.set_file_metadata('thumb', thumb_size=thumb_size) + def store_orig_metadata(self): + + # Extract metadata and keep a record of it + metadata = transcoders.discover(self.process_filename) + + # metadata's stream info here is a DiscovererContainerInfo instance, + # it gets split into DiscovererAudioInfo and DiscovererVideoInfo; + # metadata itself has container-related data in tags, like video-codec + store_metadata(self.entry, metadata) + + class InitialProcessor(CommonVideoProcessor): """ Initial processing steps for new video @@ -387,9 +377,9 @@ class InitialProcessor(CommonVideoProcessor): 'vorbis_quality', 'thumb_size']) def process(self, medium_size=None, vp8_threads=None, vp8_quality=None, - vorbis_quality=None, thumb_size=None): - self.common_setup() - + vorbis_quality=None, thumb_size=None, resolution=None): + self.common_setup(resolution=resolution) + self.store_orig_metadata() self.transcode(medium_size=medium_size, vp8_quality=vp8_quality, vp8_threads=vp8_threads, vorbis_quality=vorbis_quality) From 7cc9b6d1c7ab174485ddfbf8db63f14162e01d7b Mon Sep 17 00:00:00 2001 From: vijeth-aradhya Date: Wed, 7 Jun 2017 20:42:42 +0530 Subject: [PATCH 07/55] Add main_task, complimentary_task and cleanup --- mediagoblin/media_types/video/processing.py | 25 +++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/mediagoblin/media_types/video/processing.py b/mediagoblin/media_types/video/processing.py index a7e2d20c..4dee8d55 100644 --- a/mediagoblin/media_types/video/processing.py +++ b/mediagoblin/media_types/video/processing.py @@ -160,6 +160,31 @@ def store_metadata(media_entry, metadata): if len(stored_metadata): media_entry.media_data_init(orig_metadata=stored_metadata) +# ===================== + + +def main_task(**process_info): + processor = CommonVideoProcessor(process_info['manager'], process_info['entry']) + processor.common_setup(process_info['resolution']) + processor.transcode(medium_size=process_info['medium_size'], vp8_quality=process_info['vp8_quality'], + vp8_threads=process_info['vp8_threads'], vorbis_quality=process_info['vorbis_quality']) + processor.generate_thumb(thumb_size=process_info['thumb_size']) + processor.store_orig_metadata() + + +def complimentary_task(**process_info): + processor = CommonVideoProcessor(process_info['manager'], process_info['entry']) + processor.common_setup(process_info['resolution']) + processor.transcode(medium_size=process_info['medium_size'], vp8_quality=process_info['vp8_quality'], + vp8_threads=process_info['vp8_threads'], vorbis_quality=process_info['vorbis_quality']) + + +def processing_cleanup(**process_info): + processor = CommonVideoProcessor(process_info['manager'], process_info['entry']) + processor.delete_queue_file() + +# ===================== + class CommonVideoProcessor(MediaProcessor): """ From 9a27fa60a42cd39596cb8c4bb0331279b998bae7 Mon Sep 17 00:00:00 2001 From: vijeth-aradhya Date: Sun, 11 Jun 2017 19:07:58 +0530 Subject: [PATCH 08/55] Add additional celery config settings Fixes older webm_video backward compatibilty issue. Add 'default' queue to be used from now. Add other necessary celery settings for priority. --- mediagoblin/config_spec.ini | 5 +++-- mediagoblin/init/celery/__init__.py | 8 ++++++++ mediagoblin/media_types/video/processing.py | 5 ++++- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/mediagoblin/config_spec.ini b/mediagoblin/config_spec.ini index bd3003d0..e1a0d0c5 100644 --- a/mediagoblin/config_spec.ini +++ b/mediagoblin/config_spec.ini @@ -154,6 +154,7 @@ CELERY_RESULT_DBURI = string(default="sqlite:///%(here)s/celery.db") # default kombu stuff BROKER_URL = string(default="amqp://") +CELERY_DEFAULT_QUEUE = string(default="default") # known booleans CELERY_RESULT_PERSISTENT = boolean() @@ -165,7 +166,7 @@ CELERY_EAGER_PROPAGATES_EXCEPTIONS = boolean() CELERY_IGNORE_RESULT = boolean() CELERY_TRACK_STARTED = boolean() CELERY_DISABLE_RATE_LIMITS = boolean() -CELERY_ACKS_LATE = boolean() +CELERY_ACKS_LATE = boolean(default=True) CELERY_STORE_ERRORS_EVEN_IF_IGNORED = boolean() CELERY_SEND_TASK_ERROR_EMAILS = boolean() CELERY_SEND_EVENTS = boolean() @@ -175,7 +176,7 @@ CELERY_REDIRECT_STDOUTS = boolean() # known ints CELERYD_CONCURRENCY = integer() -CELERYD_PREFETCH_MULTIPLIER = integer() +CELERYD_PREFETCH_MULTIPLIER = integer(default=1) CELERY_AMQP_TASK_RESULT_EXPIRES = integer() CELERY_AMQP_TASK_RESULT_CONNECTION_MAX = integer() REDIS_PORT = integer() diff --git a/mediagoblin/init/celery/__init__.py b/mediagoblin/init/celery/__init__.py index 780e0055..9a67942c 100644 --- a/mediagoblin/init/celery/__init__.py +++ b/mediagoblin/init/celery/__init__.py @@ -22,6 +22,7 @@ import logging import six from celery import Celery +from kombu import Exchange, Queue from mediagoblin.tools.pluginapi import hook_runall @@ -32,6 +33,7 @@ MANDATORY_CELERY_IMPORTS = [ 'mediagoblin.processing.task', 'mediagoblin.notifications.task', 'mediagoblin.submit.task', + 'mediagoblin.media_types.video.processing', ] DEFAULT_SETTINGS_MODULE = 'mediagoblin.init.celery.dummy_settings_module' @@ -47,6 +49,12 @@ def get_celery_settings_dict(app_config, global_config, else: celery_conf = {} + # Add x-max-priority to config + celery_conf['CELERY_QUEUES'] = ( + Queue('default', Exchange('default'), routing_key='default', + queue_arguments={'x-max-priority': 10}), + ) + celery_settings = {} # Add all celery settings from config diff --git a/mediagoblin/media_types/video/processing.py b/mediagoblin/media_types/video/processing.py index 4dee8d55..5cae42f5 100644 --- a/mediagoblin/media_types/video/processing.py +++ b/mediagoblin/media_types/video/processing.py @@ -18,6 +18,7 @@ import argparse import os.path import logging import datetime +import celery import six @@ -163,6 +164,7 @@ def store_metadata(media_entry, metadata): # ===================== +@celery.task() def main_task(**process_info): processor = CommonVideoProcessor(process_info['manager'], process_info['entry']) processor.common_setup(process_info['resolution']) @@ -172,6 +174,7 @@ def main_task(**process_info): processor.store_orig_metadata() +@celery.task() def complimentary_task(**process_info): processor = CommonVideoProcessor(process_info['manager'], process_info['entry']) processor.common_setup(process_info['resolution']) @@ -179,6 +182,7 @@ def complimentary_task(**process_info): vp8_threads=process_info['vp8_threads'], vorbis_quality=process_info['vorbis_quality']) +@celery.task() def processing_cleanup(**process_info): processor = CommonVideoProcessor(process_info['manager'], process_info['entry']) processor.delete_queue_file() @@ -408,7 +412,6 @@ class InitialProcessor(CommonVideoProcessor): self.transcode(medium_size=medium_size, vp8_quality=vp8_quality, vp8_threads=vp8_threads, vorbis_quality=vorbis_quality) - self.copy_original() self.generate_thumb(thumb_size=thumb_size) self.delete_queue_file() From 25ecdec9971394064063db397232eb7f0e89fae3 Mon Sep 17 00:00:00 2001 From: vijeth-aradhya Date: Mon, 12 Jun 2017 20:53:23 +0530 Subject: [PATCH 09/55] Add priority to the celery tasks Few more changes to be made before executing the tasks. Also #1 should be handled soon after this. --- mediagoblin/media_types/ascii/processing.py | 3 +- mediagoblin/media_types/audio/processing.py | 3 +- mediagoblin/media_types/image/processing.py | 3 +- mediagoblin/media_types/pdf/processing.py | 3 +- .../media_types/raw_image/processing.py | 3 +- mediagoblin/media_types/stl/processing.py | 3 +- mediagoblin/media_types/video/processing.py | 40 +++++++++++++------ mediagoblin/processing/__init__.py | 3 +- mediagoblin/submit/lib.py | 2 +- 9 files changed, 42 insertions(+), 21 deletions(-) diff --git a/mediagoblin/media_types/ascii/processing.py b/mediagoblin/media_types/ascii/processing.py index 71ccc86e..823dc4fd 100644 --- a/mediagoblin/media_types/ascii/processing.py +++ b/mediagoblin/media_types/ascii/processing.py @@ -274,7 +274,8 @@ class AsciiProcessingManager(ProcessingManager): self.add_processor(InitialProcessor) self.add_processor(Resizer) - def workflow(self, entry, feed_url, reprocess_action, reprocess_info=None): + def workflow(self, entry, manager, feed_url, reprocess_action, + reprocess_info=None): ProcessMedia().apply_async( [entry.id, feed_url, reprocess_action, reprocess_info], {}, task_id=entry.queued_task_id) diff --git a/mediagoblin/media_types/audio/processing.py b/mediagoblin/media_types/audio/processing.py index a83d60f7..b74364bc 100644 --- a/mediagoblin/media_types/audio/processing.py +++ b/mediagoblin/media_types/audio/processing.py @@ -366,7 +366,8 @@ class AudioProcessingManager(ProcessingManager): self.add_processor(Resizer) self.add_processor(Transcoder) - def workflow(self, entry, feed_url, reprocess_action, reprocess_info=None): + def workflow(self, entry, manager, feed_url, reprocess_action, + reprocess_info=None): ProcessMedia().apply_async( [entry.id, feed_url, reprocess_action, reprocess_info], {}, task_id=entry.queued_task_id) diff --git a/mediagoblin/media_types/image/processing.py b/mediagoblin/media_types/image/processing.py index 42234eff..a189fef3 100644 --- a/mediagoblin/media_types/image/processing.py +++ b/mediagoblin/media_types/image/processing.py @@ -431,7 +431,8 @@ class ImageProcessingManager(ProcessingManager): self.add_processor(Resizer) self.add_processor(MetadataProcessing) - def workflow(self, entry, feed_url, reprocess_action, reprocess_info=None): + def workflow(self, entry, manager, feed_url, reprocess_action, + reprocess_info=None): ProcessMedia().apply_async( [entry.id, feed_url, reprocess_action, reprocess_info], {}, task_id=entry.queued_task_id) diff --git a/mediagoblin/media_types/pdf/processing.py b/mediagoblin/media_types/pdf/processing.py index d93b19bb..6a13c8e3 100644 --- a/mediagoblin/media_types/pdf/processing.py +++ b/mediagoblin/media_types/pdf/processing.py @@ -471,7 +471,8 @@ class PdfProcessingManager(ProcessingManager): self.add_processor(InitialProcessor) self.add_processor(Resizer) - def workflow(self, entry, feed_url, reprocess_action, reprocess_info=None): + def workflow(self, entry, manager, feed_url, reprocess_action, + reprocess_info=None): ProcessMedia().apply_async( [entry.id, feed_url, reprocess_action, reprocess_info], {}, task_id=entry.queued_task_id) diff --git a/mediagoblin/media_types/raw_image/processing.py b/mediagoblin/media_types/raw_image/processing.py index a385d563..7f2d155a 100644 --- a/mediagoblin/media_types/raw_image/processing.py +++ b/mediagoblin/media_types/raw_image/processing.py @@ -81,7 +81,8 @@ class RawImageProcessingManager(ProcessingManager): self.add_processor(InitialRawProcessor) self.add_processor(Resizer) - def workflow(self, entry, feed_url, reprocess_action, reprocess_info=None): + def workflow(self, entry, manager, feed_url, reprocess_action, + reprocess_info=None): ProcessMedia().apply_async( [entry.id, feed_url, reprocess_action, reprocess_info], {}, task_id=entry.queued_task_id) diff --git a/mediagoblin/media_types/stl/processing.py b/mediagoblin/media_types/stl/processing.py index 7f2f350d..9dd6d49b 100644 --- a/mediagoblin/media_types/stl/processing.py +++ b/mediagoblin/media_types/stl/processing.py @@ -369,7 +369,8 @@ class StlProcessingManager(ProcessingManager): self.add_processor(InitialProcessor) self.add_processor(Resizer) - def workflow(self, entry, feed_url, reprocess_action, reprocess_info=None): + def workflow(self, entry, manager, feed_url, reprocess_action, + reprocess_info=None): ProcessMedia().apply_async( [entry.id, feed_url, reprocess_action, reprocess_info], {}, task_id=entry.queued_task_id) diff --git a/mediagoblin/media_types/video/processing.py b/mediagoblin/media_types/video/processing.py index 5cae42f5..d039c24b 100644 --- a/mediagoblin/media_types/video/processing.py +++ b/mediagoblin/media_types/video/processing.py @@ -22,6 +22,7 @@ import celery import six +from celery import group, chord from mediagoblin import mg_globals as mgg from mediagoblin.processing import ( FilenameBuilder, BaseProcessingFail, @@ -34,7 +35,7 @@ from mediagoblin.tools.translate import lazy_pass_to_ugettext as _ from mediagoblin.media_types import MissingComponents from . import transcoders -from .util import skip_transcode +from .util import skip_transcode, ACCEPTED_RESOLUTIONS _log = logging.getLogger(__name__) _log.setLevel(logging.DEBUG) @@ -165,26 +166,26 @@ def store_metadata(media_entry, metadata): @celery.task() -def main_task(**process_info): +def main_task(resolution, medium_size, **process_info): processor = CommonVideoProcessor(process_info['manager'], process_info['entry']) - processor.common_setup(process_info['resolution']) - processor.transcode(medium_size=process_info['medium_size'], vp8_quality=process_info['vp8_quality'], + processor.common_setup(resolution) + processor.transcode(medium_size=medium_size, vp8_quality=process_info['vp8_quality'], vp8_threads=process_info['vp8_threads'], vorbis_quality=process_info['vorbis_quality']) processor.generate_thumb(thumb_size=process_info['thumb_size']) processor.store_orig_metadata() @celery.task() -def complimentary_task(**process_info): +def complimentary_task(resolution, medium_size, **process_info): processor = CommonVideoProcessor(process_info['manager'], process_info['entry']) - processor.common_setup(process_info['resolution']) - processor.transcode(medium_size=process_info['medium_size'], vp8_quality=process_info['vp8_quality'], + processor.common_setup(resolution) + processor.transcode(medium_size=medium_size, vp8_quality=process_info['vp8_quality'], vp8_threads=process_info['vp8_threads'], vorbis_quality=process_info['vorbis_quality']) @celery.task() -def processing_cleanup(**process_info): - processor = CommonVideoProcessor(process_info['manager'], process_info['entry']) +def processing_cleanup(entry, manager): + processor = CommonVideoProcessor(manager, entry) # is it manager, entry or entry, manager? processor.delete_queue_file() # ===================== @@ -523,7 +524,20 @@ class VideoProcessingManager(ProcessingManager): self.add_processor(Resizer) self.add_processor(Transcoder) - def workflow(self, entry, feed_url, reprocess_action, reprocess_info=None): - ProcessMedia().apply_async( - [entry.id, feed_url, reprocess_action, reprocess_info], {}, - task_id=entry.queued_task_id) + def workflow(self, entry, manager, feed_url, reprocess_action, + reprocess_info=None): + + reprocess_info['entry'] = entry.id # ? + reprocess_info['manager'] = manager # can celery serialize this? + + # Add args + + transcoding_tasks = group( + main_task.signature(queue='default', priority=5, immutable=True), + complimentary_task.signature(queue='default', priority=4, immutable=True), + complimentary_task.signature(queue='default', priority=3, immutable=True), + complimentary_task.signature(queue='default', priority=2, immutable=True) + complimentary_task.signature(queue='default', priority=1, immutable=True) + ) + + chord(transcoding_tasks)(processing_cleanup.signature(queue='default', immutable=True)) diff --git a/mediagoblin/processing/__init__.py b/mediagoblin/processing/__init__.py index 4e5853c1..76f81faa 100644 --- a/mediagoblin/processing/__init__.py +++ b/mediagoblin/processing/__init__.py @@ -257,7 +257,8 @@ class ProcessingManager(object): return processor - def workflow(self, entry, feed_url, reprocess_action, reprocess_info=None): + def workflow(self, entry, manager, feed_url, reprocess_action, + reprocess_info=None): """ Returns the Celery command needed to proceed with media processing *This method has to be implemented in all media types* diff --git a/mediagoblin/submit/lib.py b/mediagoblin/submit/lib.py index 402eb851..1c78f73a 100644 --- a/mediagoblin/submit/lib.py +++ b/mediagoblin/submit/lib.py @@ -267,7 +267,7 @@ def run_process_media(entry, feed_url=None, entry, manager = get_entry_and_processing_manager(entry.id) try: - manager.workflow(entry, feed_url, reprocess_action, reprocess_info) + manager.workflow(entry, manager, feed_url, reprocess_action, reprocess_info) except BaseException as exc: # The purpose of this section is because when running in "lazy" # or always-eager-with-exceptions-propagated celery mode that From bd011c940eeeddd060ccf921ad3519d20d77a015 Mon Sep 17 00:00:00 2001 From: vijeth-aradhya Date: Mon, 12 Jun 2017 23:17:44 +0530 Subject: [PATCH 10/55] Few more changes to celery priority tasking Addition of arguments to the celery tasks. --- mediagoblin/media_types/video/processing.py | 27 ++++++++++++--------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/mediagoblin/media_types/video/processing.py b/mediagoblin/media_types/video/processing.py index d039c24b..64cacb5f 100644 --- a/mediagoblin/media_types/video/processing.py +++ b/mediagoblin/media_types/video/processing.py @@ -185,7 +185,7 @@ def complimentary_task(resolution, medium_size, **process_info): @celery.task() def processing_cleanup(entry, manager): - processor = CommonVideoProcessor(manager, entry) # is it manager, entry or entry, manager? + processor = CommonVideoProcessor(manager, entry) processor.delete_queue_file() # ===================== @@ -527,17 +527,22 @@ class VideoProcessingManager(ProcessingManager): def workflow(self, entry, manager, feed_url, reprocess_action, reprocess_info=None): - reprocess_info['entry'] = entry.id # ? - reprocess_info['manager'] = manager # can celery serialize this? - - # Add args + reprocess_info['entry'] = entry + reprocess_info['manager'] = manager transcoding_tasks = group( - main_task.signature(queue='default', priority=5, immutable=True), - complimentary_task.signature(queue='default', priority=4, immutable=True), - complimentary_task.signature(queue='default', priority=3, immutable=True), - complimentary_task.signature(queue='default', priority=2, immutable=True) - complimentary_task.signature(queue='default', priority=1, immutable=True) + main_task.signature(args=('480p', ACCEPTED_RESOLUTIONS['480p']), + kwargs=reprocess_info, queue='default', + priority=5, immutable=True), + complimentary_task.signature(args=('360p', ACCEPTED_RESOLUTIONS['360p']), + kwargs=reprocess_info, queue='default', + priority=4, immutable=True), + complimentary_task.signature(args=('720p', ACCEPTED_RESOLUTIONS['720p']), + kwargs=reprocess_info, queue='default', + priority=3, immutable=True), ) - chord(transcoding_tasks)(processing_cleanup.signature(queue='default', immutable=True)) + cleanup_task = processing_cleanup.signature(args=(entry, manager), + queue='default', immutable=True) + + chord(transcoding_tasks)(cleanup_task) From d77eb56280f57e547294e29e6a1b2b4d46c15ac6 Mon Sep 17 00:00:00 2001 From: vijeth-aradhya Date: Tue, 13 Jun 2017 01:43:43 +0530 Subject: [PATCH 11/55] Celery Priority testing with debug statements Error at this line: `self.entry.set_file_metadata(self.curr_file, **file_metadata)` Otherwise, celery part should work fine. --- mediagoblin/init/celery/__init__.py | 3 + mediagoblin/media_types/ascii/processing.py | 3 +- mediagoblin/media_types/audio/processing.py | 3 +- mediagoblin/media_types/image/processing.py | 3 +- mediagoblin/media_types/pdf/processing.py | 3 +- .../media_types/raw_image/processing.py | 3 +- mediagoblin/media_types/stl/processing.py | 3 +- mediagoblin/media_types/video/processing.py | 101 ++++++++++++------ mediagoblin/processing/__init__.py | 3 +- mediagoblin/submit/lib.py | 2 +- 10 files changed, 80 insertions(+), 47 deletions(-) diff --git a/mediagoblin/init/celery/__init__.py b/mediagoblin/init/celery/__init__.py index 9a67942c..a3335958 100644 --- a/mediagoblin/init/celery/__init__.py +++ b/mediagoblin/init/celery/__init__.py @@ -55,6 +55,9 @@ def get_celery_settings_dict(app_config, global_config, queue_arguments={'x-max-priority': 10}), ) + print "CELERY_ACKS_LATE", celery_conf['CELERY_ACKS_LATE'] + print "CELERYD_PREFETCH_MULTIPLIER", celery_conf['CELERYD_PREFETCH_MULTIPLIER'] + celery_settings = {} # Add all celery settings from config diff --git a/mediagoblin/media_types/ascii/processing.py b/mediagoblin/media_types/ascii/processing.py index 823dc4fd..c9b47fb5 100644 --- a/mediagoblin/media_types/ascii/processing.py +++ b/mediagoblin/media_types/ascii/processing.py @@ -274,8 +274,7 @@ class AsciiProcessingManager(ProcessingManager): self.add_processor(InitialProcessor) self.add_processor(Resizer) - def workflow(self, entry, manager, feed_url, reprocess_action, - reprocess_info=None): + def workflow(self, entry_id, feed_url, reprocess_action, reprocess_info=None): ProcessMedia().apply_async( [entry.id, feed_url, reprocess_action, reprocess_info], {}, task_id=entry.queued_task_id) diff --git a/mediagoblin/media_types/audio/processing.py b/mediagoblin/media_types/audio/processing.py index b74364bc..15d0b0a7 100644 --- a/mediagoblin/media_types/audio/processing.py +++ b/mediagoblin/media_types/audio/processing.py @@ -366,8 +366,7 @@ class AudioProcessingManager(ProcessingManager): self.add_processor(Resizer) self.add_processor(Transcoder) - def workflow(self, entry, manager, feed_url, reprocess_action, - reprocess_info=None): + def workflow(self, entry_id, feed_url, reprocess_action, reprocess_info=None): ProcessMedia().apply_async( [entry.id, feed_url, reprocess_action, reprocess_info], {}, task_id=entry.queued_task_id) diff --git a/mediagoblin/media_types/image/processing.py b/mediagoblin/media_types/image/processing.py index a189fef3..7224a8fd 100644 --- a/mediagoblin/media_types/image/processing.py +++ b/mediagoblin/media_types/image/processing.py @@ -431,8 +431,7 @@ class ImageProcessingManager(ProcessingManager): self.add_processor(Resizer) self.add_processor(MetadataProcessing) - def workflow(self, entry, manager, feed_url, reprocess_action, - reprocess_info=None): + def workflow(self, entry_id, feed_url, reprocess_action, reprocess_info=None): ProcessMedia().apply_async( [entry.id, feed_url, reprocess_action, reprocess_info], {}, task_id=entry.queued_task_id) diff --git a/mediagoblin/media_types/pdf/processing.py b/mediagoblin/media_types/pdf/processing.py index 6a13c8e3..e6e6e0a9 100644 --- a/mediagoblin/media_types/pdf/processing.py +++ b/mediagoblin/media_types/pdf/processing.py @@ -471,8 +471,7 @@ class PdfProcessingManager(ProcessingManager): self.add_processor(InitialProcessor) self.add_processor(Resizer) - def workflow(self, entry, manager, feed_url, reprocess_action, - reprocess_info=None): + def workflow(self, entry_id, feed_url, reprocess_action, reprocess_info=None): ProcessMedia().apply_async( [entry.id, feed_url, reprocess_action, reprocess_info], {}, task_id=entry.queued_task_id) diff --git a/mediagoblin/media_types/raw_image/processing.py b/mediagoblin/media_types/raw_image/processing.py index 7f2d155a..4bfd9f3a 100644 --- a/mediagoblin/media_types/raw_image/processing.py +++ b/mediagoblin/media_types/raw_image/processing.py @@ -81,8 +81,7 @@ class RawImageProcessingManager(ProcessingManager): self.add_processor(InitialRawProcessor) self.add_processor(Resizer) - def workflow(self, entry, manager, feed_url, reprocess_action, - reprocess_info=None): + def workflow(self, entry_id, feed_url, reprocess_action, reprocess_info=None): ProcessMedia().apply_async( [entry.id, feed_url, reprocess_action, reprocess_info], {}, task_id=entry.queued_task_id) diff --git a/mediagoblin/media_types/stl/processing.py b/mediagoblin/media_types/stl/processing.py index 9dd6d49b..cd3ffd8c 100644 --- a/mediagoblin/media_types/stl/processing.py +++ b/mediagoblin/media_types/stl/processing.py @@ -369,8 +369,7 @@ class StlProcessingManager(ProcessingManager): self.add_processor(InitialProcessor) self.add_processor(Resizer) - def workflow(self, entry, manager, feed_url, reprocess_action, - reprocess_info=None): + def workflow(self, entry_id, feed_url, reprocess_action, reprocess_info=None): ProcessMedia().apply_async( [entry.id, feed_url, reprocess_action, reprocess_info], {}, task_id=entry.queued_task_id) diff --git a/mediagoblin/media_types/video/processing.py b/mediagoblin/media_types/video/processing.py index 64cacb5f..c3257c84 100644 --- a/mediagoblin/media_types/video/processing.py +++ b/mediagoblin/media_types/video/processing.py @@ -29,7 +29,7 @@ from mediagoblin.processing import ( ProgressCallback, MediaProcessor, ProcessingManager, request_from_args, get_process_filename, store_public, - copy_original) + copy_original, get_entry_and_processing_manager) from mediagoblin.processing.task import ProcessMedia from mediagoblin.tools.translate import lazy_pass_to_ugettext as _ from mediagoblin.media_types import MissingComponents @@ -166,27 +166,35 @@ def store_metadata(media_entry, metadata): @celery.task() -def main_task(resolution, medium_size, **process_info): - processor = CommonVideoProcessor(process_info['manager'], process_info['entry']) - processor.common_setup(resolution) - processor.transcode(medium_size=medium_size, vp8_quality=process_info['vp8_quality'], - vp8_threads=process_info['vp8_threads'], vorbis_quality=process_info['vorbis_quality']) - processor.generate_thumb(thumb_size=process_info['thumb_size']) - processor.store_orig_metadata() +def main_task(entry_id, resolution, medium_size, **process_info): + entry, manager = get_entry_and_processing_manager(entry_id) + print "\nEntered main_task\n" + with CommonVideoProcessor(manager, entry) as processor: + processor.common_setup(resolution) + processor.transcode(medium_size=tuple(medium_size), vp8_quality=process_info['vp8_quality'], + vp8_threads=process_info['vp8_threads'], vorbis_quality=process_info['vorbis_quality']) + processor.generate_thumb(thumb_size=process_info['thumb_size']) + processor.store_orig_metadata() + print "\nExited main_task\n" @celery.task() -def complimentary_task(resolution, medium_size, **process_info): - processor = CommonVideoProcessor(process_info['manager'], process_info['entry']) - processor.common_setup(resolution) - processor.transcode(medium_size=medium_size, vp8_quality=process_info['vp8_quality'], - vp8_threads=process_info['vp8_threads'], vorbis_quality=process_info['vorbis_quality']) +def complimentary_task(entry_id, resolution, medium_size, **process_info): + entry, manager = get_entry_and_processing_manager(entry_id) + print "\nEntered complimentary_task\n" + with CommonVideoProcessor(manager, entry) as processor: + processor.common_setup(resolution) + processor.transcode(medium_size=tuple(medium_size), vp8_quality=process_info['vp8_quality'], + vp8_threads=process_info['vp8_threads'], vorbis_quality=process_info['vorbis_quality']) + print "\nExited complimentary_task\n" @celery.task() -def processing_cleanup(entry, manager): - processor = CommonVideoProcessor(manager, entry) - processor.delete_queue_file() +def processing_cleanup(entry_id): + entry, manager = get_entry_and_processing_manager(entry_id) + with CommonVideoProcessor(manager, entry) as processor: + processor.delete_queue_file() + print "\nDeleted queue_file\n" # ===================== @@ -206,7 +214,7 @@ class CommonVideoProcessor(MediaProcessor): self.process_filename = get_process_filename( self.entry, self.workbench, self.acceptable_files) self.name_builder = FilenameBuilder(self.process_filename) - + self.transcoder = transcoders.VideoTranscoder() self.did_transcode = False @@ -218,6 +226,8 @@ class CommonVideoProcessor(MediaProcessor): self.curr_file = 'webm_video' self.part_filename = self.name_builder.fill('{basename}.medium.webm') + print self.curr_file, ": Done common_setup()" + def copy_original(self): # If we didn't transcode, then we need to keep the original raise NotImplementedError @@ -254,6 +264,7 @@ class CommonVideoProcessor(MediaProcessor): def transcode(self, medium_size=None, vp8_quality=None, vp8_threads=None, vorbis_quality=None): + print self.curr_file, ": Enter transcode" progress_callback = ProgressCallback(self.entry) tmp_dst = os.path.join(self.workbench.dir, self.part_filename) @@ -292,25 +303,34 @@ class CommonVideoProcessor(MediaProcessor): self.entry.media_files[self.curr_file].delete() else: + print self.curr_file, ": ->1.1" + print type(medium_size) + medium_size = tuple(medium_size) + print type(medium_size) + print self.curr_file, ": ->1.2" self.transcoder.transcode(self.process_filename, tmp_dst, vp8_quality=vp8_quality, vp8_threads=vp8_threads, vorbis_quality=vorbis_quality, progress_callback=progress_callback, dimensions=tuple(medium_size)) + print self.curr_file, ": ->2" if self.transcoder.dst_data: + print self.curr_file, ": ->3" # Push transcoded video to public storage _log.debug('Saving medium...') - store_public(self.entry, 'webm_video', tmp_dst, - self.name_builder.fill('{basename}.medium.webm')) + store_public(self.entry, 'webm_video', tmp_dst, self.part_filename) _log.debug('Saved medium') + print self.curr_file, ": ->4" # Is this the file_metadata that paroneayea was talking about? self.entry.set_file_metadata(self.curr_file, **file_metadata) self.did_transcode = True + print self.curr_file, ": Done transcode()" def generate_thumb(self, thumb_size=None): + print self.curr_file, ": Enter generate_thumb()" # Temporary file for the video thumbnail (cleaned up with workbench) tmp_thumb = os.path.join(self.workbench.dir, self.name_builder.fill( @@ -339,9 +359,10 @@ class CommonVideoProcessor(MediaProcessor): self.name_builder.fill('{basename}.thumbnail.jpg')) self.entry.set_file_metadata('thumb', thumb_size=thumb_size) + print self.curr_file, ": Done generate_thumb()" def store_orig_metadata(self): - + print self.curr_file, ": 2" # Extract metadata and keep a record of it metadata = transcoders.discover(self.process_filename) @@ -524,25 +545,41 @@ class VideoProcessingManager(ProcessingManager): self.add_processor(Resizer) self.add_processor(Transcoder) - def workflow(self, entry, manager, feed_url, reprocess_action, - reprocess_info=None): + def workflow(self, entry_id, feed_url, reprocess_action, reprocess_info=None): - reprocess_info['entry'] = entry - reprocess_info['manager'] = manager + reprocess_info = reprocess_info or {} + if 'vp8_quality' not in reprocess_info: + reprocess_info['vp8_quality'] = None + if 'vorbis_quality' not in reprocess_info: + reprocess_info['vorbis_quality'] = None + if 'vp8_threads' not in reprocess_info: + reprocess_info['vp8_threads'] = None + if 'thumb_size' not in reprocess_info: + reprocess_info['thumb_size'] = None - transcoding_tasks = group( - main_task.signature(args=('480p', ACCEPTED_RESOLUTIONS['480p']), + transcoding_tasks = group([ + main_task.signature(args=(entry_id, '480p', ACCEPTED_RESOLUTIONS['480p']), kwargs=reprocess_info, queue='default', priority=5, immutable=True), - complimentary_task.signature(args=('360p', ACCEPTED_RESOLUTIONS['360p']), + ]) + + cleanup_task = processing_cleanup.signature(args=(entry_id,), + queue='default', immutable=True) + + """ + complimentary_task.signature(args=(entry_id, '360p', ACCEPTED_RESOLUTIONS['360p']), kwargs=reprocess_info, queue='default', priority=4, immutable=True), - complimentary_task.signature(args=('720p', ACCEPTED_RESOLUTIONS['720p']), + complimentary_task.signature(args=(entry_id, '720p', ACCEPTED_RESOLUTIONS['720p']), kwargs=reprocess_info, queue='default', priority=3, immutable=True), - ) + main_task.apply_async(args=(entry_id, '480p', ACCEPTED_RESOLUTIONS['480p']), + kwargs=reprocess_info, queue='default', + priority=5, immutable=True) + processing_cleanup.apply_async(args=(entry_id,), queue='default', immutable=True) + """ - cleanup_task = processing_cleanup.signature(args=(entry, manager), - queue='default', immutable=True) - chord(transcoding_tasks)(cleanup_task) + + # main_task(entry_id, '480p', ACCEPTED_RESOLUTIONS['480p'], **reprocess_info) + # processing_cleanup(entry_id) diff --git a/mediagoblin/processing/__init__.py b/mediagoblin/processing/__init__.py index 76f81faa..98031bbc 100644 --- a/mediagoblin/processing/__init__.py +++ b/mediagoblin/processing/__init__.py @@ -257,8 +257,7 @@ class ProcessingManager(object): return processor - def workflow(self, entry, manager, feed_url, reprocess_action, - reprocess_info=None): + def workflow(self, entry_id, feed_url, reprocess_action, reprocess_info=None): """ Returns the Celery command needed to proceed with media processing *This method has to be implemented in all media types* diff --git a/mediagoblin/submit/lib.py b/mediagoblin/submit/lib.py index 1c78f73a..f347e715 100644 --- a/mediagoblin/submit/lib.py +++ b/mediagoblin/submit/lib.py @@ -267,7 +267,7 @@ def run_process_media(entry, feed_url=None, entry, manager = get_entry_and_processing_manager(entry.id) try: - manager.workflow(entry, manager, feed_url, reprocess_action, reprocess_info) + manager.workflow(entry.id, feed_url, reprocess_action, reprocess_info) except BaseException as exc: # The purpose of this section is because when running in "lazy" # or always-eager-with-exceptions-propagated celery mode that From 982fbde8e8bb6d0377e4e942a794e520774b6d72 Mon Sep 17 00:00:00 2001 From: vijeth-aradhya Date: Tue, 13 Jun 2017 23:24:32 +0530 Subject: [PATCH 12/55] Individual resolution transcoding fixes Few minor changes which fix the errors. Although, the user still cannot view the video since the front end part needs to changed. Fixes #3 --- mediagoblin/media_types/video/processing.py | 12 +++++------- mediagoblin/submit/lib.py | 1 - 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/mediagoblin/media_types/video/processing.py b/mediagoblin/media_types/video/processing.py index c3257c84..a59a7c0b 100644 --- a/mediagoblin/media_types/video/processing.py +++ b/mediagoblin/media_types/video/processing.py @@ -191,6 +191,7 @@ def complimentary_task(entry_id, resolution, medium_size, **process_info): @celery.task() def processing_cleanup(entry_id): + print "\nEnter processing_cleanup()\n" entry, manager = get_entry_and_processing_manager(entry_id) with CommonVideoProcessor(manager, entry) as processor: processor.delete_queue_file() @@ -303,11 +304,7 @@ class CommonVideoProcessor(MediaProcessor): self.entry.media_files[self.curr_file].delete() else: - print self.curr_file, ": ->1.1" - print type(medium_size) - medium_size = tuple(medium_size) - print type(medium_size) - print self.curr_file, ": ->1.2" + print self.curr_file, ": ->1" self.transcoder.transcode(self.process_filename, tmp_dst, vp8_quality=vp8_quality, vp8_threads=vp8_threads, @@ -319,7 +316,7 @@ class CommonVideoProcessor(MediaProcessor): print self.curr_file, ": ->3" # Push transcoded video to public storage _log.debug('Saving medium...') - store_public(self.entry, 'webm_video', tmp_dst, self.part_filename) + store_public(self.entry, self.curr_file, tmp_dst, self.part_filename) _log.debug('Saved medium') print self.curr_file, ": ->4" @@ -362,7 +359,7 @@ class CommonVideoProcessor(MediaProcessor): print self.curr_file, ": Done generate_thumb()" def store_orig_metadata(self): - print self.curr_file, ": 2" + print self.curr_file, ": Enter store_orig_metadata()" # Extract metadata and keep a record of it metadata = transcoders.discover(self.process_filename) @@ -370,6 +367,7 @@ class CommonVideoProcessor(MediaProcessor): # it gets split into DiscovererAudioInfo and DiscovererVideoInfo; # metadata itself has container-related data in tags, like video-codec store_metadata(self.entry, metadata) + print self.curr_file, ": Done store_orig_metadata()" class InitialProcessor(CommonVideoProcessor): diff --git a/mediagoblin/submit/lib.py b/mediagoblin/submit/lib.py index f347e715..a6d564f1 100644 --- a/mediagoblin/submit/lib.py +++ b/mediagoblin/submit/lib.py @@ -263,7 +263,6 @@ def run_process_media(entry, feed_url=None, :param reprocess_info: A dict containing all of the necessary reprocessing info for the given media_type""" - reprocess_info = reprocess_info or {} entry, manager = get_entry_and_processing_manager(entry.id) try: From 869048dd15108ab379466a2df8f5fd4af1e4e068 Mon Sep 17 00:00:00 2001 From: vijeth-aradhya Date: Tue, 13 Jun 2017 23:54:40 +0530 Subject: [PATCH 13/55] Add complimentary_tasks to the celery group Add a couple of complimentary_tasks to check if priority is working or not. --- mediagoblin/media_types/video/processing.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/mediagoblin/media_types/video/processing.py b/mediagoblin/media_types/video/processing.py index a59a7c0b..586a0c85 100644 --- a/mediagoblin/media_types/video/processing.py +++ b/mediagoblin/media_types/video/processing.py @@ -191,7 +191,7 @@ def complimentary_task(entry_id, resolution, medium_size, **process_info): @celery.task() def processing_cleanup(entry_id): - print "\nEnter processing_cleanup()\n" + print "\nEntered processing_cleanup()\n" entry, manager = get_entry_and_processing_manager(entry_id) with CommonVideoProcessor(manager, entry) as processor: processor.delete_queue_file() @@ -559,18 +559,18 @@ class VideoProcessingManager(ProcessingManager): main_task.signature(args=(entry_id, '480p', ACCEPTED_RESOLUTIONS['480p']), kwargs=reprocess_info, queue='default', priority=5, immutable=True), - ]) - - cleanup_task = processing_cleanup.signature(args=(entry_id,), - queue='default', immutable=True) - - """ complimentary_task.signature(args=(entry_id, '360p', ACCEPTED_RESOLUTIONS['360p']), kwargs=reprocess_info, queue='default', priority=4, immutable=True), complimentary_task.signature(args=(entry_id, '720p', ACCEPTED_RESOLUTIONS['720p']), kwargs=reprocess_info, queue='default', priority=3, immutable=True), + ]) + + cleanup_task = processing_cleanup.signature(args=(entry_id,), + queue='default', immutable=True) + + """ main_task.apply_async(args=(entry_id, '480p', ACCEPTED_RESOLUTIONS['480p']), kwargs=reprocess_info, queue='default', priority=5, immutable=True) @@ -578,6 +578,3 @@ class VideoProcessingManager(ProcessingManager): """ chord(transcoding_tasks)(cleanup_task) - - # main_task(entry_id, '480p', ACCEPTED_RESOLUTIONS['480p'], **reprocess_info) - # processing_cleanup(entry_id) From 5b91098ca7084d4d420302eacf795d8d142429f9 Mon Sep 17 00:00:00 2001 From: vijeth-aradhya Date: Tue, 13 Jun 2017 23:58:42 +0530 Subject: [PATCH 14/55] Celery concurrency is set to one --- mediagoblin/config_spec.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediagoblin/config_spec.ini b/mediagoblin/config_spec.ini index e1a0d0c5..c696c327 100644 --- a/mediagoblin/config_spec.ini +++ b/mediagoblin/config_spec.ini @@ -175,7 +175,7 @@ CELERYD_LOG_COLOR = boolean() CELERY_REDIRECT_STDOUTS = boolean() # known ints -CELERYD_CONCURRENCY = integer() +CELERYD_CONCURRENCY = integer(default=1) CELERYD_PREFETCH_MULTIPLIER = integer(default=1) CELERY_AMQP_TASK_RESULT_EXPIRES = integer() CELERY_AMQP_TASK_RESULT_CONNECTION_MAX = integer() From 33d5ac6c4de44aa8d98a8cf759c79351f5a59885 Mon Sep 17 00:00:00 2001 From: vijeth-aradhya Date: Sat, 17 Jun 2017 00:53:39 +0530 Subject: [PATCH 15/55] Simple hack to handle main workflow problem Remove redundunt workflow methods from the other media type's processing.py. Fixes #1 --- mediagoblin/media_types/ascii/processing.py | 5 ----- mediagoblin/media_types/audio/processing.py | 5 ----- mediagoblin/media_types/image/processing.py | 5 ----- mediagoblin/media_types/pdf/processing.py | 5 ----- .../media_types/raw_image/processing.py | 5 ----- mediagoblin/media_types/stl/processing.py | 5 ----- mediagoblin/media_types/video/processing.py | 21 +++++++------------ mediagoblin/processing/__init__.py | 5 ++--- mediagoblin/submit/lib.py | 6 +++++- 9 files changed, 15 insertions(+), 47 deletions(-) diff --git a/mediagoblin/media_types/ascii/processing.py b/mediagoblin/media_types/ascii/processing.py index c9b47fb5..00d04e63 100644 --- a/mediagoblin/media_types/ascii/processing.py +++ b/mediagoblin/media_types/ascii/processing.py @@ -273,8 +273,3 @@ class AsciiProcessingManager(ProcessingManager): super(AsciiProcessingManager, self).__init__() self.add_processor(InitialProcessor) self.add_processor(Resizer) - - def workflow(self, entry_id, feed_url, reprocess_action, reprocess_info=None): - ProcessMedia().apply_async( - [entry.id, feed_url, reprocess_action, reprocess_info], {}, - task_id=entry.queued_task_id) diff --git a/mediagoblin/media_types/audio/processing.py b/mediagoblin/media_types/audio/processing.py index 15d0b0a7..427309de 100644 --- a/mediagoblin/media_types/audio/processing.py +++ b/mediagoblin/media_types/audio/processing.py @@ -365,8 +365,3 @@ class AudioProcessingManager(ProcessingManager): self.add_processor(InitialProcessor) self.add_processor(Resizer) self.add_processor(Transcoder) - - def workflow(self, entry_id, feed_url, reprocess_action, reprocess_info=None): - ProcessMedia().apply_async( - [entry.id, feed_url, reprocess_action, reprocess_info], {}, - task_id=entry.queued_task_id) diff --git a/mediagoblin/media_types/image/processing.py b/mediagoblin/media_types/image/processing.py index 7224a8fd..7ddf3f35 100644 --- a/mediagoblin/media_types/image/processing.py +++ b/mediagoblin/media_types/image/processing.py @@ -431,11 +431,6 @@ class ImageProcessingManager(ProcessingManager): self.add_processor(Resizer) self.add_processor(MetadataProcessing) - def workflow(self, entry_id, feed_url, reprocess_action, reprocess_info=None): - ProcessMedia().apply_async( - [entry.id, feed_url, reprocess_action, reprocess_info], {}, - task_id=entry.queued_task_id) - if __name__ == '__main__': import sys diff --git a/mediagoblin/media_types/pdf/processing.py b/mediagoblin/media_types/pdf/processing.py index e6e6e0a9..ac4bab6d 100644 --- a/mediagoblin/media_types/pdf/processing.py +++ b/mediagoblin/media_types/pdf/processing.py @@ -470,8 +470,3 @@ class PdfProcessingManager(ProcessingManager): super(PdfProcessingManager, self).__init__() self.add_processor(InitialProcessor) self.add_processor(Resizer) - - def workflow(self, entry_id, feed_url, reprocess_action, reprocess_info=None): - ProcessMedia().apply_async( - [entry.id, feed_url, reprocess_action, reprocess_info], {}, - task_id=entry.queued_task_id) diff --git a/mediagoblin/media_types/raw_image/processing.py b/mediagoblin/media_types/raw_image/processing.py index 4bfd9f3a..740ba2dd 100644 --- a/mediagoblin/media_types/raw_image/processing.py +++ b/mediagoblin/media_types/raw_image/processing.py @@ -80,8 +80,3 @@ class RawImageProcessingManager(ProcessingManager): super(RawImageProcessingManager, self).__init__() self.add_processor(InitialRawProcessor) self.add_processor(Resizer) - - def workflow(self, entry_id, feed_url, reprocess_action, reprocess_info=None): - ProcessMedia().apply_async( - [entry.id, feed_url, reprocess_action, reprocess_info], {}, - task_id=entry.queued_task_id) diff --git a/mediagoblin/media_types/stl/processing.py b/mediagoblin/media_types/stl/processing.py index cd3ffd8c..55764aeb 100644 --- a/mediagoblin/media_types/stl/processing.py +++ b/mediagoblin/media_types/stl/processing.py @@ -368,8 +368,3 @@ class StlProcessingManager(ProcessingManager): super(StlProcessingManager, self).__init__() self.add_processor(InitialProcessor) self.add_processor(Resizer) - - def workflow(self, entry_id, feed_url, reprocess_action, reprocess_info=None): - ProcessMedia().apply_async( - [entry.id, feed_url, reprocess_action, reprocess_info], {}, - task_id=entry.queued_task_id) diff --git a/mediagoblin/media_types/video/processing.py b/mediagoblin/media_types/video/processing.py index 586a0c85..e881636c 100644 --- a/mediagoblin/media_types/video/processing.py +++ b/mediagoblin/media_types/video/processing.py @@ -30,7 +30,6 @@ from mediagoblin.processing import ( ProcessingManager, request_from_args, get_process_filename, store_public, copy_original, get_entry_and_processing_manager) -from mediagoblin.processing.task import ProcessMedia from mediagoblin.tools.translate import lazy_pass_to_ugettext as _ from mediagoblin.media_types import MissingComponents @@ -543,7 +542,7 @@ class VideoProcessingManager(ProcessingManager): self.add_processor(Resizer) self.add_processor(Transcoder) - def workflow(self, entry_id, feed_url, reprocess_action, reprocess_info=None): + def workflow(self, entry, feed_url, reprocess_action, reprocess_info=None): reprocess_info = reprocess_info or {} if 'vp8_quality' not in reprocess_info: @@ -556,25 +555,21 @@ class VideoProcessingManager(ProcessingManager): reprocess_info['thumb_size'] = None transcoding_tasks = group([ - main_task.signature(args=(entry_id, '480p', ACCEPTED_RESOLUTIONS['480p']), + main_task.signature(args=(entry.id, '480p', ACCEPTED_RESOLUTIONS['480p']), kwargs=reprocess_info, queue='default', priority=5, immutable=True), - complimentary_task.signature(args=(entry_id, '360p', ACCEPTED_RESOLUTIONS['360p']), + complimentary_task.signature(args=(entry.id, '360p', ACCEPTED_RESOLUTIONS['360p']), kwargs=reprocess_info, queue='default', priority=4, immutable=True), - complimentary_task.signature(args=(entry_id, '720p', ACCEPTED_RESOLUTIONS['720p']), + complimentary_task.signature(args=(entry.id, '720p', ACCEPTED_RESOLUTIONS['720p']), kwargs=reprocess_info, queue='default', priority=3, immutable=True), ]) - cleanup_task = processing_cleanup.signature(args=(entry_id,), + cleanup_task = processing_cleanup.signature(args=(entry.id,), queue='default', immutable=True) - """ - main_task.apply_async(args=(entry_id, '480p', ACCEPTED_RESOLUTIONS['480p']), - kwargs=reprocess_info, queue='default', - priority=5, immutable=True) - processing_cleanup.apply_async(args=(entry_id,), queue='default', immutable=True) - """ - chord(transcoding_tasks)(cleanup_task) + + # Not sure what to return since we are scheduling the task here itself + return 1 diff --git a/mediagoblin/processing/__init__.py b/mediagoblin/processing/__init__.py index 98031bbc..7d407a36 100644 --- a/mediagoblin/processing/__init__.py +++ b/mediagoblin/processing/__init__.py @@ -257,12 +257,11 @@ class ProcessingManager(object): return processor - def workflow(self, entry_id, feed_url, reprocess_action, reprocess_info=None): + def workflow(self, entry, feed_url, reprocess_action, reprocess_info=None): """ Returns the Celery command needed to proceed with media processing - *This method has to be implemented in all media types* """ - raise NotImplementedError + return None def request_from_args(args, which_args): diff --git a/mediagoblin/submit/lib.py b/mediagoblin/submit/lib.py index a6d564f1..b228dbd1 100644 --- a/mediagoblin/submit/lib.py +++ b/mediagoblin/submit/lib.py @@ -266,7 +266,11 @@ def run_process_media(entry, feed_url=None, entry, manager = get_entry_and_processing_manager(entry.id) try: - manager.workflow(entry.id, feed_url, reprocess_action, reprocess_info) + wf = manager.workflow(entry, feed_url, reprocess_action, reprocess_info) + if wf is None: + ProcessMedia().apply_async( + [entry.id, feed_url, reprocess_action, reprocess_info], {}, + task_id=entry.queued_task_id) except BaseException as exc: # The purpose of this section is because when running in "lazy" # or always-eager-with-exceptions-propagated celery mode that From 52814967cd2ed0c93d68ae06e11322442e651293 Mon Sep 17 00:00:00 2001 From: vijeth-aradhya Date: Sat, 17 Jun 2017 01:58:58 +0530 Subject: [PATCH 16/55] Fix test_celery_setup error Add mediagoblin.media_types.video.processing as a task module. --- mediagoblin/tests/test_celery_setup.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mediagoblin/tests/test_celery_setup.py b/mediagoblin/tests/test_celery_setup.py index 0749c7f4..606e5906 100644 --- a/mediagoblin/tests/test_celery_setup.py +++ b/mediagoblin/tests/test_celery_setup.py @@ -48,8 +48,9 @@ def test_setup_celery_from_config(): assert isinstance(fake_celery_module.CELERYD_ETA_SCHEDULER_PRECISION, float) assert fake_celery_module.CELERY_RESULT_PERSISTENT is True assert fake_celery_module.CELERY_IMPORTS == [ - 'foo.bar.baz', 'this.is.an.import', 'mediagoblin.processing.task', \ - 'mediagoblin.notifications.task', 'mediagoblin.submit.task'] + 'foo.bar.baz', 'this.is.an.import', 'mediagoblin.processing.task', + 'mediagoblin.notifications.task', 'mediagoblin.submit.task', + 'mediagoblin.media_types.video.processing'] assert fake_celery_module.CELERY_RESULT_BACKEND == 'database' assert fake_celery_module.CELERY_RESULT_DBURI == ( 'sqlite:///' + From 336508bb17bc9ce51b001c5b415da116fb137f73 Mon Sep 17 00:00:00 2001 From: vijeth-aradhya Date: Sat, 17 Jun 2017 02:34:58 +0530 Subject: [PATCH 17/55] Render webm_480 as default if webm_video is absent Currently, webm_video is still the higher priority in media_fetch_oder as older versions should not be broken. So, webm_480p (main_task) will be rendered by default for newer versions. All tests pass at this point. --- mediagoblin/media_types/video/__init__.py | 2 +- mediagoblin/media_types/video/processing.py | 8 ++++++++ mediagoblin/user_pages/views.py | 5 ++--- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/mediagoblin/media_types/video/__init__.py b/mediagoblin/media_types/video/__init__.py index 1862ffe1..08e92080 100644 --- a/mediagoblin/media_types/video/__init__.py +++ b/mediagoblin/media_types/video/__init__.py @@ -31,7 +31,7 @@ class VideoMediaManager(MediaManagerBase): type_icon = "images/type_icons/video.png" # Used by the media_entry.get_display_media method - media_fetch_order = [u'webm_video', u'original'] + media_fetch_order = [u'webm_video', u'webm_480p', u'original'] default_webm_type = 'video/webm; codecs="vp8, vorbis"' diff --git a/mediagoblin/media_types/video/processing.py b/mediagoblin/media_types/video/processing.py index e881636c..4da1ad23 100644 --- a/mediagoblin/media_types/video/processing.py +++ b/mediagoblin/media_types/video/processing.py @@ -166,6 +166,7 @@ def store_metadata(media_entry, metadata): @celery.task() def main_task(entry_id, resolution, medium_size, **process_info): + print "\nEntry processing\n" entry, manager = get_entry_and_processing_manager(entry_id) print "\nEntered main_task\n" with CommonVideoProcessor(manager, entry) as processor: @@ -175,6 +176,10 @@ def main_task(entry_id, resolution, medium_size, **process_info): processor.generate_thumb(thumb_size=process_info['thumb_size']) processor.store_orig_metadata() print "\nExited main_task\n" + # Make state of entry as processed + entry.state = u'processed' + entry.save() + print "\nEntry processed\n" @celery.task() @@ -544,6 +549,9 @@ class VideoProcessingManager(ProcessingManager): def workflow(self, entry, feed_url, reprocess_action, reprocess_info=None): + entry.state = u'processing' + entry.save() + reprocess_info = reprocess_info or {} if 'vp8_quality' not in reprocess_info: reprocess_info['vp8_quality'] = None diff --git a/mediagoblin/user_pages/views.py b/mediagoblin/user_pages/views.py index 484d27cd..5e629575 100644 --- a/mediagoblin/user_pages/views.py +++ b/mediagoblin/user_pages/views.py @@ -66,13 +66,12 @@ def user_home(request, page): {'user': user}) cursor = MediaEntry.query.\ - filter_by(actor = user.id, - state = u'processed').order_by(MediaEntry.created.desc()) + filter_by(actor = user.id).order_by(MediaEntry.created.desc()) pagination = Pagination(page, cursor) media_entries = pagination() - #if no data is available, return NotFound + # if no data is available, return NotFound if media_entries == None: return render_404(request) From 5161533a6fb66e24f6906d6476069277d594e738 Mon Sep 17 00:00:00 2001 From: vijeth-aradhya Date: Tue, 20 Jun 2017 17:28:34 +0530 Subject: [PATCH 18/55] Add choice of video resolutions for instance owner The instance owner can choose from the list of available resolutions. ['144p', '240p', '360p', '480p', '720p', '1080p'] Also, the default resolution is now set to 480p and the instance owner can choose the default resolution from that list as well. --- mediagoblin/media_types/video/config_spec.ini | 8 +++++ mediagoblin/media_types/video/processing.py | 30 ++++++++++++------- 2 files changed, 27 insertions(+), 11 deletions(-) diff --git a/mediagoblin/media_types/video/config_spec.ini b/mediagoblin/media_types/video/config_spec.ini index 98714f56..e899eff5 100644 --- a/mediagoblin/media_types/video/config_spec.ini +++ b/mediagoblin/media_types/video/config_spec.ini @@ -12,6 +12,14 @@ vorbis_quality = float(default=0.3) # Autoplay the video when page is loaded? auto_play = boolean(default=False) +# List of resolutions that the video should be transcoded to +# Choose among ['144p', '240p', '360p', '480p', '720p', '1080p'], +# preferrably in the order of transcoding. +available_resolutions = string_list(default=list('480p', '360p', '720p')) + +# Default resolution of video +default_resolution = string(default='480p') + [[skip_transcode]] mime_types = string_list(default=list("video/webm")) container_formats = string_list(default=list("Matroska")) diff --git a/mediagoblin/media_types/video/processing.py b/mediagoblin/media_types/video/processing.py index 4da1ad23..2a6a716f 100644 --- a/mediagoblin/media_types/video/processing.py +++ b/mediagoblin/media_types/video/processing.py @@ -549,6 +549,10 @@ class VideoProcessingManager(ProcessingManager): def workflow(self, entry, feed_url, reprocess_action, reprocess_info=None): + video_config = mgg.global_config['plugins'][MEDIA_TYPE] + def_res = video_config['default_resolution'] + priority_num = len(video_config['available_resolutions']) + 1 + entry.state = u'processing' entry.save() @@ -562,18 +566,22 @@ class VideoProcessingManager(ProcessingManager): if 'thumb_size' not in reprocess_info: reprocess_info['thumb_size'] = None - transcoding_tasks = group([ - main_task.signature(args=(entry.id, '480p', ACCEPTED_RESOLUTIONS['480p']), - kwargs=reprocess_info, queue='default', - priority=5, immutable=True), - complimentary_task.signature(args=(entry.id, '360p', ACCEPTED_RESOLUTIONS['360p']), - kwargs=reprocess_info, queue='default', - priority=4, immutable=True), - complimentary_task.signature(args=(entry.id, '720p', ACCEPTED_RESOLUTIONS['720p']), - kwargs=reprocess_info, queue='default', - priority=3, immutable=True), - ]) + tasks_list = [main_task.signature(args=(entry.id, def_res, + ACCEPTED_RESOLUTIONS[def_res]), + kwargs=reprocess_info, queue='default', + priority=priority_num, immutable=True)] + for comp_res in video_config['available_resolutions']: + if comp_res != def_res: + priority_num += -1 + tasks_list.append( + complimentary_task.signature(args=(entry.id, comp_res, + ACCEPTED_RESOLUTIONS[comp_res]), + kwargs=reprocess_info, queue='default', + priority=priority_num, immutable=True) + ) + + transcoding_tasks = group(tasks_list) cleanup_task = processing_cleanup.signature(args=(entry.id,), queue='default', immutable=True) From 602cfcb78916ac54bb415d9c2023124b6f3ad5b6 Mon Sep 17 00:00:00 2001 From: vijeth-aradhya Date: Wed, 21 Jun 2017 10:00:56 +0530 Subject: [PATCH 19/55] Use tuple as default return type of workflow() If None, then schedule the old Celery call, else the format is group and a single task in the tuple. Hence, Celery chord would be ideal to use. Closes #4 --- mediagoblin/media_types/video/processing.py | 7 ++----- mediagoblin/submit/lib.py | 4 ++++ 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/mediagoblin/media_types/video/processing.py b/mediagoblin/media_types/video/processing.py index 2a6a716f..ee2aa443 100644 --- a/mediagoblin/media_types/video/processing.py +++ b/mediagoblin/media_types/video/processing.py @@ -22,7 +22,7 @@ import celery import six -from celery import group, chord +from celery import group from mediagoblin import mg_globals as mgg from mediagoblin.processing import ( FilenameBuilder, BaseProcessingFail, @@ -585,7 +585,4 @@ class VideoProcessingManager(ProcessingManager): cleanup_task = processing_cleanup.signature(args=(entry.id,), queue='default', immutable=True) - chord(transcoding_tasks)(cleanup_task) - - # Not sure what to return since we are scheduling the task here itself - return 1 + return (transcoding_tasks, cleanup_task) diff --git a/mediagoblin/submit/lib.py b/mediagoblin/submit/lib.py index b228dbd1..9ec96923 100644 --- a/mediagoblin/submit/lib.py +++ b/mediagoblin/submit/lib.py @@ -20,6 +20,8 @@ from os.path import splitext import six +from celery import chord + from werkzeug.utils import secure_filename from werkzeug.datastructures import FileStorage @@ -271,6 +273,8 @@ def run_process_media(entry, feed_url=None, ProcessMedia().apply_async( [entry.id, feed_url, reprocess_action, reprocess_info], {}, task_id=entry.queued_task_id) + else: + chord(wf[0])(wf[1]) except BaseException as exc: # The purpose of this section is because when running in "lazy" # or always-eager-with-exceptions-propagated celery mode that From 2771f4678a483665cb33fe414b50b445f7df07c8 Mon Sep 17 00:00:00 2001 From: vijeth-aradhya Date: Sat, 24 Jun 2017 22:45:14 +0530 Subject: [PATCH 20/55] Add VideoProcessingManager.workflow() tests Testing workflow method including its return data. --- mediagoblin/tests/test_submission.py | 69 ++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/mediagoblin/tests/test_submission.py b/mediagoblin/tests/test_submission.py index f51b132c..dc310965 100644 --- a/mediagoblin/tests/test_submission.py +++ b/mediagoblin/tests/test_submission.py @@ -47,9 +47,11 @@ import os import pytest import webtest.forms import pkg_resources +import mock import six.moves.urllib.parse as urlparse +from celery import Signature from mediagoblin.tests.tools import ( fixture_add_user, fixture_add_collection, get_app) from mediagoblin import mg_globals @@ -58,6 +60,10 @@ from mediagoblin.db.base import Session from mediagoblin.tools import template from mediagoblin.media_types.image import ImageMediaManager from mediagoblin.media_types.pdf.processing import check_prerequisites as pdf_check_prerequisites +from mediagoblin.media_types.video.processing import ( + VideoProcessingManager, main_task, complimentary_task, group) +from mediagoblin.media_types.video.util import ACCEPTED_RESOLUTIONS +from mediagoblin.submit.lib import new_upload_entry from .resources import GOOD_JPG, GOOD_PNG, EVIL_FILE, EVIL_JPG, EVIL_PNG, \ BIG_BLUE, GOOD_PDF, GPS_JPG, MED_PNG, BIG_PNG @@ -101,6 +107,16 @@ def pdf_plugin_app(request): 'mediagoblin.tests', 'test_mgoblin_app_pdf.ini')) +def get_sample_entry(user): + entry = new_upload_entry(user) + entry.media_type = 'mediagoblin.media_types.video' + entry.title = 'testentry' + entry.description = u"" + entry.license = None + entry.media_metadata = {} + entry.save() + return entry + class BaseTestSubmission: @pytest.fixture(autouse=True) @@ -536,6 +552,59 @@ class TestSubmissionVideo(BaseTestSubmission): with create_av(make_video=True) as path: self.check_normal_upload('Video', path) + @mock.patch('mediagoblin.media_types.video.processing.complimentary_task.signature') + @mock.patch('mediagoblin.media_types.video.processing.main_task.signature') + def test_workflow(self, mock_main_task, mock_comp_task): + + # create a new entry and get video manager + entry = get_sample_entry(self.our_user()) + manager = VideoProcessingManager() + + # prepare things for testing + video_config = mg_globals.global_config['plugins'][entry.media_type] + def_res = video_config['default_resolution'] + priority_num = len(video_config['available_resolutions']) + 1 + main_priority = priority_num + calls = [] + reprocess_info = { + 'vorbis_quality': None, + 'vp8_threads': None, + 'thumb_size': None, + 'vp8_quality': None + } + for comp_res in video_config['available_resolutions']: + if comp_res != def_res: + priority_num += -1 + calls.append( + mock.call(args=(entry.id, comp_res, ACCEPTED_RESOLUTIONS[comp_res]), + kwargs=reprocess_info, queue='default', + priority=priority_num, immutable=True) + ) + + # call workflow method + manager.workflow(entry, feed_url=None, reprocess_action='initial') + + # test section + mock_main_task.assert_called_once_with(args=(entry.id, def_res, + ACCEPTED_RESOLUTIONS[def_res]), + kwargs=reprocess_info, queue='default', + priority=main_priority, immutable=True) + mock_comp_task.assert_has_calls(calls) + assert entry.state == u'processing' + + # delete the entry + entry.delete() + + def test_workflow_return(self): + entry = get_sample_entry(self.our_user()) + manager = VideoProcessingManager() + wf = manager.workflow(entry, feed_url=None, reprocess_action='initial') + assert type(wf) == tuple + assert len(wf) == 2 + assert isinstance(wf[0], group) + assert isinstance(wf[1], Signature) + entry.delete() + class TestSubmissionAudio(BaseTestSubmission): @pytest.fixture(autouse=True) From ee2b53dea1a4898c918c2bebbdcbc4edc614ae0b Mon Sep 17 00:00:00 2001 From: vijeth-aradhya Date: Sun, 25 Jun 2017 18:25:48 +0530 Subject: [PATCH 21/55] Add test_chord for TestSubmissionVideo Testing the arguments passed to celery.chord when a video is submitted. --- mediagoblin/tests/test_submission.py | 82 +++++++++++++++++++++++++--- 1 file changed, 74 insertions(+), 8 deletions(-) diff --git a/mediagoblin/tests/test_submission.py b/mediagoblin/tests/test_submission.py index dc310965..3f6cf739 100644 --- a/mediagoblin/tests/test_submission.py +++ b/mediagoblin/tests/test_submission.py @@ -61,9 +61,9 @@ from mediagoblin.tools import template from mediagoblin.media_types.image import ImageMediaManager from mediagoblin.media_types.pdf.processing import check_prerequisites as pdf_check_prerequisites from mediagoblin.media_types.video.processing import ( - VideoProcessingManager, main_task, complimentary_task, group) + VideoProcessingManager, main_task, complimentary_task, group, processing_cleanup) from mediagoblin.media_types.video.util import ACCEPTED_RESOLUTIONS -from mediagoblin.submit.lib import new_upload_entry +from mediagoblin.submit.lib import new_upload_entry, run_process_media from .resources import GOOD_JPG, GOOD_PNG, EVIL_FILE, EVIL_JPG, EVIL_PNG, \ BIG_BLUE, GOOD_PDF, GPS_JPG, MED_PNG, BIG_PNG @@ -107,9 +107,9 @@ def pdf_plugin_app(request): 'mediagoblin.tests', 'test_mgoblin_app_pdf.ini')) -def get_sample_entry(user): +def get_sample_entry(user, media_type): entry = new_upload_entry(user) - entry.media_type = 'mediagoblin.media_types.video' + entry.media_type = media_type entry.title = 'testentry' entry.description = u"" entry.license = None @@ -552,12 +552,13 @@ class TestSubmissionVideo(BaseTestSubmission): with create_av(make_video=True) as path: self.check_normal_upload('Video', path) + @mock.patch('mediagoblin.media_types.video.processing.processing_cleanup.signature') @mock.patch('mediagoblin.media_types.video.processing.complimentary_task.signature') @mock.patch('mediagoblin.media_types.video.processing.main_task.signature') - def test_workflow(self, mock_main_task, mock_comp_task): + def test_celery_tasks(self, mock_main_task, mock_comp_task, mock_cleanup): # create a new entry and get video manager - entry = get_sample_entry(self.our_user()) + entry = get_sample_entry(self.our_user(), 'mediagoblin.media_types.video') manager = VideoProcessingManager() # prepare things for testing @@ -590,19 +591,83 @@ class TestSubmissionVideo(BaseTestSubmission): kwargs=reprocess_info, queue='default', priority=main_priority, immutable=True) mock_comp_task.assert_has_calls(calls) + mock_cleanup.assert_called_once_with(args=(entry.id,), queue='default', + immutable=True) assert entry.state == u'processing' # delete the entry entry.delete() - def test_workflow_return(self): - entry = get_sample_entry(self.our_user()) + def test_workflow(self): + entry = get_sample_entry(self.our_user(), 'mediagoblin.media_types.video') manager = VideoProcessingManager() wf = manager.workflow(entry, feed_url=None, reprocess_action='initial') assert type(wf) == tuple assert len(wf) == 2 assert isinstance(wf[0], group) assert isinstance(wf[1], Signature) + + # more precise testing + video_config = mg_globals.global_config['plugins'][entry.media_type] + def_res = video_config['default_resolution'] + priority_num = len(video_config['available_resolutions']) + 1 + reprocess_info = { + 'vorbis_quality': None, + 'vp8_threads': None, + 'thumb_size': None, + 'vp8_quality': None + } + tasks_list = [main_task.signature(args=(entry.id, def_res, + ACCEPTED_RESOLUTIONS[def_res]), + kwargs=reprocess_info, queue='default', + priority=priority_num, immutable=True)] + for comp_res in video_config['available_resolutions']: + if comp_res != def_res: + priority_num += -1 + tasks_list.append( + complimentary_task.signature(args=(entry.id, comp_res, + ACCEPTED_RESOLUTIONS[comp_res]), + kwargs=reprocess_info, queue='default', + priority=priority_num, immutable=True) + ) + transcoding_tasks = group(tasks_list) + cleanup_task = processing_cleanup.signature(args=(entry.id,), + queue='default', immutable=True) + assert wf[0] == transcoding_tasks + assert wf[1] == cleanup_task + entry.delete() + + @mock.patch('mediagoblin.submit.lib.ProcessMedia.apply_async') + @mock.patch('mediagoblin.submit.lib.chord') + def test_celery_chord(self, mock_chord, mock_process_media): + entry = get_sample_entry(self.our_user(), 'mediagoblin.media_types.video') + + # prepare things for testing + video_config = mg_globals.global_config['plugins'][entry.media_type] + def_res = video_config['default_resolution'] + priority_num = len(video_config['available_resolutions']) + 1 + reprocess_info = { + 'vorbis_quality': None, + 'vp8_threads': None, + 'thumb_size': None, + 'vp8_quality': None + } + tasks_list = [main_task.signature(args=(entry.id, def_res, + ACCEPTED_RESOLUTIONS[def_res]), + kwargs=reprocess_info, queue='default', + priority=priority_num, immutable=True)] + for comp_res in video_config['available_resolutions']: + if comp_res != def_res: + priority_num += -1 + tasks_list.append( + complimentary_task.signature(args=(entry.id, comp_res, + ACCEPTED_RESOLUTIONS[comp_res]), + kwargs=reprocess_info, queue='default', + priority=priority_num, immutable=True) + ) + transcoding_tasks = group(tasks_list) + run_process_media(entry) + mock_chord.assert_called_once_with(transcoding_tasks) entry.delete() @@ -660,3 +725,4 @@ class TestSubmissionPDF(BaseTestSubmission): **self.upload_data(GOOD_PDF)) self.check_url(response, '/u/{0}/'.format(self.our_user().username)) assert 'mediagoblin/user_pages/user.html' in context + From ac966b232b3eab52d610d8f2df4e3029fc1381ff Mon Sep 17 00:00:00 2001 From: vijeth-aradhya Date: Sun, 25 Jun 2017 18:47:04 +0530 Subject: [PATCH 22/55] Test accepted_files and accepted_resolutions --- mediagoblin/tests/test_submission.py | 11 ++++++++++- mediagoblin/tests/test_video.py | 12 ++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/mediagoblin/tests/test_submission.py b/mediagoblin/tests/test_submission.py index 3f6cf739..fb670e7a 100644 --- a/mediagoblin/tests/test_submission.py +++ b/mediagoblin/tests/test_submission.py @@ -61,7 +61,8 @@ from mediagoblin.tools import template from mediagoblin.media_types.image import ImageMediaManager from mediagoblin.media_types.pdf.processing import check_prerequisites as pdf_check_prerequisites from mediagoblin.media_types.video.processing import ( - VideoProcessingManager, main_task, complimentary_task, group, processing_cleanup) + VideoProcessingManager, main_task, complimentary_task, group, + processing_cleanup, CommonVideoProcessor) from mediagoblin.media_types.video.util import ACCEPTED_RESOLUTIONS from mediagoblin.submit.lib import new_upload_entry, run_process_media @@ -670,6 +671,14 @@ class TestSubmissionVideo(BaseTestSubmission): mock_chord.assert_called_once_with(transcoding_tasks) entry.delete() + def test_accepted_files(self): + entry = get_sample_entry(self.our_user(), 'mediagoblin.media_types.video') + manager = VideoProcessingManager() + processor = CommonVideoProcessor(manager, entry) + acceptable_files = ['original, best_quality', 'webm_144p', 'webm_360p', + 'webm_480p', 'webm_720p', 'webm_1080p', 'webm_video'] + assert processor.acceptable_files == acceptable_files + class TestSubmissionAudio(BaseTestSubmission): @pytest.fixture(autouse=True) diff --git a/mediagoblin/tests/test_video.py b/mediagoblin/tests/test_video.py index 79244515..748c857f 100644 --- a/mediagoblin/tests/test_video.py +++ b/mediagoblin/tests/test_video.py @@ -30,6 +30,7 @@ Gst.init(None) from mediagoblin.media_types.video.transcoders import (capture_thumb, VideoTranscoder) +from mediagoblin.media_types.video.util import ACCEPTED_RESOLUTIONS from mediagoblin.media_types.tools import discover @contextmanager @@ -130,3 +131,14 @@ def test_transcoder(): dimensions=(640, 640)) assert len(discover(result_name).get_video_streams()) == 1 assert len(discover(result_name).get_audio_streams()) == 1 + +def test_accepted_resolutions(): + accepted_resolutions = { + '144p' : (256, 144), + '240p' : (352, 240), + '360p' : (480, 360), + '480p' : (858, 480), + '720p' : (1280, 720), + '1080p' : (1920, 1080), + } + assert accepted_resolutions == ACCEPTED_RESOLUTIONS From 7fc550cf2b49d573333050da2fcd6a3586acf747 Mon Sep 17 00:00:00 2001 From: vijeth-aradhya Date: Sun, 25 Jun 2017 19:32:28 +0530 Subject: [PATCH 23/55] Test whether video is transcoded to all resolutions After `self.check_normal_upload`, query the database and check whether the video got transcoded to all the required resolutions. --- mediagoblin/tests/test_submission.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/mediagoblin/tests/test_submission.py b/mediagoblin/tests/test_submission.py index fb670e7a..ff45b107 100644 --- a/mediagoblin/tests/test_submission.py +++ b/mediagoblin/tests/test_submission.py @@ -540,6 +540,7 @@ class TestSubmissionVideo(BaseTestSubmission): @pytest.fixture(autouse=True) def setup(self, video_plugin_app): self.test_app = video_plugin_app + self.media_type = 'mediagoblin.media_types.video' # TODO: Possibly abstract into a decorator like: # @as_authenticated_user('chris') @@ -553,13 +554,20 @@ class TestSubmissionVideo(BaseTestSubmission): with create_av(make_video=True) as path: self.check_normal_upload('Video', path) + media = mg_globals.database.MediaEntry.query.filter_by( + title=u'Video').first() + + video_config = mg_globals.global_config['plugins'][self.media_type] + for each_res in video_config['available_resolutions']: + assert (('webm_' + str(each_res)) in media.media_files) + @mock.patch('mediagoblin.media_types.video.processing.processing_cleanup.signature') @mock.patch('mediagoblin.media_types.video.processing.complimentary_task.signature') @mock.patch('mediagoblin.media_types.video.processing.main_task.signature') def test_celery_tasks(self, mock_main_task, mock_comp_task, mock_cleanup): # create a new entry and get video manager - entry = get_sample_entry(self.our_user(), 'mediagoblin.media_types.video') + entry = get_sample_entry(self.our_user(), self.media_type) manager = VideoProcessingManager() # prepare things for testing @@ -600,7 +608,7 @@ class TestSubmissionVideo(BaseTestSubmission): entry.delete() def test_workflow(self): - entry = get_sample_entry(self.our_user(), 'mediagoblin.media_types.video') + entry = get_sample_entry(self.our_user(), self.media_type) manager = VideoProcessingManager() wf = manager.workflow(entry, feed_url=None, reprocess_action='initial') assert type(wf) == tuple @@ -641,7 +649,7 @@ class TestSubmissionVideo(BaseTestSubmission): @mock.patch('mediagoblin.submit.lib.ProcessMedia.apply_async') @mock.patch('mediagoblin.submit.lib.chord') def test_celery_chord(self, mock_chord, mock_process_media): - entry = get_sample_entry(self.our_user(), 'mediagoblin.media_types.video') + entry = get_sample_entry(self.our_user(), self.media_type) # prepare things for testing video_config = mg_globals.global_config['plugins'][entry.media_type] From bd0b36be7916f756a701f29bb6b0aff8cb624f9c Mon Sep 17 00:00:00 2001 From: vijeth-aradhya Date: Sun, 25 Jun 2017 20:23:37 +0530 Subject: [PATCH 24/55] Implement copy_original and _keep_best _keep_best is changed to keep_best now, and is added in processing_cleanup. webm is added to ACCEPTED_RESOLUTIONS. --- mediagoblin/media_types/video/processing.py | 33 +++++++++++++++++++-- mediagoblin/media_types/video/util.py | 13 ++++---- 2 files changed, 37 insertions(+), 9 deletions(-) diff --git a/mediagoblin/media_types/video/processing.py b/mediagoblin/media_types/video/processing.py index ee2aa443..f2fba38e 100644 --- a/mediagoblin/media_types/video/processing.py +++ b/mediagoblin/media_types/video/processing.py @@ -198,6 +198,10 @@ def processing_cleanup(entry_id): print "\nEntered processing_cleanup()\n" entry, manager = get_entry_and_processing_manager(entry_id) with CommonVideoProcessor(manager, entry) as processor: + # no need to specify a resolution here + processor.common_setup() + processor.copy_original() + processor.keep_best() processor.delete_queue_file() print "\nDeleted queue_file\n" @@ -235,13 +239,36 @@ class CommonVideoProcessor(MediaProcessor): def copy_original(self): # If we didn't transcode, then we need to keep the original - raise NotImplementedError + self.did_transcode = False + for each_res in self.video_config['available_resolutions']: + if ('webm_' + str(each_res)) in self.entry.media_files: + print "here == 1.1" + self.did_transcode = True + break + if not self.did_transcode or \ + (self.video_config['keep_original'] and self.did_transcode): + copy_original( + self.entry, self.process_filename, + self.name_builder.fill('{basename}{ext}')) - def _keep_best(self): + def keep_best(self): """ If there is no original, keep the best file that we have """ - raise NotImplementedError + best_file = None + best_file_dim = (0, 0) + for each_res in self.video_config['available_resolutions']: + curr_dim = ACCEPTED_RESOLUTIONS[each_res] + if curr_dim[0] >= best_file_dim[0] and curr_dim[1] >= best_file_dim[1]: + best_file = each_res + best_file_dim = curr_dim + if not self.entry.media_files.get('best_quality'): + # Save the best quality file if no original? + if not self.entry.media_files.get('original') and \ + self.entry.media_files.get(str(best_file)): + self.entry.media_files['best_quality'] = self.entry \ + .media_files[str(best_file)] + def _skip_processing(self, keyname, **kwargs): file_metadata = self.entry.get_file_metadata(keyname) diff --git a/mediagoblin/media_types/video/util.py b/mediagoblin/media_types/video/util.py index 1f5e907d..cf8dc72d 100644 --- a/mediagoblin/media_types/video/util.py +++ b/mediagoblin/media_types/video/util.py @@ -19,12 +19,13 @@ import logging from mediagoblin import mg_globals as mgg ACCEPTED_RESOLUTIONS = { - '144p' : (256, 144), - '240p' : (352, 240), - '360p' : (480, 360), - '480p' : (858, 480), - '720p' : (1280, 720), - '1080p' : (1920, 1080), + '144p': (256, 144), + '240p': (352, 240), + '360p': (480, 360), + '480p': (858, 480), + '720p': (1280, 720), + '1080p': (1920, 1080), + 'webm': (640, 640), } _log = logging.getLogger(__name__) From fe40b02e7dd9741bcae2ace83c10d9f09af42d2c Mon Sep 17 00:00:00 2001 From: vijeth-aradhya Date: Tue, 27 Jun 2017 01:04:55 +0530 Subject: [PATCH 25/55] Fix test_accepted_resolutions Add {'webm': (640, 640)} --- mediagoblin/tests/test_video.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/mediagoblin/tests/test_video.py b/mediagoblin/tests/test_video.py index 748c857f..0f477bfe 100644 --- a/mediagoblin/tests/test_video.py +++ b/mediagoblin/tests/test_video.py @@ -134,11 +134,12 @@ def test_transcoder(): def test_accepted_resolutions(): accepted_resolutions = { - '144p' : (256, 144), - '240p' : (352, 240), - '360p' : (480, 360), - '480p' : (858, 480), - '720p' : (1280, 720), - '1080p' : (1920, 1080), + '144p': (256, 144), + '240p': (352, 240), + '360p': (480, 360), + '480p': (858, 480), + '720p': (1280, 720), + '1080p': (1920, 1080), + 'webm': (640, 640), } assert accepted_resolutions == ACCEPTED_RESOLUTIONS From ae7ebecdbba7c7ae66e547920502c77a65c4dc0f Mon Sep 17 00:00:00 2001 From: vijeth-aradhya Date: Sat, 1 Jul 2017 04:06:02 +0530 Subject: [PATCH 26/55] Add property to media_fetch_order Due to configurable resolution settings, this change is necessary to set the default resolution and the others. --- mediagoblin/media_types/video/__init__.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/mediagoblin/media_types/video/__init__.py b/mediagoblin/media_types/video/__init__.py index 08e92080..ea7bc021 100644 --- a/mediagoblin/media_types/video/__init__.py +++ b/mediagoblin/media_types/video/__init__.py @@ -14,6 +14,7 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +from mediagoblin import mg_globals as mgg from mediagoblin.media_types import MediaManagerBase from mediagoblin.media_types.video.processing import (VideoProcessingManager, sniff_handler, sniffer) @@ -31,9 +32,17 @@ class VideoMediaManager(MediaManagerBase): type_icon = "images/type_icons/video.png" # Used by the media_entry.get_display_media method - media_fetch_order = [u'webm_video', u'webm_480p', u'original'] default_webm_type = 'video/webm; codecs="vp8, vorbis"' + @property + def media_fetch_order(self): + video_config = mgg.global_config['plugins'][MEDIA_TYPE] + video_res = video_config['available_resolutions'] + video_res.remove(video_config['default_resolution']) + video_res.insert(0, video_config['default_resolution']) + video_res = map((lambda x: unicode('webm_' + str(x), 'utf-8')), video_res) + return ([u'webm_video'] + video_res + [u'original']) + def get_media_type_and_manager(ext): if ext in ACCEPTED_EXTENSIONS: From 809ebf4536ed652325013d1a78e2239849e85cd5 Mon Sep 17 00:00:00 2001 From: vijeth-aradhya Date: Sun, 16 Jul 2017 20:43:08 +0530 Subject: [PATCH 27/55] Add get_all_media() in MediaEntryMixin Returns all available qualties of a media with it's dimensions and resolution label. --- mediagoblin/db/mixin.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/mediagoblin/db/mixin.py b/mediagoblin/db/mixin.py index e8b121d0..b47dc0e7 100644 --- a/mediagoblin/db/mixin.py +++ b/mediagoblin/db/mixin.py @@ -249,6 +249,33 @@ class MediaEntryMixin(GenerateSlugMixin, GeneratePublicIDMixin): if media_size in media_sizes: return media_size, self.media_files[media_size] + def get_all_media(self): + """ + Returns all available qualties of a media + """ + fetch_order = self.media_manager.media_fetch_order + + # No fetching order found? well, give up! + if not fetch_order: + return None + + media_sizes = self.media_files.keys() + + all_media_path = [] + + for media_size in fetch_order: + if media_size in media_sizes: + file_metadata = self.get_file_metadata(media_size) + size = file_metadata['medium_size'] + if media_size != 'webm': + all_media_path.append((media_size[5:], size, + self.media_files[media_size])) + else: + sall_media_path.append(('default', size, + self.media_files[media_size])) + + return all_media_path + def main_mediafile(self): pass From f6ba0e561bfb62991d70a4c002edf142d9d7540c Mon Sep 17 00:00:00 2001 From: vijeth-aradhya Date: Sun, 16 Jul 2017 21:14:25 +0530 Subject: [PATCH 28/55] Modify video template to accomodate multiple resolutions Add required JS/CSS plugin files. Add JS file to fire up the plugin. Modify video tag accordingly. --- .../mediagoblin/media_displays/video.html | 41 +++++++------------ 1 file changed, 14 insertions(+), 27 deletions(-) diff --git a/mediagoblin/templates/mediagoblin/media_displays/video.html b/mediagoblin/templates/mediagoblin/media_displays/video.html index 5aac3529..4ba3d1d5 100644 --- a/mediagoblin/templates/mediagoblin/media_displays/video.html +++ b/mediagoblin/templates/mediagoblin/media_displays/video.html @@ -22,44 +22,31 @@ {{ super() }} + + {# Sadly commented out till we can get the mediagoblin skin ported over # to the newest video.js release ;\ #} - {# - - #} + - {%- endblock %} {% block mediagoblin_media %}
- {% set display_type, display_path = media.get_display_media() %} + {% set all_media_path = media.get_all_media() %} -