refactor get_orig_filename to return an acceptable filename to the processor.

If there is an original video file and we skip transcoding, delete the webm_640 file

mediagoblin/media_types/ascii/processing.py

@@ -26,7 +26,7 @@ from mediagoblin import mg_globals as mgg
 from mediagoblin.processing import (
     create_pub_filepath, FilenameBuilder,
     MediaProcessor, ProcessingManager,
-    get_orig_filename, copy_original,
+    get_process_filename, copy_original,
     store_public, request_from_args)
 from mediagoblin.media_types.ascii import asciitoimage
 
@@ -52,6 +52,8 @@ class CommonAsciiProcessor(MediaProcessor):
     """
     Provides a base for various ascii processing steps
     """
+    acceptable_files = ['original', 'unicode']
+
     def common_setup(self):
         self.ascii_config = mgg.global_config[
             'media_type:mediagoblin.media_types.ascii']
@@ -61,16 +63,16 @@ class CommonAsciiProcessor(MediaProcessor):
             self.workbench.dir, 'convirsions')
         os.mkdir(self.conversions_subdir)
 
-        # Pull down and set up the original file
+        # Pull down and set up the processing file
-        self.orig_filename = get_orig_filename(
+        self.process_filename = get_process_filename(
-            self.entry, self.workbench)
+            self.entry, self.workbench, self.acceptable_files)
-        self.name_builder = FilenameBuilder(self.orig_filename)
+        self.name_builder = FilenameBuilder(self.process_filename)
 
         self.charset = None
 
     def copy_original(self):
         copy_original(
-            self.entry, self.orig_filename,
+            self.entry, self.process_filename,
             self.name_builder.fill('{basename}{ext}'))
 
     def _detect_charset(self, orig_file):
@@ -91,7 +93,7 @@ class CommonAsciiProcessor(MediaProcessor):
         orig_file.seek(0)
 
     def store_unicode_file(self):
-        with file(self.orig_filename, 'rb') as orig_file:
+        with file(self.process_filename, 'rb') as orig_file:
             self._detect_charset(orig_file)
             unicode_filepath = create_pub_filepath(self.entry,
                                                    'ascii-portable.txt')
@@ -110,7 +112,7 @@ class CommonAsciiProcessor(MediaProcessor):
         self.entry.media_files['unicode'] = unicode_filepath
 
     def generate_thumb(self, font=None, thumb_size=None):
-        with file(self.orig_filename, 'rb') as orig_file:
+        with file(self.process_filename, 'rb') as orig_file:
             # If no font kwarg, check config
             if not font:
                 font = self.ascii_config.get('thumbnail_font', None)

mediagoblin/media_types/audio/processing.py

@@ -22,7 +22,7 @@ from mediagoblin import mg_globals as mgg
 from mediagoblin.processing import (
     BadMediaFail, FilenameBuilder,
     ProgressCallback, MediaProcessor, ProcessingManager,
-    request_from_args, get_orig_filename,
+    request_from_args, get_process_filename,
     store_public, copy_original)
 
 from mediagoblin.media_types.audio.transcoders import (
@@ -52,6 +52,7 @@ class CommonAudioProcessor(MediaProcessor):
     """
     Provides a base for various audio processing steps
     """
+    acceptable_files = ['original', 'webm_audio']
 
     def common_setup(self):
         """
@@ -61,10 +62,10 @@ class CommonAudioProcessor(MediaProcessor):
         self.audio_config = mgg \
             .global_config['media_type:mediagoblin.media_types.audio']
 
-        # Pull down and set up the original file
+        # Pull down and set up the processing file
-        self.orig_filename = get_orig_filename(
+        self.process_filename = get_process_filename(
-            self.entry, self.workbench)
+            self.entry, self.workbench, self.acceptable_files)
-        self.name_builder = FilenameBuilder(self.orig_filename)
+        self.name_builder = FilenameBuilder(self.process_filename)
 
         self.transcoder = AudioTranscoder()
         self.thumbnailer = AudioThumbnailer()
@@ -72,7 +73,7 @@ class CommonAudioProcessor(MediaProcessor):
     def copy_original(self):
         if self.audio_config['keep_original']:
             copy_original(
-                self.entry, self.orig_filename,
+                self.entry, self.process_filename,
                 self.name_builder.fill('{basename}{ext}'))
 
     def transcode(self, quality=None):
@@ -85,7 +86,7 @@ class CommonAudioProcessor(MediaProcessor):
                                           '{basename}{ext}'))
 
         self.transcoder.transcode(
-            self.orig_filename,
+            self.process_filename,
             webm_audio_tmp,
             quality=quality,
             progress_callback=progress_callback)
@@ -107,7 +108,7 @@ class CommonAudioProcessor(MediaProcessor):
 
         _log.info('Creating OGG source for spectrogram')
         self.transcoder.transcode(
-            self.orig_filename,
+            self.process_filename,
             wav_tmp,
             mux_string='vorbisenc quality={0} ! oggmux'.format(
                 self.audio_config['quality']))

mediagoblin/media_types/image/processing.py

@@ -26,7 +26,7 @@ from mediagoblin import mg_globals as mgg
 from mediagoblin.processing import (
     BadMediaFail, FilenameBuilder,
     MediaProcessor, ProcessingManager,
-    request_from_args, get_orig_filename,
+    request_from_args, get_process_filename,
     store_public, copy_original)
 from mediagoblin.tools.exif import exif_fix_image_orientation, \
     extract_exif, clean_exif, get_gps_data, get_useful, \
@@ -131,6 +131,9 @@ class CommonImageProcessor(MediaProcessor):
     """
     Provides a base for various media processing steps
     """
+    # list of acceptable file keys in order of prefrence for reprocessing
+    acceptable_files = ['original', 'medium']
+
     def common_setup(self):
         """
         Set up the workbench directory and pull down the original file
@@ -144,13 +147,13 @@ class CommonImageProcessor(MediaProcessor):
             self.workbench.dir, 'convirsions')
         os.mkdir(self.conversions_subdir)
 
-        # Pull down and set up the original file
+        # Pull down and set up the processing file
-        self.orig_filename = get_orig_filename(
+        self.process_filename = get_process_filename(
-            self.entry, self.workbench)
+            self.entry, self.workbench, self.acceptable_files)
-        self.name_builder = FilenameBuilder(self.orig_filename)
+        self.name_builder = FilenameBuilder(self.process_filename)
 
         # Exif extraction
-        self.exif_tags = extract_exif(self.orig_filename)
+        self.exif_tags = extract_exif(self.process_filename)
 
     def generate_medium_if_applicable(self, size=None, quality=None,
                                       filter=None):
@@ -159,7 +162,7 @@ class CommonImageProcessor(MediaProcessor):
         if not filter:
             filter = self.image_config['resize_filter']
 
-        resize_tool(self.entry, False, 'medium', self.orig_filename,
+        resize_tool(self.entry, False, 'medium', self.process_filename,
                     self.name_builder.fill('{basename}.medium{ext}'),
                     self.conversions_subdir, self.exif_tags, quality,
                     filter, size)
@@ -170,14 +173,14 @@ class CommonImageProcessor(MediaProcessor):
         if not filter:
             filter = self.image_config['resize_filter']
 
-        resize_tool(self.entry, True, 'thumb', self.orig_filename,
+        resize_tool(self.entry, True, 'thumb', self.process_filename,
                     self.name_builder.fill('{basename}.thumbnail{ext}'),
                     self.conversions_subdir, self.exif_tags, quality,
                     filter, size)
 
     def copy_original(self):
         copy_original(
-            self.entry, self.orig_filename,
+            self.entry, self.process_filename,
             self.name_builder.fill('{basename}{ext}'))
 
     def extract_metadata(self):

mediagoblin/media_types/pdf/processing.py

@@ -23,7 +23,7 @@ from mediagoblin import mg_globals as mgg
 from mediagoblin.processing import (
     FilenameBuilder, BadMediaFail,
     MediaProcessor, ProcessingManager,
-    request_from_args, get_orig_filename,
+    request_from_args, get_process_filename,
     store_public, copy_original)
 from mediagoblin.tools.translate import fake_ugettext_passthrough as _
 
@@ -239,20 +239,22 @@ class CommonPdfProcessor(MediaProcessor):
     """
     Provides a base for various pdf processing steps
     """
+    acceptable_files = ['original', 'pdf']
+
     def common_setup(self):
         """
         Set up common pdf processing steps
         """
-        # Pull down and set up the original file
+        # Pull down and set up the processing file
-        self.orig_filename = get_orig_filename(
+        self.process_filename = get_process_filename(
-            self.entry, self.workbench)
+            self.entry, self.workbench, self.acceptable_files)
-        self.name_builder = FilenameBuilder(self.orig_filename)
+        self.name_builder = FilenameBuilder(self.process_filename)
 
         self._set_pdf_filename()
 
     def _set_pdf_filename(self):
         if self.name_builder.ext == 'pdf':
-            self.pdf_filename = self.orig_filename
+            self.pdf_filename = self.process_filename
         elif self.entry.media_files.get('pdf'):
             self.pdf_filename = self.workbench.localized_file(
                 mgg.public_store, self.entry.media_files['pdf'])
@@ -261,7 +263,7 @@ class CommonPdfProcessor(MediaProcessor):
 
     def copy_original(self):
         copy_original(
-            self.entry, self.orig_filename,
+            self.entry, self.process_filename,
             self.name_builder.fill('{basename}{ext}'))
 
     def generate_thumb(self, thumb_size=None):
@@ -290,11 +292,11 @@ class CommonPdfProcessor(MediaProcessor):
         """
         Store the pdf. If the file is not a pdf, make it a pdf
         """
-        tmp_pdf = self.orig_filename
+        tmp_pdf = self.process_filename
 
         unoconv = where('unoconv')
         Popen(executable=unoconv,
-              args=[unoconv, '-v', '-f', 'pdf', self.orig_filename]).wait()
+              args=[unoconv, '-v', '-f', 'pdf', self.process_filename]).wait()
 
         if not os.path.exists(tmp_pdf):
             _log.debug('unoconv failed to convert file to pdf')

mediagoblin/media_types/stl/processing.py

@@ -25,7 +25,7 @@ from mediagoblin import mg_globals as mgg
 from mediagoblin.processing import (
     FilenameBuilder, MediaProcessor,
     ProcessingManager, request_from_args,
-    get_orig_filename, store_public,
+    get_process_filename, store_public,
     copy_original)
 
 from mediagoblin.media_types.stl import model_loader
@@ -83,12 +83,13 @@ class CommonStlProcessor(MediaProcessor):
     """
     Provides a common base for various stl processing steps
     """
+    acceptable_files = ['original']
 
     def common_setup(self):
-        # Pull down and set up the original file
+        # Pull down and set up the processing file
-        self.orig_filename = get_orig_filename(
+        self.process_filename = get_process_filename(
-            self.entry, self.workbench)
+            self.entry, self.workbench, self.acceptable_files)
-        self.name_builder = FilenameBuilder(self.orig_filename)
+        self.name_builder = FilenameBuilder(self.process_filename)
 
         self._set_ext()
         self._set_model()
@@ -107,7 +108,7 @@ class CommonStlProcessor(MediaProcessor):
         Attempt to parse the model file and divine some useful
         information about it.
         """
-        with open(self.orig_filename, 'rb') as model_file:
+        with open(self.process_filename, 'rb') as model_file:
             self.model = model_loader.auto_detect(model_file, self.ext)
 
     def _set_greatest(self):
@@ -117,14 +118,14 @@ class CommonStlProcessor(MediaProcessor):
 
     def copy_original(self):
         copy_original(
-            self.entry, self.orig_filename,
+            self.entry, self.process_filename,
             self.name_builder.fill('{basename}{ext}'))
 
     def _snap(self, keyname, name, camera, size, project="ORTHO"):
         filename = self.name_builder.fill(name)
         workbench_path = self.workbench.joinpath(filename)
         shot = {
-            "model_path": self.orig_filename,
+            "model_path": self.process_filename,
             "model_ext": self.ext,
             "camera_coord": camera,
             "camera_focus": self.model.average,

mediagoblin/media_types/video/processing.py

@@ -24,7 +24,7 @@ from mediagoblin.processing import (
     FilenameBuilder, BaseProcessingFail,
     ProgressCallback, MediaProcessor,
     ProcessingManager, request_from_args,
-    get_orig_filename, store_public,
+    get_process_filename, store_public,
     copy_original)
 from mediagoblin.tools.translate import lazy_pass_to_ugettext as _
 
@@ -122,15 +122,16 @@ class CommonVideoProcessor(MediaProcessor):
     """
     Provides a base for various video processing steps
     """
+    acceptable_files = ['original', 'webm_640']
 
     def common_setup(self):
         self.video_config = mgg \
             .global_config['media_type:mediagoblin.media_types.video']
 
-        # Pull down and set up the original file
+        # Pull down and set up the processing file
-        self.orig_filename = get_orig_filename(
+        self.process_filename = get_process_filename(
-            self.entry, self.workbench)
+            self.entry, self.workbench, self.acceptable_files)
-        self.name_builder = FilenameBuilder(self.orig_filename)
+        self.name_builder = FilenameBuilder(self.process_filename)
 
         self.transcoder = transcoders.VideoTranscoder()
         self.did_transcode = False
@@ -140,7 +141,7 @@ class CommonVideoProcessor(MediaProcessor):
         if not self.did_transcode or \
            (self.video_config['keep_original'] and self.did_transcode):
             copy_original(
-                self.entry, self.orig_filename,
+                self.entry, self.process_filename,
                 self.name_builder.fill('{basename}{ext}'))
 
     def transcode(self, medium_size=None, vp8_quality=None, vp8_threads=None,
@@ -161,7 +162,7 @@ class CommonVideoProcessor(MediaProcessor):
             vorbis_quality = self.video_config['vorbis_quality']
 
         # Extract metadata and keep a record of it
-        metadata = self.transcoder.discover(self.orig_filename)
+        metadata = self.transcoder.discover(self.process_filename)
         store_metadata(self.entry, metadata)
 
         # Figure out whether or not we need to transcode this video or
@@ -171,8 +172,14 @@ class CommonVideoProcessor(MediaProcessor):
 
             dst_dimensions = metadata['videowidth'], metadata['videoheight']
 
+            # If there is an original and transcoded, delete the transcoded
+            # since it must be of lower quality then the original
+            if self.entry.media_files.get('original') and \
+               self.entry.media_files.get('webm_640'):
+                self.entry.media_files['webm_640'].delete()
+
         else:
-            self.transcoder.transcode(self.orig_filename, tmp_dst,
+            self.transcoder.transcode(self.process_filename, tmp_dst,
                                       vp8_quality=vp8_quality,
                                       vp8_threads=vp8_threads,
                                       vorbis_quality=vorbis_quality,
@@ -206,7 +213,7 @@ class CommonVideoProcessor(MediaProcessor):
                           mgg.global_config['media:thumb']['max_height'])
 
         transcoders.VideoThumbnailerMarkII(
-            self.orig_filename,
+            self.process_filename,
             tmp_thumb,
             thumb_size[0],
             thumb_size[1])

mediagoblin/processing/__init__.py

@@ -327,29 +327,34 @@ def mark_entry_failed(entry_id, exc):
              u'fail_metadata': {}})
 
 
-def get_orig_filename(entry, workbench):
+def get_process_filename(entry, workbench, acceptable_files):
     """
-    Get the a filename for the original, on local storage
+    Try and get the queued file if available, otherwise return the first file
+    in the acceptable_files that we have.
 
-    If the media entry has a queued_media_file, use that, otherwise
+    If no acceptable_files, raise ProcessFileNotFound
-    use the original.
-
-    In the future, this will return the highest quality file available
-    if neither the original or queued file are available by checking
-    some ordered list of preferred keys.
     """
     if entry.queued_media_file:
-        orig_filepath = entry.queued_media_file
+        filepath = entry.queued_media_file
         storage = mgg.queue_store
     else:
-        orig_filepath = entry.media_files['original']
+        for keyname in acceptable_files:
-        storage = mgg.public_store
+            if entry.media_files.get(keyname):
+                filepath = entry.media_files[keyname]
+                storage = mgg.public_store
+                break
 
-    orig_filename = workbench.localized_file(
+    if not filepath:
-        storage, orig_filepath,
+        raise ProcessFileNotFound()
+
+    filename = workbench.localized_file(
+        storage, filepath,
         'source')
 
-    return orig_filename
+    if not os.path.exists(filename):
+        raise ProcessFileNotFound()
+
+    return filename
 
 
 def store_public(entry, keyname, local_file, target_name=None,
@@ -415,3 +420,11 @@ class PublicStoreFail(BaseProcessingFail):
     Error that should be raised when copying to public store fails
     """
     general_message = _('Copying to public storage failed.')
+
+
+class ProcessFileNotFound(BaseProcessingFail):
+    """
+    Error that should be raised when an acceptable file for processing
+    is not found.
+    """
+    general_message = _(u'An acceptable processing file was not found')