From 5a34a80d0a9d8a930d2a8c49f48d3fe08b60a237 Mon Sep 17 00:00:00 2001 From: Joar Wandborg Date: Tue, 14 Feb 2012 23:26:07 +0100 Subject: [PATCH 01/12] Audio media handler, media sniffing, video fixes * Added audio processing code * Added audio display template * Added audio configuration setting * Changed video docstring --- mediagoblin/config_spec.ini | 7 +- mediagoblin/media_types/__init__.py | 8 + mediagoblin/media_types/audio/__init__.py | 25 +++ mediagoblin/media_types/audio/processing.py | 75 +++++++++ mediagoblin/media_types/audio/transcoder.py | 18 +++ mediagoblin/media_types/audio/transcoders.py | 150 ++++++++++++++++++ mediagoblin/media_types/video/processing.py | 13 +- mediagoblin/media_types/video/transcoders.py | 7 +- .../mediagoblin/media_displays/audio.html | 47 ++++++ 9 files changed, 334 insertions(+), 16 deletions(-) create mode 100644 mediagoblin/media_types/audio/__init__.py create mode 100644 mediagoblin/media_types/audio/processing.py create mode 100644 mediagoblin/media_types/audio/transcoder.py create mode 100644 mediagoblin/media_types/audio/transcoders.py create mode 100644 mediagoblin/templates/mediagoblin/media_displays/audio.html diff --git a/mediagoblin/config_spec.ini b/mediagoblin/config_spec.ini index 2d410899..452d9745 100644 --- a/mediagoblin/config_spec.ini +++ b/mediagoblin/config_spec.ini @@ -65,11 +65,14 @@ base_url = string(default="/mgoblin_media/") storage_class = string(default="mediagoblin.storage.filestorage:BasicFileStorage") base_dir = string(default="%(here)s/user_dev/media/queue") - -# Should we keep the original file? [media_type:mediagoblin.media_types.video] +# Should we keep the original file? keep_original = boolean(default=False) +[media_type:mediagoblin.media_types.audio] +# vorbisenc qualiy +quality = float(default=0.3) + [beaker.cache] type = string(default="file") diff --git a/mediagoblin/media_types/__init__.py b/mediagoblin/media_types/__init__.py index 5128826b..c53ef1ab 100644 --- a/mediagoblin/media_types/__init__.py +++ b/mediagoblin/media_types/__init__.py @@ -28,6 +28,14 @@ class InvalidFileType(Exception): pass +def sniff_media(media): + ''' + Iterate through the enabled media types and find those suited + for a certain file. + ''' + pass + + def get_media_types(): """ Generator, yields the available media types diff --git a/mediagoblin/media_types/audio/__init__.py b/mediagoblin/media_types/audio/__init__.py new file mode 100644 index 00000000..9b33f9e3 --- /dev/null +++ b/mediagoblin/media_types/audio/__init__.py @@ -0,0 +1,25 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from mediagoblin.media_types.audio.processing import process_audio, \ + sniff_handler + +MEDIA_MANAGER = { + 'human_readable': 'Audio', + 'processor': process_audio, + 'sniff_handler': sniff_handler, + 'display_template': 'mediagoblin/media_displays/audio.html', + 'accepted_extensions': ['mp3', 'flac', 'ogg', 'wav', 'm4a']} diff --git a/mediagoblin/media_types/audio/processing.py b/mediagoblin/media_types/audio/processing.py new file mode 100644 index 00000000..beb12391 --- /dev/null +++ b/mediagoblin/media_types/audio/processing.py @@ -0,0 +1,75 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import logging +import tempfile +import os + +from mediagoblin import mg_globals as mgg +from mediagoblin.processing import create_pub_filepath + +from mediagoblin.media_types.audio.transcoders import AudioTranscoder + +_log = logging.getLogger() + +def sniff_handler(media): + return True + +def process_audio(entry): + audio_config = mgg.global_config['media_type:mediagoblin.media_types.audio'] + + workbench = mgg.workbench_manager.create_workbench() + + queued_filepath = entry.queued_media_file + queued_filename = workbench.localized_file( + mgg.queue_store, queued_filepath, + 'source') + + ogg_filepath = create_pub_filepath( + entry, + '{original}.webm'.format( + original=os.path.splitext( + queued_filepath[-1])[0])) + + ogg_tmp = tempfile.NamedTemporaryFile() + + with ogg_tmp: + transcoder = AudioTranscoder() + + transcoder.transcode( + queued_filename, + ogg_tmp.name, + quality=audio_config['quality']) + + data = transcoder.discover(ogg_tmp.name) + + _log.debug('Saving medium...') + mgg.public_store.get_file(ogg_filepath, 'wb').write( + ogg_tmp.read()) + + entry.media_files['ogg'] = ogg_filepath + + entry.media_data['audio'] = { + u'length': int(data.audiolength)} + + thumbnail_tmp = tempfile.NamedTemporaryFile() + + with thumbnail_tmp: + entry.media_files['thumb'] = ['fake', 'thumb', 'path.jpg'] + + mgg.queue_store.delete_file(queued_filepath) + + entry.save() diff --git a/mediagoblin/media_types/audio/transcoder.py b/mediagoblin/media_types/audio/transcoder.py new file mode 100644 index 00000000..e1000faa --- /dev/null +++ b/mediagoblin/media_types/audio/transcoder.py @@ -0,0 +1,18 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +class AudioTranscoder(object): + diff --git a/mediagoblin/media_types/audio/transcoders.py b/mediagoblin/media_types/audio/transcoders.py new file mode 100644 index 00000000..e59214b0 --- /dev/null +++ b/mediagoblin/media_types/audio/transcoders.py @@ -0,0 +1,150 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import pdb +import logging + +from mediagoblin.processing import BadMediaFail + + +_log = logging.getLogger(__name__) + +CPU_COUNT = 2 # Just assuming for now + +# IMPORT MULTIPROCESSING +try: + import multiprocessing + try: + CPU_COUNT = multiprocessing.cpu_count() + except NotImplementedError: + _log.warning('multiprocessing.cpu_count not implemented!\n' + 'Assuming 2 CPU cores') +except ImportError: + _log.warning('Could not import multiprocessing, assuming 2 CPU cores') + +# IMPORT GOBJECT +try: + import gobject + gobject.threads_init() +except ImportError: + raise Exception('gobject could not be found') + +# IMPORT PYGST +try: + import pygst + + # We won't settle for less. For now, this is an arbitrary limit + # as we have not tested with > 0.10 + pygst.require('0.10') + + import gst + + import gst.extend.discoverer +except ImportError: + raise Exception('gst/pygst >= 0.10 could not be imported') + +class AudioTranscoder(object): + def __init__(self): + _log.info('Initializing {0}'.format(self.__class__.__name__)) + + # Instantiate MainLoop + self._loop = gobject.MainLoop() + + def discover(self, src): + _log.info('Discovering {0}'.format(src)) + self._discovery_path = src + + self._discoverer = gst.extend.discoverer.Discoverer( + self._discovery_path) + self._discoverer.connect('discovered', self.__on_discovered) + self._discoverer.discover() + + self._loop.run() # Run MainLoop + + # Once MainLoop has returned, return discovery data + return self._discovery_data + + def __on_discovered(self, data, is_media): + if not is_media: + self.halt() + _log.error('Could not discover {0}'.format(self._src_path)) + raise BadMediaFail() + + _log.debug('Discovered: {0}'.format(data.__dict__)) + + self._discovery_data = data + + # Gracefully shut down MainLoop + self.halt() + + def transcode(self, src, dst, **kw): + self._discovery_data = kw.get('data', self.discover(src)) + + self.__on_progress = kw.get('progress_callback') + + quality = kw.get('quality', 0.3) + + # Set up pipeline + self.pipeline = gst.parse_launch( + 'filesrc location="{src}" ! ' + 'decodebin2 ! queue ! audiorate tolerance={tolerance} ! ' + 'audioconvert ! audio/x-raw-float,channels=2 ! ' + 'vorbisenc quality={quality} ! webmmux ! ' + 'progressreport silent=true ! ' + 'filesink location="{dst}"'.format( + src=src, + tolerance=80000000, + quality=quality, + dst=dst)) + + self.bus = self.pipeline.get_bus() + self.bus.add_signal_watch() + self.bus.connect('message', self.__on_bus_message) + + self.pipeline.set_state(gst.STATE_PLAYING) + + self._loop.run() + + def __on_bus_message(self, bus, message): + _log.debug(message) + + if (message.type == gst.MESSAGE_ELEMENT + and message.structure.get_name() == 'progress'): + data = dict(message.structure) + + if self.__on_progress: + self.__on_progress(data) + + _log.info('{0}% done...'.format( + data.get('percent'))) + elif message.type == gst.MESSAGE_EOS: + _log.info('Done') + self.halt() + + def halt(self): + _log.info('Quitting MainLoop gracefully...') + gobject.idle_add(self._loop.quit) + +if __name__ == '__main__': + import sys + logging.basicConfig() + _log.setLevel(logging.INFO) + + transcoder = AudioTranscoder() + data = transcoder.discover(sys.argv[1]) + res = transcoder.transcode(*sys.argv[1:3]) + + pdb.set_trace() diff --git a/mediagoblin/media_types/video/processing.py b/mediagoblin/media_types/video/processing.py index 9dc23c55..089d96fb 100644 --- a/mediagoblin/media_types/video/processing.py +++ b/mediagoblin/media_types/video/processing.py @@ -31,15 +31,8 @@ _log.setLevel(logging.DEBUG) def process_video(entry): """ - Code to process a video - - Much of this code is derived from the arista-transcoder script in - the arista PyPI package and changed to match the needs of - MediaGoblin - - This function sets up the arista video encoder in some kind of new thread - and attaches callbacks to that child process, hopefully, the - entry-complete callback will be called when the video is done. + Process a video entry, transcode the queued media files (originals) and + create a thumbnail for the entry. """ video_config = mgg.global_config['media_type:mediagoblin.media_types.video'] @@ -54,7 +47,7 @@ def process_video(entry): entry, '{original}-640p.webm'.format( original=os.path.splitext( - queued_filepath[-1])[0] # Select the + queued_filepath[-1])[0] # Select the file name without .ext )) thumbnail_filepath = create_pub_filepath( diff --git a/mediagoblin/media_types/video/transcoders.py b/mediagoblin/media_types/video/transcoders.py index 6137c3bf..903bd810 100644 --- a/mediagoblin/media_types/video/transcoders.py +++ b/mediagoblin/media_types/video/transcoders.py @@ -38,17 +38,16 @@ try: pass except ImportError: _log.warning('Could not import multiprocessing, defaulting to 2 CPU cores') - pass try: import gtk -except: +except ImportError: raise Exception('Could not find pygtk') try: import gobject gobject.threads_init() -except: +except ImportError: raise Exception('gobject could not be found') try: @@ -56,7 +55,7 @@ try: pygst.require('0.10') import gst from gst.extend import discoverer -except: +except ImportError: raise Exception('gst/pygst 0.10 could not be found') diff --git a/mediagoblin/templates/mediagoblin/media_displays/audio.html b/mediagoblin/templates/mediagoblin/media_displays/audio.html new file mode 100644 index 00000000..802a85c1 --- /dev/null +++ b/mediagoblin/templates/mediagoblin/media_displays/audio.html @@ -0,0 +1,47 @@ +{# +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +#} + +{% extends 'mediagoblin/user_pages/media.html' %} + +{% block mediagoblin_media %} +
+ +
+ {% if 'original' in media.media_files %} +

+ + {%- trans -%} + Original + {%- endtrans -%} + +

+ {% endif %} +{% endblock %} From ec4261a449c11b015190bc90dd9ae828261065cd Mon Sep 17 00:00:00 2001 From: Joar Wandborg Date: Wed, 15 Feb 2012 01:15:29 +0100 Subject: [PATCH 02/12] Changed media processing delegation to a 'sniffing' method - Added sniff handlers to all media plugins All of them except audio returning False for ANYTHING at the moment. --- mediagoblin/media_types/__init__.py | 15 ++++++++++++++- mediagoblin/media_types/ascii/__init__.py | 4 +++- mediagoblin/media_types/ascii/processing.py | 3 +++ mediagoblin/media_types/audio/processing.py | 11 +++++++++-- mediagoblin/media_types/audio/transcoders.py | 12 +++++++++--- mediagoblin/media_types/image/__init__.py | 4 +++- mediagoblin/media_types/image/processing.py | 3 +++ mediagoblin/media_types/video/__init__.py | 4 +++- mediagoblin/media_types/video/processing.py | 2 ++ mediagoblin/submit/views.py | 9 ++++++--- 10 files changed, 55 insertions(+), 12 deletions(-) diff --git a/mediagoblin/media_types/__init__.py b/mediagoblin/media_types/__init__.py index c53ef1ab..19d822a3 100644 --- a/mediagoblin/media_types/__init__.py +++ b/mediagoblin/media_types/__init__.py @@ -16,10 +16,13 @@ import os import sys +import logging +import tempfile from mediagoblin import mg_globals from mediagoblin.tools.translate import lazy_pass_to_ugettext as _ +_log = logging.getLogger(__name__) class FileTypeNotSupported(Exception): pass @@ -33,7 +36,17 @@ def sniff_media(media): Iterate through the enabled media types and find those suited for a certain file. ''' - pass + media_file = tempfile.NamedTemporaryFile() + media_file.write(media.file.read()) + media.file.seek(0) + for media_type, manager in get_media_managers(): + _log.info('Sniffing {0}'.format(media_type)) + if manager['sniff_handler'](media_file, media=media): + return media_type, manager + + raise FileTypeNotSupported( + # TODO: Provide information on which file types are supported + _(u'Sorry, I don\'t support that file type :(')) def get_media_types(): diff --git a/mediagoblin/media_types/ascii/__init__.py b/mediagoblin/media_types/ascii/__init__.py index 1c8ca562..856d1d7b 100644 --- a/mediagoblin/media_types/ascii/__init__.py +++ b/mediagoblin/media_types/ascii/__init__.py @@ -14,13 +14,15 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -from mediagoblin.media_types.ascii.processing import process_ascii +from mediagoblin.media_types.ascii.processing import process_ascii, \ + sniff_handler MEDIA_MANAGER = { "human_readable": "ASCII", "processor": process_ascii, # alternately a string, # 'mediagoblin.media_types.image.processing'? + "sniff_handler": sniff_handler, "display_template": "mediagoblin/media_displays/ascii.html", "default_thumb": "images/media_thumbs/ascii.jpg", "accepted_extensions": [ diff --git a/mediagoblin/media_types/ascii/processing.py b/mediagoblin/media_types/ascii/processing.py index 7ece866e..f698b97a 100644 --- a/mediagoblin/media_types/ascii/processing.py +++ b/mediagoblin/media_types/ascii/processing.py @@ -24,6 +24,9 @@ from mediagoblin.media_types.ascii import asciitoimage _log = logging.getLogger(__name__) +def sniff_handler(media_file, **kw): + return False + def process_ascii(entry): ''' Code to process a txt file diff --git a/mediagoblin/media_types/audio/processing.py b/mediagoblin/media_types/audio/processing.py index beb12391..7aa7ace8 100644 --- a/mediagoblin/media_types/audio/processing.py +++ b/mediagoblin/media_types/audio/processing.py @@ -25,8 +25,15 @@ from mediagoblin.media_types.audio.transcoders import AudioTranscoder _log = logging.getLogger() -def sniff_handler(media): - return True +def sniff_handler(media_file, **kw): + transcoder = AudioTranscoder() + try: + data = transcoder.discover(media_file.name) + + if data.is_audio == True and data.is_video == False: + return True + except: + return False def process_audio(entry): audio_config = mgg.global_config['media_type:mediagoblin.media_types.audio'] diff --git a/mediagoblin/media_types/audio/transcoders.py b/mediagoblin/media_types/audio/transcoders.py index e59214b0..c5634964 100644 --- a/mediagoblin/media_types/audio/transcoders.py +++ b/mediagoblin/media_types/audio/transcoders.py @@ -62,8 +62,10 @@ class AudioTranscoder(object): # Instantiate MainLoop self._loop = gobject.MainLoop() + self._failed = None def discover(self, src): + self._src_path = src _log.info('Discovering {0}'.format(src)) self._discovery_path = src @@ -74,14 +76,17 @@ class AudioTranscoder(object): self._loop.run() # Run MainLoop + if self._failed: + raise self._failed + # Once MainLoop has returned, return discovery data - return self._discovery_data + return getattr(self, '_discovery_data', False) def __on_discovered(self, data, is_media): if not is_media: - self.halt() + self._failed = BadMediaFail() _log.error('Could not discover {0}'.format(self._src_path)) - raise BadMediaFail() + self.halt() _log.debug('Discovered: {0}'.format(data.__dict__)) @@ -91,6 +96,7 @@ class AudioTranscoder(object): self.halt() def transcode(self, src, dst, **kw): + _log.info('Transcoding {0} into {1}'.format(src, dst)) self._discovery_data = kw.get('data', self.discover(src)) self.__on_progress = kw.get('progress_callback') diff --git a/mediagoblin/media_types/image/__init__.py b/mediagoblin/media_types/image/__init__.py index 98e0c32a..d4720fab 100644 --- a/mediagoblin/media_types/image/__init__.py +++ b/mediagoblin/media_types/image/__init__.py @@ -14,13 +14,15 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -from mediagoblin.media_types.image.processing import process_image +from mediagoblin.media_types.image.processing import process_image, \ + sniff_handler MEDIA_MANAGER = { "human_readable": "Image", "processor": process_image, # alternately a string, # 'mediagoblin.media_types.image.processing'? + "sniff_handler": sniff_handler, "display_template": "mediagoblin/media_displays/image.html", "default_thumb": "images/media_thumbs/image.jpg", "accepted_extensions": ["jpg", "jpeg", "png", "gif", "tiff"]} diff --git a/mediagoblin/media_types/image/processing.py b/mediagoblin/media_types/image/processing.py index 769de89b..d301a69f 100644 --- a/mediagoblin/media_types/image/processing.py +++ b/mediagoblin/media_types/image/processing.py @@ -23,6 +23,9 @@ from mediagoblin.processing import BadMediaFail, \ from mediagoblin.tools.exif import exif_fix_image_orientation, \ extract_exif, clean_exif, get_gps_data, get_useful +def sniff_handler(media_file, **kw): + return False + def process_image(entry): """ Code to process an image diff --git a/mediagoblin/media_types/video/__init__.py b/mediagoblin/media_types/video/__init__.py index ed542f16..5351abe6 100644 --- a/mediagoblin/media_types/video/__init__.py +++ b/mediagoblin/media_types/video/__init__.py @@ -14,13 +14,15 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -from mediagoblin.media_types.video.processing import process_video +from mediagoblin.media_types.video.processing import process_video, \ + sniff_handler MEDIA_MANAGER = { "human_readable": "Video", "processor": process_video, # alternately a string, # 'mediagoblin.media_types.image.processing'? + "sniff_handler": sniff_handler, "display_template": "mediagoblin/media_displays/video.html", "default_thumb": "images/media_thumbs/video.jpg", "accepted_extensions": [ diff --git a/mediagoblin/media_types/video/processing.py b/mediagoblin/media_types/video/processing.py index 089d96fb..1890ef0c 100644 --- a/mediagoblin/media_types/video/processing.py +++ b/mediagoblin/media_types/video/processing.py @@ -28,6 +28,8 @@ logging.basicConfig() _log = logging.getLogger(__name__) _log.setLevel(logging.DEBUG) +def sniff_handler(media_file, **kw): + return False def process_video(entry): """ diff --git a/mediagoblin/submit/views.py b/mediagoblin/submit/views.py index cdd097ec..4fafe1e3 100644 --- a/mediagoblin/submit/views.py +++ b/mediagoblin/submit/views.py @@ -35,7 +35,7 @@ from mediagoblin.decorators import require_active_login from mediagoblin.submit import forms as submit_forms, security from mediagoblin.processing import mark_entry_failed, ProcessMedia from mediagoblin.messages import add_message, SUCCESS -from mediagoblin.media_types import get_media_type_and_manager, \ +from mediagoblin.media_types import sniff_media, \ InvalidFileType, FileTypeNotSupported @@ -55,7 +55,11 @@ def submit_start(request): else: try: filename = request.POST['file'].filename - media_type, media_manager = get_media_type_and_manager(filename) + + # Sniff the submitted media to determine which + # media plugin should handle processing + media_type, media_manager = sniff_media( + request.POST['file']) # create entry and save in database entry = request.db.MediaEntry() @@ -164,7 +168,6 @@ def submit_start(request): This section is intended to catch exceptions raised in mediagobling.media_types ''' - if isinstance(e, InvalidFileType) or \ isinstance(e, FileTypeNotSupported): submit_form.file.errors.append( From 92f129b5c707bdeaceb0cdf9807342b52db33b45 Mon Sep 17 00:00:00 2001 From: Joar Wandborg Date: Thu, 16 Feb 2012 18:43:15 +0100 Subject: [PATCH 03/12] Added sniffing logic for image media type For now, it's a re-implementation of the old file-extension checking logic, as I have not found a GStreamer-like "discoverer" in PIL. --- mediagoblin/media_types/image/processing.py | 24 +++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/mediagoblin/media_types/image/processing.py b/mediagoblin/media_types/image/processing.py index d301a69f..364a5afa 100644 --- a/mediagoblin/media_types/image/processing.py +++ b/mediagoblin/media_types/image/processing.py @@ -16,6 +16,7 @@ import Image import os +import logging from mediagoblin import mg_globals as mgg from mediagoblin.processing import BadMediaFail, \ @@ -23,7 +24,30 @@ from mediagoblin.processing import BadMediaFail, \ from mediagoblin.tools.exif import exif_fix_image_orientation, \ extract_exif, clean_exif, get_gps_data, get_useful +_log = logging.getLogger(__name__) + +SUPPORTED_FILETYPES = ['png', 'gif', 'jpg', 'jpeg'] + def sniff_handler(media_file, **kw): + if not kw.get('media') == None: # That's a double negative! + name, ext = os.path.splitext(kw['media'].filename) + clean_ext = ext[1:].lower() # Strip the . from ext and make lowercase + + _log.debug('name: {0}\next: {1}\nlower_ext: {2}'.format( + name, + ext, + clean_ext)) + + if clean_ext in SUPPORTED_FILETYPES: + _log.info('Found file extension in supported filetypes') + return True + else: + _log.debug('Media present, extension not found in {1}'.format( + SUPPORTED_FILETYPES)) + else: + _log.warning('Need additional information (keyword argument \'media\')' + ' to be able to handle sniffing') + return False def process_image(entry): From 9f46a79ddebcc3b2a8bc1dd1ef4f67575bfb53c6 Mon Sep 17 00:00:00 2001 From: Joar Wandborg Date: Thu, 16 Feb 2012 18:59:55 +0100 Subject: [PATCH 04/12] Removed old audio.transcoder file (the real one is audio.transcoders) --- mediagoblin/media_types/audio/transcoder.py | 18 ------------------ 1 file changed, 18 deletions(-) delete mode 100644 mediagoblin/media_types/audio/transcoder.py diff --git a/mediagoblin/media_types/audio/transcoder.py b/mediagoblin/media_types/audio/transcoder.py deleted file mode 100644 index e1000faa..00000000 --- a/mediagoblin/media_types/audio/transcoder.py +++ /dev/null @@ -1,18 +0,0 @@ -# GNU MediaGoblin -- federated, autonomous media hosting -# Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -class AudioTranscoder(object): - From 10085b77391c0ca0b33a6ba82ca46da1babbea8f Mon Sep 17 00:00:00 2001 From: Joar Wandborg Date: Tue, 28 Feb 2012 21:59:38 +0100 Subject: [PATCH 05/12] Audio thumbnailing & spectrograms, media plugins use sniffing * Added extlib/freesound/audioprocessing.py * config_spec * Added create_spectrogram setting * Added media:medium and media:thumb max_{width,height} settings * Added sniffing logic to - audio.processing:sniff_handler - video.processing:sniff_handler * Changed audio.processing:sniff_handler logic * Added audio thumbnailing functionality to audio.processing (works only with create_spectrogram enabled) * Refractored contexts in audio.processing * Added audio.transcoders:AudioThumbnailer Used for creating spectrograms and spectrogram thumbnails - Wadsworth's Constant, we meet again :) * audio.transcoders:AudioTranscoder - Added mux_string kwarg - Delete self.pipeline on self.halt() * Changed str.format formatting in image.processing:sniff_handler Had {1} without an {0}, changed to {0} * Refractored VideoTranscoder to use transcode() for transcoding instead of __init__() * Added discover() method to video.transcoders:VideoTranscoder * Added spectrogram display to media_displays/audio.html * Updated test_submission to reflect changes in media plugin delegation --- extlib/freesound/audioprocessing.py | 616 ++++++++++++++++++ mediagoblin/config_spec.ini | 9 + mediagoblin/media_types/ascii/processing.py | 9 + .../media_types/audio/audioprocessing.py | 1 + mediagoblin/media_types/audio/processing.py | 63 +- mediagoblin/media_types/audio/transcoders.py | 90 ++- mediagoblin/media_types/image/processing.py | 2 +- mediagoblin/media_types/video/processing.py | 15 +- mediagoblin/media_types/video/transcoders.py | 53 +- .../mediagoblin/media_displays/audio.html | 9 +- mediagoblin/tests/test_submission.py | 3 +- 11 files changed, 843 insertions(+), 27 deletions(-) create mode 100644 extlib/freesound/audioprocessing.py create mode 120000 mediagoblin/media_types/audio/audioprocessing.py diff --git a/extlib/freesound/audioprocessing.py b/extlib/freesound/audioprocessing.py new file mode 100644 index 00000000..2c2b35b5 --- /dev/null +++ b/extlib/freesound/audioprocessing.py @@ -0,0 +1,616 @@ +#!/usr/bin/env python +# processing.py -- various audio processing functions +# Copyright (C) 2008 MUSIC TECHNOLOGY GROUP (MTG) +# UNIVERSITAT POMPEU FABRA +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# Authors: +# Bram de Jong +# 2012, Joar Wandborg + +from PIL import Image, ImageDraw, ImageColor #@UnresolvedImport +from functools import partial +import math +import numpy +import os +import re +import signal + + +def get_sound_type(input_filename): + sound_type = os.path.splitext(input_filename.lower())[1].strip(".") + + if sound_type == "fla": + sound_type = "flac" + elif sound_type == "aif": + sound_type = "aiff" + + return sound_type + + +try: + import scikits.audiolab as audiolab +except ImportError: + print "WARNING: audiolab is not installed so wav2png will not work" +import subprocess + +class AudioProcessingException(Exception): + pass + +class TestAudioFile(object): + """A class that mimics audiolab.sndfile but generates noise instead of reading + a wave file. Additionally it can be told to have a "broken" header and thus crashing + in the middle of the file. Also useful for testing ultra-short files of 20 samples.""" + def __init__(self, num_frames, has_broken_header=False): + self.seekpoint = 0 + self.nframes = num_frames + self.samplerate = 44100 + self.channels = 1 + self.has_broken_header = has_broken_header + + def seek(self, seekpoint): + self.seekpoint = seekpoint + + def read_frames(self, frames_to_read): + if self.has_broken_header and self.seekpoint + frames_to_read > self.num_frames / 2: + raise RuntimeError() + + num_frames_left = self.num_frames - self.seekpoint + will_read = num_frames_left if num_frames_left < frames_to_read else frames_to_read + self.seekpoint += will_read + return numpy.random.random(will_read)*2 - 1 + + +def get_max_level(filename): + max_value = 0 + buffer_size = 4096 + audio_file = audiolab.Sndfile(filename, 'r') + n_samples_left = audio_file.nframes + + while n_samples_left: + to_read = min(buffer_size, n_samples_left) + + try: + samples = audio_file.read_frames(to_read) + except RuntimeError: + # this can happen with a broken header + break + + # convert to mono by selecting left channel only + if audio_file.channels > 1: + samples = samples[:,0] + + max_value = max(max_value, numpy.abs(samples).max()) + + n_samples_left -= to_read + + audio_file.close() + + return max_value + +class AudioProcessor(object): + """ + The audio processor processes chunks of audio an calculates the spectrac centroid and the peak + samples in that chunk of audio. + """ + def __init__(self, input_filename, fft_size, window_function=numpy.hanning): + max_level = get_max_level(input_filename) + + self.audio_file = audiolab.Sndfile(input_filename, 'r') + self.fft_size = fft_size + self.window = window_function(self.fft_size) + self.spectrum_range = None + self.lower = 100 + self.higher = 22050 + self.lower_log = math.log10(self.lower) + self.higher_log = math.log10(self.higher) + self.clip = lambda val, low, high: min(high, max(low, val)) + + # figure out what the maximum value is for an FFT doing the FFT of a DC signal + fft = numpy.fft.rfft(numpy.ones(fft_size) * self.window) + max_fft = (numpy.abs(fft)).max() + # set the scale to normalized audio and normalized FFT + self.scale = 1.0/max_level/max_fft if max_level > 0 else 1 + + def read(self, start, size, resize_if_less=False): + """ read size samples starting at start, if resize_if_less is True and less than size + samples are read, resize the array to size and fill with zeros """ + + # number of zeros to add to start and end of the buffer + add_to_start = 0 + add_to_end = 0 + + if start < 0: + # the first FFT window starts centered around zero + if size + start <= 0: + return numpy.zeros(size) if resize_if_less else numpy.array([]) + else: + self.audio_file.seek(0) + + add_to_start = -start # remember: start is negative! + to_read = size + start + + if to_read > self.audio_file.nframes: + add_to_end = to_read - self.audio_file.nframes + to_read = self.audio_file.nframes + else: + self.audio_file.seek(start) + + to_read = size + if start + to_read >= self.audio_file.nframes: + to_read = self.audio_file.nframes - start + add_to_end = size - to_read + + try: + samples = self.audio_file.read_frames(to_read) + except RuntimeError: + # this can happen for wave files with broken headers... + return numpy.zeros(size) if resize_if_less else numpy.zeros(2) + + # convert to mono by selecting left channel only + if self.audio_file.channels > 1: + samples = samples[:,0] + + if resize_if_less and (add_to_start > 0 or add_to_end > 0): + if add_to_start > 0: + samples = numpy.concatenate((numpy.zeros(add_to_start), samples), axis=1) + + if add_to_end > 0: + samples = numpy.resize(samples, size) + samples[size - add_to_end:] = 0 + + return samples + + + def spectral_centroid(self, seek_point, spec_range=110.0): + """ starting at seek_point read fft_size samples, and calculate the spectral centroid """ + + samples = self.read(seek_point - self.fft_size/2, self.fft_size, True) + + samples *= self.window + fft = numpy.fft.rfft(samples) + spectrum = self.scale * numpy.abs(fft) # normalized abs(FFT) between 0 and 1 + length = numpy.float64(spectrum.shape[0]) + + # scale the db spectrum from [- spec_range db ... 0 db] > [0..1] + db_spectrum = ((20*(numpy.log10(spectrum + 1e-60))).clip(-spec_range, 0.0) + spec_range)/spec_range + + energy = spectrum.sum() + spectral_centroid = 0 + + if energy > 1e-60: + # calculate the spectral centroid + + if self.spectrum_range == None: + self.spectrum_range = numpy.arange(length) + + spectral_centroid = (spectrum * self.spectrum_range).sum() / (energy * (length - 1)) * self.audio_file.samplerate * 0.5 + + # clip > log10 > scale between 0 and 1 + spectral_centroid = (math.log10(self.clip(spectral_centroid, self.lower, self.higher)) - self.lower_log) / (self.higher_log - self.lower_log) + + return (spectral_centroid, db_spectrum) + + + def peaks(self, start_seek, end_seek): + """ read all samples between start_seek and end_seek, then find the minimum and maximum peak + in that range. Returns that pair in the order they were found. So if min was found first, + it returns (min, max) else the other way around. """ + + # larger blocksizes are faster but take more mem... + # Aha, Watson, a clue, a tradeof! + block_size = 4096 + + max_index = -1 + max_value = -1 + min_index = -1 + min_value = 1 + + if start_seek < 0: + start_seek = 0 + + if end_seek > self.audio_file.nframes: + end_seek = self.audio_file.nframes + + if end_seek <= start_seek: + samples = self.read(start_seek, 1) + return (samples[0], samples[0]) + + if block_size > end_seek - start_seek: + block_size = end_seek - start_seek + + for i in range(start_seek, end_seek, block_size): + samples = self.read(i, block_size) + + local_max_index = numpy.argmax(samples) + local_max_value = samples[local_max_index] + + if local_max_value > max_value: + max_value = local_max_value + max_index = local_max_index + + local_min_index = numpy.argmin(samples) + local_min_value = samples[local_min_index] + + if local_min_value < min_value: + min_value = local_min_value + min_index = local_min_index + + return (min_value, max_value) if min_index < max_index else (max_value, min_value) + + +def interpolate_colors(colors, flat=False, num_colors=256): + """ given a list of colors, create a larger list of colors interpolating + the first one. If flatten is True a list of numers will be returned. If + False, a list of (r,g,b) tuples. num_colors is the number of colors wanted + in the final list """ + + palette = [] + + for i in range(num_colors): + index = (i * (len(colors) - 1))/(num_colors - 1.0) + index_int = int(index) + alpha = index - float(index_int) + + if alpha > 0: + r = (1.0 - alpha) * colors[index_int][0] + alpha * colors[index_int + 1][0] + g = (1.0 - alpha) * colors[index_int][1] + alpha * colors[index_int + 1][1] + b = (1.0 - alpha) * colors[index_int][2] + alpha * colors[index_int + 1][2] + else: + r = (1.0 - alpha) * colors[index_int][0] + g = (1.0 - alpha) * colors[index_int][1] + b = (1.0 - alpha) * colors[index_int][2] + + if flat: + palette.extend((int(r), int(g), int(b))) + else: + palette.append((int(r), int(g), int(b))) + + return palette + + +def desaturate(rgb, amount): + """ + desaturate colors by amount + amount == 0, no change + amount == 1, grey + """ + luminosity = sum(rgb) / 3.0 + desat = lambda color: color - amount * (color - luminosity) + + return tuple(map(int, map(desat, rgb))) + + +class WaveformImage(object): + """ + Given peaks and spectral centroids from the AudioProcessor, this class will construct + a wavefile image which can be saved as PNG. + """ + def __init__(self, image_width, image_height, palette=1): + if image_height % 2 == 0: + raise AudioProcessingException, "Height should be uneven: images look much better at uneven height" + + if palette == 1: + background_color = (0,0,0) + colors = [ + (50,0,200), + (0,220,80), + (255,224,0), + (255,70,0), + ] + elif palette == 2: + background_color = (0,0,0) + colors = [self.color_from_value(value/29.0) for value in range(0,30)] + elif palette == 3: + background_color = (213, 217, 221) + colors = map( partial(desaturate, amount=0.7), [ + (50,0,200), + (0,220,80), + (255,224,0), + ]) + elif palette == 4: + background_color = (213, 217, 221) + colors = map( partial(desaturate, amount=0.8), [self.color_from_value(value/29.0) for value in range(0,30)]) + + self.image = Image.new("RGB", (image_width, image_height), background_color) + + self.image_width = image_width + self.image_height = image_height + + self.draw = ImageDraw.Draw(self.image) + self.previous_x, self.previous_y = None, None + + self.color_lookup = interpolate_colors(colors) + self.pix = self.image.load() + + def color_from_value(self, value): + """ given a value between 0 and 1, return an (r,g,b) tuple """ + + return ImageColor.getrgb("hsl(%d,%d%%,%d%%)" % (int( (1.0 - value) * 360 ), 80, 50)) + + def draw_peaks(self, x, peaks, spectral_centroid): + """ draw 2 peaks at x using the spectral_centroid for color """ + + y1 = self.image_height * 0.5 - peaks[0] * (self.image_height - 4) * 0.5 + y2 = self.image_height * 0.5 - peaks[1] * (self.image_height - 4) * 0.5 + + line_color = self.color_lookup[int(spectral_centroid*255.0)] + + if self.previous_y != None: + self.draw.line([self.previous_x, self.previous_y, x, y1, x, y2], line_color) + else: + self.draw.line([x, y1, x, y2], line_color) + + self.previous_x, self.previous_y = x, y2 + + self.draw_anti_aliased_pixels(x, y1, y2, line_color) + + def draw_anti_aliased_pixels(self, x, y1, y2, color): + """ vertical anti-aliasing at y1 and y2 """ + + y_max = max(y1, y2) + y_max_int = int(y_max) + alpha = y_max - y_max_int + + if alpha > 0.0 and alpha < 1.0 and y_max_int + 1 < self.image_height: + current_pix = self.pix[x, y_max_int + 1] + + r = int((1-alpha)*current_pix[0] + alpha*color[0]) + g = int((1-alpha)*current_pix[1] + alpha*color[1]) + b = int((1-alpha)*current_pix[2] + alpha*color[2]) + + self.pix[x, y_max_int + 1] = (r,g,b) + + y_min = min(y1, y2) + y_min_int = int(y_min) + alpha = 1.0 - (y_min - y_min_int) + + if alpha > 0.0 and alpha < 1.0 and y_min_int - 1 >= 0: + current_pix = self.pix[x, y_min_int - 1] + + r = int((1-alpha)*current_pix[0] + alpha*color[0]) + g = int((1-alpha)*current_pix[1] + alpha*color[1]) + b = int((1-alpha)*current_pix[2] + alpha*color[2]) + + self.pix[x, y_min_int - 1] = (r,g,b) + + def save(self, filename): + # draw a zero "zero" line + a = 25 + for x in range(self.image_width): + self.pix[x, self.image_height/2] = tuple(map(lambda p: p+a, self.pix[x, self.image_height/2])) + + self.image.save(filename) + + +class SpectrogramImage(object): + """ + Given spectra from the AudioProcessor, this class will construct a wavefile image which + can be saved as PNG. + """ + def __init__(self, image_width, image_height, fft_size): + self.image_width = image_width + self.image_height = image_height + self.fft_size = fft_size + + self.image = Image.new("RGBA", (image_height, image_width)) + + colors = [ + (0, 0, 0, 0), + (58/4, 68/4, 65/4, 255), + (80/2, 100/2, 153/2, 255), + (90, 180, 100, 255), + (224, 224, 44, 255), + (255, 60, 30, 255), + (255, 255, 255, 255) + ] + self.palette = interpolate_colors(colors) + + # generate the lookup which translates y-coordinate to fft-bin + self.y_to_bin = [] + f_min = 100.0 + f_max = 22050.0 + y_min = math.log10(f_min) + y_max = math.log10(f_max) + for y in range(self.image_height): + freq = math.pow(10.0, y_min + y / (image_height - 1.0) *(y_max - y_min)) + bin = freq / 22050.0 * (self.fft_size/2 + 1) + + if bin < self.fft_size/2: + alpha = bin - int(bin) + + self.y_to_bin.append((int(bin), alpha * 255)) + + # this is a bit strange, but using image.load()[x,y] = ... is + # a lot slower than using image.putadata and then rotating the image + # so we store all the pixels in an array and then create the image when saving + self.pixels = [] + + def draw_spectrum(self, x, spectrum): + # for all frequencies, draw the pixels + for (index, alpha) in self.y_to_bin: + self.pixels.append( self.palette[int((255.0-alpha) * spectrum[index] + alpha * spectrum[index + 1])] ) + + # if the FFT is too small to fill up the image, fill with black to the top + for y in range(len(self.y_to_bin), self.image_height): #@UnusedVariable + self.pixels.append(self.palette[0]) + + def save(self, filename, quality=80): + assert filename.lower().endswith(".jpg") + self.image.putdata(self.pixels) + self.image.transpose(Image.ROTATE_90).save(filename, quality=quality) + + +def create_wave_images(input_filename, output_filename_w, output_filename_s, image_width, image_height, fft_size, progress_callback=None): + """ + Utility function for creating both wavefile and spectrum images from an audio input file. + """ + processor = AudioProcessor(input_filename, fft_size, numpy.hanning) + samples_per_pixel = processor.audio_file.nframes / float(image_width) + + waveform = WaveformImage(image_width, image_height) + spectrogram = SpectrogramImage(image_width, image_height, fft_size) + + for x in range(image_width): + + if progress_callback and x % (image_width/10) == 0: + progress_callback((x*100)/image_width) + + seek_point = int(x * samples_per_pixel) + next_seek_point = int((x + 1) * samples_per_pixel) + + (spectral_centroid, db_spectrum) = processor.spectral_centroid(seek_point) + peaks = processor.peaks(seek_point, next_seek_point) + + waveform.draw_peaks(x, peaks, spectral_centroid) + spectrogram.draw_spectrum(x, db_spectrum) + + if progress_callback: + progress_callback(100) + + waveform.save(output_filename_w) + spectrogram.save(output_filename_s) + + +class NoSpaceLeftException(Exception): + pass + +def convert_to_pcm(input_filename, output_filename): + """ + converts any audio file type to pcm audio + """ + + if not os.path.exists(input_filename): + raise AudioProcessingException, "file %s does not exist" % input_filename + + sound_type = get_sound_type(input_filename) + + if sound_type == "mp3": + cmd = ["lame", "--decode", input_filename, output_filename] + elif sound_type == "ogg": + cmd = ["oggdec", input_filename, "-o", output_filename] + elif sound_type == "flac": + cmd = ["flac", "-f", "-d", "-s", "-o", output_filename, input_filename] + else: + return False + + process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + (stdout, stderr) = process.communicate() + + if process.returncode != 0 or not os.path.exists(output_filename): + if "No space left on device" in stderr + " " + stdout: + raise NoSpaceLeftException + raise AudioProcessingException, "failed converting to pcm data:\n" + " ".join(cmd) + "\n" + stderr + "\n" + stdout + + return True + + +def stereofy_and_find_info(stereofy_executble_path, input_filename, output_filename): + """ + converts a pcm wave file to two channel, 16 bit integer + """ + + if not os.path.exists(input_filename): + raise AudioProcessingException, "file %s does not exist" % input_filename + + cmd = [stereofy_executble_path, "--input", input_filename, "--output", output_filename] + + process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + (stdout, stderr) = process.communicate() + + if process.returncode != 0 or not os.path.exists(output_filename): + if "No space left on device" in stderr + " " + stdout: + raise NoSpaceLeftException + raise AudioProcessingException, "failed calling stereofy data:\n" + " ".join(cmd) + "\n" + stderr + "\n" + stdout + + stdout = (stdout + " " + stderr).replace("\n", " ") + + duration = 0 + m = re.match(r".*#duration (?P[\d\.]+).*", stdout) + if m != None: + duration = float(m.group("duration")) + + channels = 0 + m = re.match(r".*#channels (?P\d+).*", stdout) + if m != None: + channels = float(m.group("channels")) + + samplerate = 0 + m = re.match(r".*#samplerate (?P\d+).*", stdout) + if m != None: + samplerate = float(m.group("samplerate")) + + bitdepth = None + m = re.match(r".*#bitdepth (?P\d+).*", stdout) + if m != None: + bitdepth = float(m.group("bitdepth")) + + bitrate = (os.path.getsize(input_filename) * 8.0) / 1024.0 / duration if duration > 0 else 0 + + return dict(duration=duration, channels=channels, samplerate=samplerate, bitrate=bitrate, bitdepth=bitdepth) + + +def convert_to_mp3(input_filename, output_filename, quality=70): + """ + converts the incoming wave file to a mp3 file + """ + + if not os.path.exists(input_filename): + raise AudioProcessingException, "file %s does not exist" % input_filename + + command = ["lame", "--silent", "--abr", str(quality), input_filename, output_filename] + + process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + (stdout, stderr) = process.communicate() + + if process.returncode != 0 or not os.path.exists(output_filename): + raise AudioProcessingException, stdout + +def convert_to_ogg(input_filename, output_filename, quality=1): + """ + converts the incoming wave file to n ogg file + """ + + if not os.path.exists(input_filename): + raise AudioProcessingException, "file %s does not exist" % input_filename + + command = ["oggenc", "-q", str(quality), input_filename, "-o", output_filename] + + process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + (stdout, stderr) = process.communicate() + + if process.returncode != 0 or not os.path.exists(output_filename): + raise AudioProcessingException, stdout + +def convert_using_ffmpeg(input_filename, output_filename): + """ + converts the incoming wave file to stereo pcm using fffmpeg + """ + TIMEOUT = 3 * 60 + def alarm_handler(signum, frame): + raise AudioProcessingException, "timeout while waiting for ffmpeg" + + if not os.path.exists(input_filename): + raise AudioProcessingException, "file %s does not exist" % input_filename + + command = ["ffmpeg", "-y", "-i", input_filename, "-ac","1","-acodec", "pcm_s16le", "-ar", "44100", output_filename] + + process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + signal.signal(signal.SIGALRM,alarm_handler) + signal.alarm(TIMEOUT) + (stdout, stderr) = process.communicate() + signal.alarm(0) + if process.returncode != 0 or not os.path.exists(output_filename): + raise AudioProcessingException, stdout diff --git a/mediagoblin/config_spec.ini b/mediagoblin/config_spec.ini index 452d9745..b429677c 100644 --- a/mediagoblin/config_spec.ini +++ b/mediagoblin/config_spec.ini @@ -65,6 +65,14 @@ base_url = string(default="/mgoblin_media/") storage_class = string(default="mediagoblin.storage.filestorage:BasicFileStorage") base_dir = string(default="%(here)s/user_dev/media/queue") +[media:medium] +max_width = integer(default=640) +max_height = integer(default=640) + +[media:thumb] +max_width = integer(default=180) +max_height = integer(default=180) + [media_type:mediagoblin.media_types.video] # Should we keep the original file? keep_original = boolean(default=False) @@ -72,6 +80,7 @@ keep_original = boolean(default=False) [media_type:mediagoblin.media_types.audio] # vorbisenc qualiy quality = float(default=0.3) +create_spectrogram = boolean(default=False) [beaker.cache] diff --git a/mediagoblin/media_types/ascii/processing.py b/mediagoblin/media_types/ascii/processing.py index f698b97a..75184c1f 100644 --- a/mediagoblin/media_types/ascii/processing.py +++ b/mediagoblin/media_types/ascii/processing.py @@ -24,7 +24,16 @@ from mediagoblin.media_types.ascii import asciitoimage _log = logging.getLogger(__name__) +SUPPORTED_EXTENSIONS = ['txt', 'asc', 'nfo'] + def sniff_handler(media_file, **kw): + if not kw.get('media') == None: + name, ext = os.path.splitext(kw['media'].filename) + clean_ext = ext[1:].lower() + + if clean_ext in SUPPORTED_EXTENSIONS: + return True + return False def process_ascii(entry): diff --git a/mediagoblin/media_types/audio/audioprocessing.py b/mediagoblin/media_types/audio/audioprocessing.py new file mode 120000 index 00000000..c5e3c52c --- /dev/null +++ b/mediagoblin/media_types/audio/audioprocessing.py @@ -0,0 +1 @@ +../../../extlib/freesound/audioprocessing.py \ No newline at end of file diff --git a/mediagoblin/media_types/audio/processing.py b/mediagoblin/media_types/audio/processing.py index 7aa7ace8..6769f605 100644 --- a/mediagoblin/media_types/audio/processing.py +++ b/mediagoblin/media_types/audio/processing.py @@ -21,9 +21,10 @@ import os from mediagoblin import mg_globals as mgg from mediagoblin.processing import create_pub_filepath -from mediagoblin.media_types.audio.transcoders import AudioTranscoder +from mediagoblin.media_types.audio.transcoders import AudioTranscoder, \ + AudioThumbnailer -_log = logging.getLogger() +_log = logging.getLogger(__name__) def sniff_handler(media_file, **kw): transcoder = AudioTranscoder() @@ -33,7 +34,9 @@ def sniff_handler(media_file, **kw): if data.is_audio == True and data.is_video == False: return True except: - return False + pass + + return False def process_audio(entry): audio_config = mgg.global_config['media_type:mediagoblin.media_types.audio'] @@ -51,10 +54,9 @@ def process_audio(entry): original=os.path.splitext( queued_filepath[-1])[0])) - ogg_tmp = tempfile.NamedTemporaryFile() + transcoder = AudioTranscoder() - with ogg_tmp: - transcoder = AudioTranscoder() + with tempfile.NamedTemporaryFile() as ogg_tmp: transcoder.transcode( queued_filename, @@ -72,11 +74,54 @@ def process_audio(entry): entry.media_data['audio'] = { u'length': int(data.audiolength)} - thumbnail_tmp = tempfile.NamedTemporaryFile() + if audio_config['create_spectrogram']: + spectrogram_filepath = create_pub_filepath( + entry, + '{original}-spectrogram.jpg'.format( + original=os.path.splitext( + queued_filepath[-1])[0])) - with thumbnail_tmp: + with tempfile.NamedTemporaryFile(suffix='.wav') as wav_tmp: + _log.info('Creating WAV source for spectrogram') + transcoder.transcode( + queued_filename, + wav_tmp.name, + mux_string='wavenc') + + thumbnailer = AudioThumbnailer() + + with tempfile.NamedTemporaryFile(suffix='.jpg') as spectrogram_tmp: + thumbnailer.spectrogram( + wav_tmp.name, + spectrogram_tmp.name, + width=mgg.global_config['media:medium']['max_width']) + + _log.debug('Saving spectrogram...') + mgg.public_store.get_file(spectrogram_filepath, 'wb').write( + spectrogram_tmp.read()) + + entry.media_files['spectrogram'] = spectrogram_filepath + + with tempfile.NamedTemporaryFile(suffix='.jpg') as thumb_tmp: + thumbnailer.thumbnail_spectrogram( + spectrogram_tmp.name, + thumb_tmp.name, + (mgg.global_config['media:thumb']['max_width'], + mgg.global_config['media:thumb']['max_height'])) + + thumb_filepath = create_pub_filepath( + entry, + '{original}-thumbnail.jpg'.format( + original=os.path.splitext( + queued_filepath[-1])[0])) + + mgg.public_store.get_file(thumb_filepath, 'wb').write( + thumb_tmp.read()) + + entry.media_files['thumb'] = thumb_filepath + else: entry.media_files['thumb'] = ['fake', 'thumb', 'path.jpg'] - + mgg.queue_store.delete_file(queued_filepath) entry.save() diff --git a/mediagoblin/media_types/audio/transcoders.py b/mediagoblin/media_types/audio/transcoders.py index c5634964..7649309c 100644 --- a/mediagoblin/media_types/audio/transcoders.py +++ b/mediagoblin/media_types/audio/transcoders.py @@ -16,8 +16,10 @@ import pdb import logging +from PIL import Image from mediagoblin.processing import BadMediaFail +from mediagoblin.media_types.audio import audioprocessing _log = logging.getLogger(__name__) @@ -56,6 +58,73 @@ try: except ImportError: raise Exception('gst/pygst >= 0.10 could not be imported') +import numpy + +class AudioThumbnailer(object): + def __init__(self): + _log.info('Initializing {0}'.format(self.__class__.__name__)) + + def spectrogram(self, src, dst, **kw): + width = kw['width'] + height = int(kw.get('height', float(width) * 0.3)) + fft_size = kw.get('fft_size', 2048) + callback = kw.get('progress_callback') + + processor = audioprocessing.AudioProcessor( + src, + fft_size, + numpy.hanning) + + samples_per_pixel = processor.audio_file.nframes / float(width) + + spectrogram = audioprocessing.SpectrogramImage(width, height, fft_size) + + for x in range(width): + if callback and x % (width / 10) == 0: + callback((x * 100) / width) + + seek_point = int(x * samples_per_pixel) + + (spectral_centroid, db_spectrum) = processor.spectral_centroid( + seek_point) + + spectrogram.draw_spectrum(x, db_spectrum) + + if callback: + callback(100) + + spectrogram.save(dst) + + def thumbnail_spectrogram(self, src, dst, thumb_size): + ''' + Takes a spectrogram and creates a thumbnail from it + ''' + if not (type(thumb_size) == tuple and len(thumb_size) == 2): + raise Exception('size argument should be a tuple(width, height)') + + im = Image.open(src) + + im_w, im_h = [float(i) for i in im.size] + th_w, th_h = [float(i) for i in thumb_size] + + wadsworth_position = im_w * 0.3 + + start_x = max(( + wadsworth_position - (th_w / 2.0), + 0.0)) + + stop_x = start_x + (im_h * (th_w / th_h)) + + th = im.crop(( + int(start_x), 0, + int(stop_x), int(im_h))) + + if th.size[0] > th_w or th.size[1] > th_h: + th.thumbnail(thumb_size, Image.ANTIALIAS) + + th.save(dst) + + class AudioTranscoder(object): def __init__(self): _log.info('Initializing {0}'.format(self.__class__.__name__)) @@ -103,17 +172,21 @@ class AudioTranscoder(object): quality = kw.get('quality', 0.3) + mux_string = kw.get( + 'mux_string', + 'vorbisenc quality={0} ! webmmux'.format(quality)) + # Set up pipeline self.pipeline = gst.parse_launch( 'filesrc location="{src}" ! ' 'decodebin2 ! queue ! audiorate tolerance={tolerance} ! ' 'audioconvert ! audio/x-raw-float,channels=2 ! ' - 'vorbisenc quality={quality} ! webmmux ! ' + '{mux_string} ! ' 'progressreport silent=true ! ' 'filesink location="{dst}"'.format( src=src, tolerance=80000000, - quality=quality, + mux_string=mux_string, dst=dst)) self.bus = self.pipeline.get_bus() @@ -141,6 +214,9 @@ class AudioTranscoder(object): self.halt() def halt(self): + if getattr(self, 'pipeline', False): + self.pipeline.set_state(gst.STATE_NULL) + del self.pipeline _log.info('Quitting MainLoop gracefully...') gobject.idle_add(self._loop.quit) @@ -149,8 +225,12 @@ if __name__ == '__main__': logging.basicConfig() _log.setLevel(logging.INFO) - transcoder = AudioTranscoder() - data = transcoder.discover(sys.argv[1]) - res = transcoder.transcode(*sys.argv[1:3]) + #transcoder = AudioTranscoder() + #data = transcoder.discover(sys.argv[1]) + #res = transcoder.transcode(*sys.argv[1:3]) + + thumbnailer = AudioThumbnailer() + + thumbnailer.spectrogram(*sys.argv[1:], width=640) pdb.set_trace() diff --git a/mediagoblin/media_types/image/processing.py b/mediagoblin/media_types/image/processing.py index 364a5afa..28cde2aa 100644 --- a/mediagoblin/media_types/image/processing.py +++ b/mediagoblin/media_types/image/processing.py @@ -42,7 +42,7 @@ def sniff_handler(media_file, **kw): _log.info('Found file extension in supported filetypes') return True else: - _log.debug('Media present, extension not found in {1}'.format( + _log.debug('Media present, extension not found in {0}'.format( SUPPORTED_FILETYPES)) else: _log.warning('Need additional information (keyword argument \'media\')' diff --git a/mediagoblin/media_types/video/processing.py b/mediagoblin/media_types/video/processing.py index 1890ef0c..d2562e3b 100644 --- a/mediagoblin/media_types/video/processing.py +++ b/mediagoblin/media_types/video/processing.py @@ -29,6 +29,18 @@ _log = logging.getLogger(__name__) _log.setLevel(logging.DEBUG) def sniff_handler(media_file, **kw): + transcoder = transcoders.VideoTranscoder() + try: + data = transcoder.discover(media_file.name) + + _log.debug('Discovered: {0}'.format(data.__dict__)) + + if data.is_video == True: + return True + except: + _log.error('Exception caught when trying to discover {0}'.format( + kw.get('media'))) + return False def process_video(entry): @@ -61,7 +73,8 @@ def process_video(entry): with tmp_dst: # Transcode queued file to a VP8/vorbis file that fits in a 640x640 square - transcoder = transcoders.VideoTranscoder(queued_filename, tmp_dst.name) + transcoder = transcoders.VideoTranscoder() + transcoder.transcode(queued_filename, tmp_dst.name) # Push transcoded video to public storage _log.debug('Saving medium...') diff --git a/mediagoblin/media_types/video/transcoders.py b/mediagoblin/media_types/video/transcoders.py index 903bd810..6c2e885e 100644 --- a/mediagoblin/media_types/video/transcoders.py +++ b/mediagoblin/media_types/video/transcoders.py @@ -25,8 +25,6 @@ import pdb import urllib _log = logging.getLogger(__name__) -logging.basicConfig() -_log.setLevel(logging.DEBUG) CPU_COUNT = 2 try: @@ -340,10 +338,15 @@ class VideoTranscoder: that it was refined afterwards and therefore is done more correctly. ''' - def __init__(self, src, dst, **kwargs): + def __init__(self): _log.info('Initializing VideoTranscoder...') self.loop = gobject.MainLoop() + + def transcode(self, src, dst, **kwargs): + ''' + Transcode a video file into a 'medium'-sized version. + ''' self.source_path = src self.destination_path = dst @@ -357,6 +360,30 @@ class VideoTranscoder: self._setup() self._run() + def discover(self, src): + ''' + Discover properties about a media file + ''' + _log.info('Discovering {0}'.format(src)) + + self.source_path = src + self._setup_discover(discovered_callback=self.__on_discovered) + + self.discoverer.discover() + + self.loop.run() + + return self._discovered_data + + def __on_discovered(self, data, is_media): + if not is_media: + self.__stop() + raise Exception('Could not discover {0}'.format(self.source_path)) + + self._discovered_data = data + + self.__stop_mainloop() + def _setup(self): self._setup_discover() self._setup_pipeline() @@ -369,12 +396,14 @@ class VideoTranscoder: _log.debug('Initializing MainLoop()') self.loop.run() - def _setup_discover(self): + def _setup_discover(self, **kw): _log.debug('Setting up discoverer') self.discoverer = discoverer.Discoverer(self.source_path) # Connect self.__discovered to the 'discovered' event - self.discoverer.connect('discovered', self.__discovered) + self.discoverer.connect( + 'discovered', + kw.get('discovered_callback', self.__discovered)) def __discovered(self, data, is_media): ''' @@ -614,14 +643,15 @@ class VideoTranscoder: if __name__ == '__main__': os.nice(19) + logging.basicConfig() from optparse import OptionParser parser = OptionParser( - usage='%prog [-v] -a [ video | thumbnail ] SRC DEST') + usage='%prog [-v] -a [ video | thumbnail | discover ] SRC [ DEST ]') parser.add_option('-a', '--action', dest='action', - help='One of "video" or "thumbnail"') + help='One of "video", "discover" or "thumbnail"') parser.add_option('-v', dest='verbose', @@ -645,13 +675,18 @@ if __name__ == '__main__': _log.debug(args) - if not len(args) == 2: + if not len(args) == 2 and not options.action == 'discover': parser.print_help() sys.exit() + transcoder = VideoTranscoder() + if options.action == 'thumbnail': VideoThumbnailer(*args) elif options.action == 'video': def cb(data): print('I\'m a callback!') - transcoder = VideoTranscoder(*args, progress_callback=cb) + transcoder.transcode(*args, progress_callback=cb) + elif options.action == 'discover': + print transcoder.discover(*args).__dict__ + diff --git a/mediagoblin/templates/mediagoblin/media_displays/audio.html b/mediagoblin/templates/mediagoblin/media_displays/audio.html index 802a85c1..f130e323 100644 --- a/mediagoblin/templates/mediagoblin/media_displays/audio.html +++ b/mediagoblin/templates/mediagoblin/media_displays/audio.html @@ -20,7 +20,14 @@ {% block mediagoblin_media %}
-
+ {% endif %} +