Merge branch 'master' of http://git.gitorious.org/mediagoblin/mediagoblin

mediagoblin/app.py

@@ -25,6 +25,7 @@ from mediagoblin import routing, util, storage, staticdirect
 from mediagoblin.db.open import setup_connection_and_db_from_config
 from mediagoblin.globals import setup_globals
 from mediagoblin.celery_setup import setup_celery_from_config
+from mediagoblin.workbench import WorkbenchManager, DEFAULT_WORKBENCH_DIR
 
 
 class Error(Exception): pass
@@ -39,7 +40,8 @@ class MediaGoblinApp(object):
                  public_store, queue_store,
                  staticdirector,
                  email_sender_address, email_debug_mode,
-                 user_template_path=None):
+                 user_template_path=None,
+                 workbench_path=DEFAULT_WORKBENCH_DIR):
         # Get the template environment
         self.template_loader = util.get_jinja_loader(user_template_path)
         
@@ -66,7 +68,8 @@ class MediaGoblinApp(object):
             db_connection=connection,
             database=self.db,
             public_store=self.public_store,
-            queue_store=self.queue_store)
+            queue_store=self.queue_store,
+            workbench_manager=WorkbenchManager(workbench_path))
 
     def __call__(self, environ, start_response):
         request = Request(environ)
@@ -154,6 +157,7 @@ def paste_app_factory(global_config, **app_config):
         email_sender_address=app_config.get(
             'email_sender_address', 'notice@mediagoblin.example.org'),
         email_debug_mode=asbool(app_config.get('email_debug_mode')),
-        user_template_path=app_config.get('local_templates'))
+        user_template_path=app_config.get('local_templates'),
+        workbench_path=app_config.get('workbench_path', DEFAULT_WORKBENCH_DIR))
 
     return mgoblin_app

mediagoblin/celery_setup/from_celery.py

@@ -23,7 +23,7 @@ from mediagoblin import storage
 from mediagoblin.db.open import setup_connection_and_db_from_config
 from mediagoblin.celery_setup import setup_celery_from_config
 from mediagoblin.globals import setup_globals
-from mediagoblin import globals as mgoblin_globals
+from mediagoblin.workbench import WorkbenchManager, DEFAULT_WORKBENCH_DIR
 
 
 OUR_MODULENAME = 'mediagoblin.celery_setup.from_celery'
@@ -76,6 +76,10 @@ def setup_self(setup_globals_func=setup_globals):
     queue_store = storage.storage_system_from_paste_config(
         mgoblin_section, 'queuestore')
 
+    workbench_manager = WorkbenchManager(
+        mgoblin_section.get(
+            'workbench_path', DEFAULT_WORKBENCH_DIR))
+
     setup_globals_func(
         db_connection=connection,
         database=db,
@@ -84,7 +88,8 @@ def setup_self(setup_globals_func=setup_globals):
         email_sender_address=mgoblin_section.get(
             'email_sender_address', 
             'notice@mediagoblin.example.org'),
-        queue_store=queue_store)
+        queue_store=queue_store,
+        workbench_manager=workbench_manager)
 
 
 if os.environ['CELERY_CONFIG_MODULE'] == OUR_MODULENAME:

mediagoblin/process_media/__init__.py

@@ -18,7 +18,7 @@ import Image
 from mediagoblin.db.util import ObjectId
 from celery.task import task
 
-from mediagoblin.globals import database, queue_store, public_store
+from mediagoblin import globals as mg_globals
 
 
 THUMB_SIZE = 200, 200
@@ -26,40 +26,50 @@ THUMB_SIZE = 200, 200
 
 @task
 def process_media_initial(media_id):
-    entry = database.MediaEntry.one(
+    workbench = mg_globals.workbench_manager.create_workbench()
+
+    entry = mg_globals.database.MediaEntry.one(
         {'_id': ObjectId(media_id)})
 
     queued_filepath = entry['queued_media_file']
-    queued_file = queue_store.get_file(queued_filepath, 'r')
+    queued_filename = mg_globals.workbench_manager.localized_file(
+        workbench, mg_globals.queue_store, queued_filepath,
+        'source')
+
+    queued_file = file(queued_filename, 'r')
 
     with queued_file:
         thumb = Image.open(queued_file)
         thumb.thumbnail(THUMB_SIZE, Image.ANTIALIAS)
 
-        thumb_filepath = public_store.get_unique_filepath(
+        thumb_filepath = mg_globals.public_store.get_unique_filepath(
             ['media_entries',
              unicode(entry['_id']),
              'thumbnail.jpg'])
 
-        with public_store.get_file(thumb_filepath, 'w') as thumb_file:
+        thumb_file = mg_globals.public_store.get_file(thumb_filepath, 'w')
+        with thumb_file:
             thumb.save(thumb_file, "JPEG")
 
     # we have to re-read because unlike PIL, not everything reads
     # things in string representation :)
-    queued_file = queue_store.get_file(queued_filepath, 'rb')
+    queued_file = file(queued_filename, 'rb')
 
     with queued_file:
-        main_filepath = public_store.get_unique_filepath(
+        main_filepath = mg_globals.public_store.get_unique_filepath(
             ['media_entries',
              unicode(entry['_id']),
              queued_filepath[-1]])
         
-        with public_store.get_file(main_filepath, 'wb') as main_file:
+        with mg_globals.public_store.get_file(main_filepath, 'wb') as main_file:
             main_file.write(queued_file.read())
 
-    queue_store.delete_file(queued_filepath)
+    mg_globals.queue_store.delete_file(queued_filepath)
     media_files_dict = entry.setdefault('media_files', {})
     media_files_dict['thumb'] = thumb_filepath
     media_files_dict['main'] = main_filepath
     entry['state'] = u'processed'
     entry.save()
+
+    # clean up workbench
+    mg_globals.workbench_manager.destroy_workbench(workbench)

mediagoblin/storage.py

@@ -16,6 +16,7 @@
 
 import os
 import re
+import shutil
 import urlparse
 import uuid
 
@@ -60,6 +61,9 @@ class StorageInterface(object):
     StorageInterface.
     """
 
+    # Whether this file store is on the local filesystem.
+    local_storage = False
+
     def __raise_not_implemented(self):
         """
         Raise a warning about some component not implemented by a
@@ -127,12 +131,43 @@ class StorageInterface(object):
         else:
             return filepath
 
+    def get_local_path(self, filepath):
+        """
+        If this is a local_storage implementation, give us a link to
+        the local filesystem reference to this file.
+
+        >>> storage_handler.get_local_path(['foo', 'bar', 'baz.jpg'])
+        u'/path/to/mounting/foo/bar/baz.jpg'
+        """
+        # Subclasses should override this method, if applicable.
+        self.__raise_not_implemented()
+
+    def copy_locally(self, filepath, dest_path):
+        """
+        Copy this file locally.
+
+        A basic working method for this is provided that should
+        function both for local_storage systems and remote storge
+        systems, but if more efficient systems for copying locally
+        apply to your system, override this method with something more
+        appropriate.
+        """
+        if self.local_storage:
+            shutil.copy(
+                self.get_local_path(filepath), dest_path)
+        else:
+            with self.get_file(filepath, 'rb') as source_file:
+                with file(dest_path, 'wb') as dest_file:
+                    dest_file.write(source_file.read())
+
 
 class BasicFileStorage(StorageInterface):
     """
     Basic local filesystem implementation of storage API
     """
 
+    local_storage = True
+
     def __init__(self, base_dir, base_url=None, **kwargs):
         """
         Keyword arguments:
@@ -177,6 +212,9 @@ class BasicFileStorage(StorageInterface):
             self.base_url,
             '/'.join(clean_listy_filepath(filepath)))
 
+    def get_local_path(self, filepath):
+        return self._resolve_filepath(filepath)
+
 
 ###########
 # Utilities
@@ -187,7 +225,7 @@ def clean_listy_filepath(listy_filepath):
     Take a listy filepath (like ['dir1', 'dir2', 'filename.jpg']) and
     clean out any nastiness from it.
 
-    For example:
+
     >>> clean_listy_filepath([u'/dir1/', u'foo/../nasty', u'linooks.jpg'])
     [u'dir1', u'foo_.._nasty', u'linooks.jpg']
 
@@ -253,3 +291,5 @@ def storage_system_from_paste_config(paste_config, storage_prefix):
 
     storage_class = util.import_component(storage_class)
     return storage_class(**config_params)
+
+

mediagoblin/tests/test_storage.py

@@ -52,6 +52,11 @@ class FakeStorageSystem():
         self.foobie = foobie
         self.blech = blech
 
+class FakeRemoteStorage(storage.BasicFileStorage):
+    # Theoretically despite this, all the methods should work but it
+    # should force copying to the workbench
+    local_storage = False
+
 
 def test_storage_system_from_paste_config():
     this_storage = storage.storage_system_from_paste_config(
@@ -81,9 +86,12 @@ def test_storage_system_from_paste_config():
 # Basic file storage tests
 ##########################
 
-def get_tmp_filestorage(mount_url=None):
+def get_tmp_filestorage(mount_url=None, fake_remote=False):
     tmpdir = tempfile.mkdtemp()
-    this_storage = storage.BasicFileStorage(tmpdir, mount_url)
+    if fake_remote:
+        this_storage = FakeRemoteStorage(tmpdir, mount_url)
+    else:
+        this_storage = storage.BasicFileStorage(tmpdir, mount_url)
     return tmpdir, this_storage
 
 
@@ -214,3 +222,36 @@ def test_basic_storage_url_for_file():
         ['dir1', 'dir2', 'filename.txt'])
     expected = 'http://media.example.org/ourmedia/dir1/dir2/filename.txt'
     assert result == expected
+
+
+def test_basic_storage_get_local_path():
+    tmpdir, this_storage = get_tmp_filestorage()
+    
+    result = this_storage.get_local_path(
+        ['dir1', 'dir2', 'filename.txt'])
+
+    expected = os.path.join(
+        tmpdir, 'dir1/dir2/filename.txt')
+
+    assert result == expected
+
+
+def test_basic_storage_is_local():
+    tmpdir, this_storage = get_tmp_filestorage()
+    assert this_storage.local_storage is True
+
+
+def test_basic_storage_copy_locally():
+    tmpdir, this_storage = get_tmp_filestorage()
+
+    dest_tmpdir = tempfile.mkdtemp()
+
+    filepath = ['dir1', 'dir2', 'ourfile.txt']
+    with this_storage.get_file(filepath, 'w') as our_file:
+        our_file.write('Testing this file')
+
+    new_file_dest = os.path.join(dest_tmpdir, 'file2.txt')
+
+    this_storage.copy_locally(filepath, new_file_dest)
+    
+    assert file(new_file_dest).read() == 'Testing this file'

mediagoblin/tests/test_workbench.py

@@ -0,0 +1,96 @@
+# GNU MediaGoblin -- federated, autonomous media hosting
+# Copyright (C) 2011 Free Software Foundation, Inc
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import tempfile
+
+from nose.tools import assert_raises
+
+from mediagoblin import workbench
+from mediagoblin.tests.test_storage import get_tmp_filestorage
+
+
+class TestWorkbench(object):
+    def setUp(self):
+        self.workbench_manager = workbench.WorkbenchManager(
+            os.path.join(tempfile.gettempdir(), u'mgoblin_workbench_testing'))
+
+    def test_create_workbench(self):
+        workbench = self.workbench_manager.create_workbench()
+        assert os.path.isdir(workbench)
+        assert workbench.startswith(self.workbench_manager.base_workbench_dir)
+
+    def test_destroy_workbench(self):
+        # kill a workbench
+        this_workbench = self.workbench_manager.create_workbench()
+        tmpfile = file(os.path.join(this_workbench, 'temp.txt'), 'w')
+        with tmpfile:
+            tmpfile.write('lollerskates')
+
+        assert os.path.exists(os.path.join(this_workbench, 'temp.txt'))
+
+        self.workbench_manager.destroy_workbench(this_workbench)
+        assert not os.path.exists(os.path.join(this_workbench, 'temp.txt'))
+        assert not os.path.exists(this_workbench)
+
+        # make sure we can't kill other stuff though
+        dont_kill_this = tempfile.mkdtemp()
+
+        assert_raises(
+            workbench.WorkbenchOutsideScope,
+            self.workbench_manager.destroy_workbench,
+            dont_kill_this)
+
+    def test_localized_file(self):
+        tmpdir, this_storage = get_tmp_filestorage()
+        this_workbench = self.workbench_manager.create_workbench()
+        
+        # Write a brand new file
+        filepath = ['dir1', 'dir2', 'ourfile.txt']
+
+        with this_storage.get_file(filepath, 'w') as our_file:
+            our_file.write('Our file')
+
+        # with a local file storage
+        filename = self.workbench_manager.localized_file(
+            this_workbench, this_storage, filepath)
+        assert filename == os.path.join(
+            tmpdir, 'dir1/dir2/ourfile.txt')
+
+        # with a fake remote file storage
+        tmpdir, this_storage = get_tmp_filestorage(fake_remote=True)
+
+        # ... write a brand new file, again ;)
+        with this_storage.get_file(filepath, 'w') as our_file:
+            our_file.write('Our file')
+
+        filename = self.workbench_manager.localized_file(
+            this_workbench, this_storage, filepath)
+        assert filename == os.path.join(
+            this_workbench, 'ourfile.txt')
+        
+        # fake remote file storage, filename_if_copying set
+        filename = self.workbench_manager.localized_file(
+            this_workbench, this_storage, filepath, 'thisfile')
+        assert filename == os.path.join(
+            this_workbench, 'thisfile.txt')
+
+        # fake remote file storage, filename_if_copying set,
+        # keep_extension_if_copying set to false
+        filename = self.workbench_manager.localized_file(
+            this_workbench, this_storage, filepath, 'thisfile.text', False)
+        assert filename == os.path.join(
+            this_workbench, 'thisfile.text')

mediagoblin/workbench.py

@@ -0,0 +1,135 @@
+# GNU MediaGoblin -- federated, autonomous media hosting
+# Copyright (C) 2011 Free Software Foundation, Inc
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import shutil
+import tempfile
+
+
+DEFAULT_WORKBENCH_DIR = os.path.join(
+    tempfile.gettempdir(), u'mgoblin_workbench')
+
+
+# Exception(s)
+# ------------
+
+class WorkbenchOutsideScope(Exception):
+    """
+    Raised when a workbench is outside a WorkbenchManager scope.
+    """
+    pass
+
+
+# Actual workbench stuff
+# ----------------------
+
+class WorkbenchManager(object):
+    """
+    A system for generating and destroying workbenches.
+
+    Workbenches are actually just subdirectories of a temporary storage space
+    for during the processing stage.
+    """
+
+    def __init__(self, base_workbench_dir):
+        self.base_workbench_dir = os.path.abspath(base_workbench_dir)
+        if not os.path.exists(self.base_workbench_dir):
+            os.makedirs(self.base_workbench_dir)
+        
+    def create_workbench(self):
+        """
+        Create and return the path to a new workbench (directory).
+        """
+        return tempfile.mkdtemp(dir=self.base_workbench_dir)
+
+    def destroy_workbench(self, workbench):
+        """
+        Destroy this workbench!  Deletes the directory and all its contents!
+
+        Makes sure the workbench actually belongs to this manager though.
+        """
+        # just in case
+        workbench = os.path.abspath(workbench)
+
+        if not workbench.startswith(self.base_workbench_dir):
+            raise WorkbenchOutsideScope(
+                "Can't destroy workbench outside the base workbench dir")
+
+        shutil.rmtree(workbench)
+
+    def localized_file(self, workbench, storage, filepath,
+                       filename_if_copying=None,
+                       keep_extension_if_copying=True):
+        """
+        Possibly localize the file from this storage system (for read-only
+        purposes, modifications should be written to a new file.).
+
+        If the file is already local, just return the absolute filename of that
+        local file.  Otherwise, copy the file locally to the workbench, and
+        return the absolute path of the new file.
+
+        If it is copying locally, we might want to require a filename like
+        "source.jpg" to ensure that we won't conflict with other filenames in
+        our workbench... if that's the case, make sure filename_if_copying is
+        set to something like 'source.jpg'.  Relatedly, if you set
+        keep_extension_if_copying, you don't have to set an extension on
+        filename_if_copying yourself, it'll be set for you (assuming such an
+        extension can be extacted from the filename in the filepath).
+
+        Returns:
+          localized_filename
+
+        Examples:
+          >>> wb_manager.localized_file(
+          ...     '/our/workbench/subdir', local_storage,
+          ...     ['path', 'to', 'foobar.jpg'])
+          u'/local/storage/path/to/foobar.jpg'
+
+          >>> wb_manager.localized_file(
+          ...     '/our/workbench/subdir', remote_storage,
+          ...     ['path', 'to', 'foobar.jpg'])
+          '/our/workbench/subdir/foobar.jpg'
+
+          >>> wb_manager.localized_file(
+          ...     '/our/workbench/subdir', remote_storage,
+          ...     ['path', 'to', 'foobar.jpg'], 'source.jpeg', False)
+          '/our/workbench/subdir/foobar.jpeg'
+
+          >>> wb_manager.localized_file(
+          ...     '/our/workbench/subdir', remote_storage,
+          ...     ['path', 'to', 'foobar.jpg'], 'source', True)
+          '/our/workbench/subdir/foobar.jpg'
+        """
+        if storage.local_storage:
+            return storage.get_local_path(filepath)
+        else:
+            if filename_if_copying is None:
+                dest_filename = filepath[-1]
+            else:
+                orig_filename, orig_ext = os.path.splitext(filepath[-1])
+                if keep_extension_if_copying and orig_ext:
+                    dest_filename = filename_if_copying + orig_ext
+                else:
+                    dest_filename = filename_if_copying
+
+            full_dest_filename = os.path.join(
+                workbench, dest_filename)
+
+            # copy it over
+            storage.copy_locally(
+                filepath, full_dest_filename)
+
+            return full_dest_filename