heckyel/mediagoblin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' of http://git.gitorious.org/mediagoblin/mediagoblin

Browse Source
This commit is contained in:
Joar Wandborg 2011-06-12 14:37:49 +02:00
commit a48014d67a
7 changed files with 348 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
mediagoblin/app.py
View File

@ -25,6 +25,7 @@ from mediagoblin import routing, util, storage, staticdirect
from mediagoblin.db.open import setup_connection_and_db_from_config
from mediagoblin.globals import setup_globals
from mediagoblin.celery_setup import setup_celery_from_config
from mediagoblin.workbench import WorkbenchManager, DEFAULT_WORKBENCH_DIR
class Error(Exception): pass
@ -39,7 +40,8 @@ class MediaGoblinApp(object):
public_store, queue_store,
staticdirector,
email_sender_address, email_debug_mode,
user_template_path=None):
user_template_path=None,
workbench_path=DEFAULT_WORKBENCH_DIR):
# Get the template environment
self.template_loader = util.get_jinja_loader(user_template_path)
@ -66,7 +68,8 @@ class MediaGoblinApp(object):
db_connection=connection,
database=self.db,
public_store=self.public_store,
queue_store=self.queue_store)
queue_store=self.queue_store,
workbench_manager=WorkbenchManager(workbench_path))
def __call__(self, environ, start_response):
request = Request(environ)
@ -154,6 +157,7 @@ def paste_app_factory(global_config, **app_config):
email_sender_address=app_config.get(
'email_sender_address', 'notice@mediagoblin.example.org'),
email_debug_mode=asbool(app_config.get('email_debug_mode')),
user_template_path=app_config.get('local_templates'))
user_template_path=app_config.get('local_templates'),
workbench_path=app_config.get('workbench_path', DEFAULT_WORKBENCH_DIR))
return mgoblin_app

9
mediagoblin/celery_setup/from_celery.py
View File

@ -23,7 +23,7 @@ from mediagoblin import storage
from mediagoblin.db.open import setup_connection_and_db_from_config
from mediagoblin.celery_setup import setup_celery_from_config
from mediagoblin.globals import setup_globals
from mediagoblin import globals as mgoblin_globals
from mediagoblin.workbench import WorkbenchManager, DEFAULT_WORKBENCH_DIR
OUR_MODULENAME = 'mediagoblin.celery_setup.from_celery'
@ -76,6 +76,10 @@ def setup_self(setup_globals_func=setup_globals):
queue_store = storage.storage_system_from_paste_config(
mgoblin_section, 'queuestore')
workbench_manager = WorkbenchManager(
mgoblin_section.get(
'workbench_path', DEFAULT_WORKBENCH_DIR))
setup_globals_func(
db_connection=connection,
database=db,
@ -84,7 +88,8 @@ def setup_self(setup_globals_func=setup_globals):
email_sender_address=mgoblin_section.get(
'email_sender_address',
'notice@mediagoblin.example.org'),
queue_store=queue_store)
queue_store=queue_store,
workbench_manager=workbench_manager)
if os.environ['CELERY_CONFIG_MODULE'] == OUR_MODULENAME:

28
mediagoblin/process_media/__init__.py
View File

@ -18,7 +18,7 @@ import Image
from mediagoblin.db.util import ObjectId
from celery.task import task
from mediagoblin.globals import database, queue_store, public_store
from mediagoblin import globals as mg_globals
THUMB_SIZE = 200, 200
@ -26,40 +26,50 @@ THUMB_SIZE = 200, 200
@task
def process_media_initial(media_id):
entry = database.MediaEntry.one(
workbench = mg_globals.workbench_manager.create_workbench()
entry = mg_globals.database.MediaEntry.one(
{'_id': ObjectId(media_id)})
queued_filepath = entry['queued_media_file']
queued_file = queue_store.get_file(queued_filepath, 'r')
queued_filename = mg_globals.workbench_manager.localized_file(
workbench, mg_globals.queue_store, queued_filepath,
'source')
queued_file = file(queued_filename, 'r')
with queued_file:
thumb = Image.open(queued_file)
thumb.thumbnail(THUMB_SIZE, Image.ANTIALIAS)
thumb_filepath = public_store.get_unique_filepath(
thumb_filepath = mg_globals.public_store.get_unique_filepath(
['media_entries',
unicode(entry['_id']),
'thumbnail.jpg'])
with public_store.get_file(thumb_filepath, 'w') as thumb_file:
thumb_file = mg_globals.public_store.get_file(thumb_filepath, 'w')
with thumb_file:
thumb.save(thumb_file, "JPEG")
# we have to re-read because unlike PIL, not everything reads
# things in string representation :)
queued_file = queue_store.get_file(queued_filepath, 'rb')
queued_file = file(queued_filename, 'rb')
with queued_file:
main_filepath = public_store.get_unique_filepath(
main_filepath = mg_globals.public_store.get_unique_filepath(
['media_entries',
unicode(entry['_id']),
queued_filepath[-1]])
with public_store.get_file(main_filepath, 'wb') as main_file:
with mg_globals.public_store.get_file(main_filepath, 'wb') as main_file:
main_file.write(queued_file.read())
queue_store.delete_file(queued_filepath)
mg_globals.queue_store.delete_file(queued_filepath)
media_files_dict = entry.setdefault('media_files', {})
media_files_dict['thumb'] = thumb_filepath
media_files_dict['main'] = main_filepath
entry['state'] = u'processed'
entry.save()
# clean up workbench
mg_globals.workbench_manager.destroy_workbench(workbench)

42
mediagoblin/storage.py
View File

@ -16,6 +16,7 @@
import os
import re
import shutil
import urlparse
import uuid
@ -60,6 +61,9 @@ class StorageInterface(object):
StorageInterface.
"""
# Whether this file store is on the local filesystem.
local_storage = False
def __raise_not_implemented(self):
"""
Raise a warning about some component not implemented by a
@ -127,12 +131,43 @@ class StorageInterface(object):
else:
return filepath
def get_local_path(self, filepath):
"""
If this is a local_storage implementation, give us a link to
the local filesystem reference to this file.
>>> storage_handler.get_local_path(['foo', 'bar', 'baz.jpg'])
u'/path/to/mounting/foo/bar/baz.jpg'
"""
# Subclasses should override this method, if applicable.
self.__raise_not_implemented()
def copy_locally(self, filepath, dest_path):
"""
Copy this file locally.
A basic working method for this is provided that should
function both for local_storage systems and remote storge
systems, but if more efficient systems for copying locally
apply to your system, override this method with something more
appropriate.
"""
if self.local_storage:
shutil.copy(
self.get_local_path(filepath), dest_path)
else:
with self.get_file(filepath, 'rb') as source_file:
with file(dest_path, 'wb') as dest_file:
dest_file.write(source_file.read())
class BasicFileStorage(StorageInterface):
"""
Basic local filesystem implementation of storage API
"""
local_storage = True
def __init__(self, base_dir, base_url=None, **kwargs):
"""
Keyword arguments:
@ -177,6 +212,9 @@ class BasicFileStorage(StorageInterface):
self.base_url,
'/'.join(clean_listy_filepath(filepath)))
def get_local_path(self, filepath):
return self._resolve_filepath(filepath)
###########
# Utilities
@ -187,7 +225,7 @@ def clean_listy_filepath(listy_filepath):
Take a listy filepath (like ['dir1', 'dir2', 'filename.jpg']) and
clean out any nastiness from it.
For example:
>>> clean_listy_filepath([u'/dir1/', u'foo/../nasty', u'linooks.jpg'])
[u'dir1', u'foo_.._nasty', u'linooks.jpg']
@ -253,3 +291,5 @@ def storage_system_from_paste_config(paste_config, storage_prefix):
storage_class = util.import_component(storage_class)
return storage_class(**config_params)

45
mediagoblin/tests/test_storage.py
View File

@ -52,6 +52,11 @@ class FakeStorageSystem():
self.foobie = foobie
self.blech = blech
class FakeRemoteStorage(storage.BasicFileStorage):
# Theoretically despite this, all the methods should work but it
# should force copying to the workbench
local_storage = False
def test_storage_system_from_paste_config():
this_storage = storage.storage_system_from_paste_config(
@ -81,9 +86,12 @@ def test_storage_system_from_paste_config():
# Basic file storage tests
##########################
def get_tmp_filestorage(mount_url=None):
def get_tmp_filestorage(mount_url=None, fake_remote=False):
tmpdir = tempfile.mkdtemp()
this_storage = storage.BasicFileStorage(tmpdir, mount_url)
if fake_remote:
this_storage = FakeRemoteStorage(tmpdir, mount_url)
else:
this_storage = storage.BasicFileStorage(tmpdir, mount_url)
return tmpdir, this_storage
@ -214,3 +222,36 @@ def test_basic_storage_url_for_file():
['dir1', 'dir2', 'filename.txt'])
expected = 'http://media.example.org/ourmedia/dir1/dir2/filename.txt'
assert result == expected
def test_basic_storage_get_local_path():
tmpdir, this_storage = get_tmp_filestorage()
result = this_storage.get_local_path(
['dir1', 'dir2', 'filename.txt'])
expected = os.path.join(
tmpdir, 'dir1/dir2/filename.txt')
assert result == expected
def test_basic_storage_is_local():
tmpdir, this_storage = get_tmp_filestorage()
assert this_storage.local_storage is True
def test_basic_storage_copy_locally():
tmpdir, this_storage = get_tmp_filestorage()
dest_tmpdir = tempfile.mkdtemp()
filepath = ['dir1', 'dir2', 'ourfile.txt']
with this_storage.get_file(filepath, 'w') as our_file:
our_file.write('Testing this file')
new_file_dest = os.path.join(dest_tmpdir, 'file2.txt')
this_storage.copy_locally(filepath, new_file_dest)
assert file(new_file_dest).read() == 'Testing this file'

96
mediagoblin/tests/test_workbench.py Normal file
View File

@ -0,0 +1,96 @@
# GNU MediaGoblin -- federated, autonomous media hosting
# Copyright (C) 2011 Free Software Foundation, Inc
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import os
import tempfile
from nose.tools import assert_raises
from mediagoblin import workbench
from mediagoblin.tests.test_storage import get_tmp_filestorage
class TestWorkbench(object):
def setUp(self):
self.workbench_manager = workbench.WorkbenchManager(
os.path.join(tempfile.gettempdir(), u'mgoblin_workbench_testing'))
def test_create_workbench(self):
workbench = self.workbench_manager.create_workbench()
assert os.path.isdir(workbench)
assert workbench.startswith(self.workbench_manager.base_workbench_dir)
def test_destroy_workbench(self):
# kill a workbench
this_workbench = self.workbench_manager.create_workbench()
tmpfile = file(os.path.join(this_workbench, 'temp.txt'), 'w')
with tmpfile:
tmpfile.write('lollerskates')
assert os.path.exists(os.path.join(this_workbench, 'temp.txt'))
self.workbench_manager.destroy_workbench(this_workbench)
assert not os.path.exists(os.path.join(this_workbench, 'temp.txt'))
assert not os.path.exists(this_workbench)
# make sure we can't kill other stuff though
dont_kill_this = tempfile.mkdtemp()
assert_raises(
workbench.WorkbenchOutsideScope,
self.workbench_manager.destroy_workbench,
dont_kill_this)
def test_localized_file(self):
tmpdir, this_storage = get_tmp_filestorage()
this_workbench = self.workbench_manager.create_workbench()
# Write a brand new file
filepath = ['dir1', 'dir2', 'ourfile.txt']
with this_storage.get_file(filepath, 'w') as our_file:
our_file.write('Our file')
# with a local file storage
filename = self.workbench_manager.localized_file(
this_workbench, this_storage, filepath)
assert filename == os.path.join(
tmpdir, 'dir1/dir2/ourfile.txt')
# with a fake remote file storage
tmpdir, this_storage = get_tmp_filestorage(fake_remote=True)
# ... write a brand new file, again ;)
with this_storage.get_file(filepath, 'w') as our_file:
our_file.write('Our file')
filename = self.workbench_manager.localized_file(
this_workbench, this_storage, filepath)
assert filename == os.path.join(
this_workbench, 'ourfile.txt')
# fake remote file storage, filename_if_copying set
filename = self.workbench_manager.localized_file(
this_workbench, this_storage, filepath, 'thisfile')
assert filename == os.path.join(
this_workbench, 'thisfile.txt')
# fake remote file storage, filename_if_copying set,
# keep_extension_if_copying set to false
filename = self.workbench_manager.localized_file(
this_workbench, this_storage, filepath, 'thisfile.text', False)
assert filename == os.path.join(
this_workbench, 'thisfile.text')

135
mediagoblin/workbench.py Normal file
View File

@ -0,0 +1,135 @@
# GNU MediaGoblin -- federated, autonomous media hosting
# Copyright (C) 2011 Free Software Foundation, Inc
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import os
import shutil
import tempfile
DEFAULT_WORKBENCH_DIR = os.path.join(
tempfile.gettempdir(), u'mgoblin_workbench')
# Exception(s)
# ------------
class WorkbenchOutsideScope(Exception):
"""
Raised when a workbench is outside a WorkbenchManager scope.
"""
pass
# Actual workbench stuff
# ----------------------
class WorkbenchManager(object):
"""
A system for generating and destroying workbenches.
Workbenches are actually just subdirectories of a temporary storage space
for during the processing stage.
"""
def __init__(self, base_workbench_dir):
self.base_workbench_dir = os.path.abspath(base_workbench_dir)
if not os.path.exists(self.base_workbench_dir):
os.makedirs(self.base_workbench_dir)
def create_workbench(self):
"""
Create and return the path to a new workbench (directory).
"""
return tempfile.mkdtemp(dir=self.base_workbench_dir)
def destroy_workbench(self, workbench):
"""
Destroy this workbench! Deletes the directory and all its contents!
Makes sure the workbench actually belongs to this manager though.
"""
# just in case
workbench = os.path.abspath(workbench)
if not workbench.startswith(self.base_workbench_dir):
raise WorkbenchOutsideScope(
"Can't destroy workbench outside the base workbench dir")
shutil.rmtree(workbench)
def localized_file(self, workbench, storage, filepath,
filename_if_copying=None,
keep_extension_if_copying=True):
"""
Possibly localize the file from this storage system (for read-only
purposes, modifications should be written to a new file.).
If the file is already local, just return the absolute filename of that
local file. Otherwise, copy the file locally to the workbench, and
return the absolute path of the new file.
If it is copying locally, we might want to require a filename like
"source.jpg" to ensure that we won't conflict with other filenames in
our workbench... if that's the case, make sure filename_if_copying is
set to something like 'source.jpg'. Relatedly, if you set
keep_extension_if_copying, you don't have to set an extension on
filename_if_copying yourself, it'll be set for you (assuming such an
extension can be extacted from the filename in the filepath).
Returns:
localized_filename
Examples:
>>> wb_manager.localized_file(
... '/our/workbench/subdir', local_storage,
... ['path', 'to', 'foobar.jpg'])
u'/local/storage/path/to/foobar.jpg'
>>> wb_manager.localized_file(
... '/our/workbench/subdir', remote_storage,
... ['path', 'to', 'foobar.jpg'])
'/our/workbench/subdir/foobar.jpg'
>>> wb_manager.localized_file(
... '/our/workbench/subdir', remote_storage,
... ['path', 'to', 'foobar.jpg'], 'source.jpeg', False)
'/our/workbench/subdir/foobar.jpeg'
>>> wb_manager.localized_file(
... '/our/workbench/subdir', remote_storage,
... ['path', 'to', 'foobar.jpg'], 'source', True)
'/our/workbench/subdir/foobar.jpg'
"""
if storage.local_storage:
return storage.get_local_path(filepath)
else:
if filename_if_copying is None:
dest_filename = filepath[-1]
else:
orig_filename, orig_ext = os.path.splitext(filepath[-1])
if keep_extension_if_copying and orig_ext:
dest_filename = filename_if_copying + orig_ext
else:
dest_filename = filename_if_copying
full_dest_filename = os.path.join(
workbench, dest_filename)
# copy it over
storage.copy_locally(
filepath, full_dest_filename)
return full_dest_filename