heckyel/mediagoblin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Non-performance improvements to cloudfiles

Browse Source 
Improved documentation and added logging to cloudfiles
This commit is contained in:
Joar Wandborg 2012-08-05 22:52:30 +02:00
parent 8d9aa03fba
commit a07d052f0d
1 changed files with 27 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
mediagoblin/storage/cloudfiles.py
View File

@ -26,6 +26,9 @@ from mediagoblin.storage import StorageInterface, clean_listy_filepath
import cloudfiles import cloudfiles
import mimetypes import mimetypes
import logging
_log = logging.getLogger(__name__)
class CloudFilesStorage(StorageInterface): class CloudFilesStorage(StorageInterface):
@ -46,7 +49,7 @@ class CloudFilesStorage(StorageInterface):
mimetypes.add_type("video/webm", "webm") mimetypes.add_type("video/webm", "webm")
if not self.param_host: if not self.param_host:
print('No CloudFiles host URL specified, ' _log.info('No CloudFiles host URL specified, '
'defaulting to Rackspace US') 'defaulting to Rackspace US')
self.connection = cloudfiles.get_connection( self.connection = cloudfiles.get_connection(
@ -55,6 +58,10 @@ class CloudFilesStorage(StorageInterface):
servicenet=True if self.param_use_servicenet == 'true' or \ servicenet=True if self.param_use_servicenet == 'true' or \
self.param_use_servicenet == True else False) self.param_use_servicenet == True else False)
_log.debug('Connected to {0} (auth: {1})'.format(
self.connection.connection.host,
self.connection.auth.host))
if not self.param_container == \ if not self.param_container == \
self.connection.get_container(self.param_container): self.connection.get_container(self.param_container):
self.container = self.connection.create_container( self.container = self.connection.create_container(
@ -65,6 +72,9 @@ class CloudFilesStorage(StorageInterface):
self.container = self.connection.get_container( self.container = self.connection.get_container(
self.param_container) self.param_container)
_log.debug('Container: {0}'.format(
self.container.name))
self.container_uri = self.container.public_uri() self.container_uri = self.container.public_uri()
def _resolve_filepath(self, filepath): def _resolve_filepath(self, filepath):
@ -73,14 +83,14 @@ class CloudFilesStorage(StorageInterface):
def file_exists(self, filepath): def file_exists(self, filepath):
try: try:
self.container.get_object( self._resolve_filepath(filepath)) self.container.get_object(self._resolve_filepath(filepath))
return True return True
except cloudfiles.errors.NoSuchObject: except cloudfiles.errors.NoSuchObject:
return False return False
def get_file(self, filepath, *args, **kwargs): def get_file(self, filepath, *args, **kwargs):
""" """
- Doesn't care about the "mode" argument - Doesn't care about the "mode" argument.
""" """
try: try:
obj = self.container.get_object( obj = self.container.get_object(
@ -89,14 +99,15 @@ class CloudFilesStorage(StorageInterface):
obj = self.container.create_object( obj = self.container.create_object(
self._resolve_filepath(filepath)) self._resolve_filepath(filepath))
# Detect the mimetype ourselves, since some extensions (webm)
# may not be universally accepted as video/webm
mimetype = mimetypes.guess_type( mimetype = mimetypes.guess_type(
filepath[-1]) filepath[-1])
if mimetype: if mimetype:
# Set the mimetype on the CloudFiles object
obj.content_type = mimetype[0] obj.content_type = mimetype[0]
# this should finally fix the bug #429 obj.metadata = {'mime-type': mimetype[0]}
meta_data = {'mime-type' : mimetype[0]}
obj.metadata = meta_data
return CloudFilesStorageObjectWrapper(obj, *args, **kwargs) return CloudFilesStorageObjectWrapper(obj, *args, **kwargs)
@ -111,7 +122,6 @@ class CloudFilesStorage(StorageInterface):
finally: finally:
pass pass
def file_url(self, filepath): def file_url(self, filepath):
return '/'.join([ return '/'.join([
self.container_uri, self.container_uri,
@ -123,7 +133,7 @@ class CloudFilesStorageObjectWrapper():
Wrapper for python-cloudfiles's cloudfiles.storage_object.Object Wrapper for python-cloudfiles's cloudfiles.storage_object.Object
used to circumvent the mystic `medium.jpg` corruption issue, where used to circumvent the mystic `medium.jpg` corruption issue, where
we had both python-cloudfiles and PIL doing buffering on both we had both python-cloudfiles and PIL doing buffering on both
ends and that breaking things. ends and causing breakage.
This wrapper currently meets mediagoblin's needs for a public_store This wrapper currently meets mediagoblin's needs for a public_store
file-like object. file-like object.
@ -132,6 +142,8 @@ class CloudFilesStorageObjectWrapper():
self.storage_object = storage_object self.storage_object = storage_object
def read(self, *args, **kwargs): def read(self, *args, **kwargs):
_log.debug('Reading {0}'.format(
self.storage_object.name))
return self.storage_object.read(*args, **kwargs) return self.storage_object.read(*args, **kwargs)
def write(self, data, *args, **kwargs): def write(self, data, *args, **kwargs):
@ -146,11 +158,18 @@ class CloudFilesStorageObjectWrapper():
However if we should need it it would be easy implement. However if we should need it it would be easy implement.
""" """
if self.storage_object.size and type(data) == str: if self.storage_object.size and type(data) == str:
_log.debug('{0} is > 0 in size, appending data'.format(
self.storage_object.name))
data = self.read() + data data = self.read() + data
_log.debug('Writing {0}'.format(
self.storage_object.name))
self.storage_object.write(data, *args, **kwargs) self.storage_object.write(data, *args, **kwargs)
def close(self): def close(self):
"""
Not implemented.
"""
pass pass
def __enter__(self): def __enter__(self):