Fix bug 647
Reading a file into memory resulted in depletion of memory. Now files are read and written lazily, by chunks.
This commit is contained in:
parent
33055472f9
commit
2b4c339de6
@ -193,27 +193,6 @@ class CloudFilesStorageObjectWrapper():
|
|||||||
return self.storage_object.read(*args, **kwargs)
|
return self.storage_object.read(*args, **kwargs)
|
||||||
|
|
||||||
def write(self, data, *args, **kwargs):
|
def write(self, data, *args, **kwargs):
|
||||||
"""
|
|
||||||
write data to the cloudfiles storage object
|
|
||||||
|
|
||||||
The original motivation for this wrapper is to ensure
|
|
||||||
that buffered writing to a cloudfiles storage object does not overwrite
|
|
||||||
any preexisting data.
|
|
||||||
|
|
||||||
Currently this method does not support any write modes except "append".
|
|
||||||
However if we should need it it would be easy implement.
|
|
||||||
"""
|
|
||||||
_log.warn(
|
|
||||||
'{0}.write() has bad performance! Use .send instead for now'\
|
|
||||||
.format(self.__class__.__name__))
|
|
||||||
|
|
||||||
if self.storage_object.size and type(data) == str:
|
|
||||||
_log.debug('{0} is > 0 in size, appending data'.format(
|
|
||||||
self.storage_object.name))
|
|
||||||
data = self.read() + data
|
|
||||||
|
|
||||||
_log.debug('Writing {0}'.format(
|
|
||||||
self.storage_object.name))
|
|
||||||
self.storage_object.write(data, *args, **kwargs)
|
self.storage_object.write(data, *args, **kwargs)
|
||||||
|
|
||||||
def send(self, *args, **kw):
|
def send(self, *args, **kw):
|
||||||
|
@ -14,6 +14,7 @@
|
|||||||
# You should have received a copy of the GNU Affero General Public License
|
# You should have received a copy of the GNU Affero General Public License
|
||||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
|
import io
|
||||||
import os
|
import os
|
||||||
import shutil
|
import shutil
|
||||||
|
|
||||||
@ -24,6 +25,15 @@ from mediagoblin.storage import (
|
|||||||
clean_listy_filepath,
|
clean_listy_filepath,
|
||||||
NoWebServing)
|
NoWebServing)
|
||||||
|
|
||||||
|
class FileObjectAwareFile(io.FileIO):
|
||||||
|
def write(self, data):
|
||||||
|
if hasattr(data, 'read'):
|
||||||
|
# We can call data.read(). It means that the data is a file-like
|
||||||
|
# object, which should be saved RAM-friendly way
|
||||||
|
shutil.copyfileobj(data, self)
|
||||||
|
else:
|
||||||
|
super(FileObjectAwareFile, self).write(data)
|
||||||
|
|
||||||
|
|
||||||
class BasicFileStorage(StorageInterface):
|
class BasicFileStorage(StorageInterface):
|
||||||
"""
|
"""
|
||||||
@ -60,7 +70,7 @@ class BasicFileStorage(StorageInterface):
|
|||||||
os.makedirs(directory)
|
os.makedirs(directory)
|
||||||
|
|
||||||
# Grab and return the file in the mode specified
|
# Grab and return the file in the mode specified
|
||||||
return open(self._resolve_filepath(filepath), mode)
|
return FileObjectAwareFile(self._resolve_filepath(filepath), mode)
|
||||||
|
|
||||||
def delete_file(self, filepath):
|
def delete_file(self, filepath):
|
||||||
"""Delete file at filepath
|
"""Delete file at filepath
|
||||||
|
@ -157,7 +157,7 @@ def submit_media(mg_app, user, submitted_file, filename,
|
|||||||
queue_file = prepare_queue_task(mg_app, entry, filename)
|
queue_file = prepare_queue_task(mg_app, entry, filename)
|
||||||
|
|
||||||
with queue_file:
|
with queue_file:
|
||||||
queue_file.write(submitted_file.read())
|
queue_file.write(submitted_file)
|
||||||
|
|
||||||
# Get file size and round to 2 decimal places
|
# Get file size and round to 2 decimal places
|
||||||
file_size = mg_app.queue_store.get_file_size(
|
file_size = mg_app.queue_store.get_file_size(
|
||||||
|
Loading…
x
Reference in New Issue
Block a user