Fix bug 647

Reading a file into memory resulted in depletion of memory. Now files
are read and written lazily, by chunks.

mediagoblin/storage/cloudfiles.py

@@ -193,27 +193,6 @@ class CloudFilesStorageObjectWrapper():
         return self.storage_object.read(*args, **kwargs)
 
     def write(self, data, *args, **kwargs):
-        """
-        write data to the cloudfiles storage object
-
-        The original motivation for this wrapper is to ensure
-        that buffered writing to a cloudfiles storage object does not overwrite
-        any preexisting data.
-
-        Currently this method does not support any write modes except "append".
-        However if we should need it it would be easy implement.
-        """
-        _log.warn(
-            '{0}.write() has bad performance! Use .send instead for now'\
-            .format(self.__class__.__name__))
-
-        if self.storage_object.size and type(data) == str:
-            _log.debug('{0} is > 0 in size, appending data'.format(
-                self.storage_object.name))
-            data = self.read() + data
-
-        _log.debug('Writing {0}'.format(
-            self.storage_object.name))
         self.storage_object.write(data, *args, **kwargs)
 
     def send(self, *args, **kw):

mediagoblin/storage/filestorage.py

@@ -14,6 +14,7 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+import io
 import os
 import shutil
 
@@ -24,6 +25,15 @@ from mediagoblin.storage import (
     clean_listy_filepath,
     NoWebServing)
 
+class FileObjectAwareFile(io.FileIO):
+    def write(self, data):
+        if hasattr(data, 'read'):
+            # We can call data.read(). It means that the data is a file-like
+            # object, which should be saved RAM-friendly way
+            shutil.copyfileobj(data, self)
+        else:
+            super(FileObjectAwareFile, self).write(data)
+
 
 class BasicFileStorage(StorageInterface):
     """
@@ -60,7 +70,7 @@ class BasicFileStorage(StorageInterface):
                 os.makedirs(directory)
 
         # Grab and return the file in the mode specified
-        return open(self._resolve_filepath(filepath), mode)
+        return FileObjectAwareFile(self._resolve_filepath(filepath), mode)
 
     def delete_file(self, filepath):
         """Delete file at filepath

mediagoblin/submit/lib.py

@@ -157,7 +157,7 @@ def submit_media(mg_app, user, submitted_file, filename,
     queue_file = prepare_queue_task(mg_app, entry, filename)
 
     with queue_file:
-        queue_file.write(submitted_file.read())
+        queue_file.write(submitted_file)
 
     # Get file size and round to 2 decimal places
     file_size = mg_app.queue_store.get_file_size(