heckyel/mediagoblin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Do not read complete videos in RAM (#419)

Browse Source 
We were reading the complete "medium" "thumbnail" and "original"
in RAM via dst.write(src.read()). Just call the appropriate storage
methods copy_local_to_storage which are responsible for streaming
local files efficiently.

The efficiency of this patch depends on the separate branch that
actually implements chunked copying for Storage().copy_local_to_storage()
This commit is contained in:
Sebastian Spaeth 2012-12-19 15:03:43 +01:00
parent 9408938bcc
commit c00e18fe49
1 changed files with 21 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
mediagoblin/media_types/video/processing.py
View File

@ -75,9 +75,8 @@ def process_video(entry, workbench=None):
thumbnail_filepath = create_pub_filepath( thumbnail_filepath = create_pub_filepath(
entry, name_builder.fill('{basename}.thumbnail.jpg')) entry, name_builder.fill('{basename}.thumbnail.jpg'))
# Create a temporary file for the video destination # Create a temporary file for the video destination (cleaned up with workbench)
tmp_dst = NamedTemporaryFile(dir=workbench.dir) tmp_dst = NamedTemporaryFile(dir=workbench.dir, delete=False)
with tmp_dst: with tmp_dst:
# Transcode queued file to a VP8/vorbis file that fits in a 640x640 square # Transcode queued file to a VP8/vorbis file that fits in a 640x640 square
progress_callback = ProgressCallback(entry) progress_callback = ProgressCallback(entry)
@ -90,9 +89,7 @@ def process_video(entry, workbench=None):
# Push transcoded video to public storage # Push transcoded video to public storage
_log.debug('Saving medium...') _log.debug('Saving medium...')
# TODO (#419, we read everything in RAM here!) mgg.public_store.copy_local_to_storage(tmp_dst.name, medium_filepath)
mgg.public_store.get_file(medium_filepath, 'wb').write(
tmp_dst.read())
_log.debug('Saved medium') _log.debug('Saved medium')
entry.media_files['webm_640'] = medium_filepath entry.media_files['webm_640'] = medium_filepath
@ -102,8 +99,8 @@ def process_video(entry, workbench=None):
width=transcoder.dst_data.videowidth, width=transcoder.dst_data.videowidth,
height=transcoder.dst_data.videoheight) height=transcoder.dst_data.videoheight)
# Create a temporary file for the video thumbnail # Temporary file for the video thumbnail (cleaned up with workbench)
tmp_thumb = NamedTemporaryFile(dir=workbench.dir, suffix='.jpg') tmp_thumb = NamedTemporaryFile(dir=workbench.dir, suffix='.jpg', delete=False)
with tmp_thumb: with tmp_thumb:
# Create a thumbnail.jpg that fits in a 180x180 square # Create a thumbnail.jpg that fits in a 180x180 square
@ -114,31 +111,14 @@ def process_video(entry, workbench=None):
# Push the thumbnail to public storage # Push the thumbnail to public storage
_log.debug('Saving thumbnail...') _log.debug('Saving thumbnail...')
mgg.public_store.get_file(thumbnail_filepath, 'wb').write( mgg.public_store.copy_local_to_storage(tmp_thumb.name, thumbnail_filepath)
tmp_thumb.read())
_log.debug('Saved thumbnail')
entry.media_files['thumb'] = thumbnail_filepath entry.media_files['thumb'] = thumbnail_filepath
if video_config['keep_original']: if video_config['keep_original']:
# Push original file to public storage # Push original file to public storage
queued_file = file(queued_filename, 'rb')
with queued_file:
original_filepath = create_pub_filepath(
entry,
queued_filepath[-1])
with mgg.public_store.get_file(original_filepath, 'wb') as \
original_file:
_log.debug('Saving original...') _log.debug('Saving original...')
# TODO (#419, we read everything in RAM here!) original_filepath = create_pub_filepath(entry, queued_filepath[-1])
original_file.write(queued_file.read()) mgg.public_store.copy_local_to_storage(queued_filename, original_filepath)
_log.debug('Saved original')
entry.media_files['original'] = original_filepath entry.media_files['original'] = original_filepath
mgg.queue_store.delete_file(queued_filepath) mgg.queue_store.delete_file(queued_filepath)
# clean up workbench
workbench.destroy_self()