From e86d4f5d51dfdd42e2fe91e268bd0f335732908a Mon Sep 17 00:00:00 2001 From: Joar Wandborg Date: Sun, 7 Aug 2011 02:58:03 +0200 Subject: [PATCH 01/12] Feature #298 - Environment tarball * Added command hooks for gmg_commands.import_export * Added (DANGEROUSLY BLEEDING EDGE) gmg_commands.import_export --- mediagoblin/gmg_commands/__init__.py | 8 +++ mediagoblin/gmg_commands/import_export.py | 87 +++++++++++++++++++++++ 2 files changed, 95 insertions(+) create mode 100644 mediagoblin/gmg_commands/import_export.py diff --git a/mediagoblin/gmg_commands/__init__.py b/mediagoblin/gmg_commands/__init__.py index 921f0430..8226fd0e 100644 --- a/mediagoblin/gmg_commands/__init__.py +++ b/mediagoblin/gmg_commands/__init__.py @@ -44,6 +44,14 @@ SUBCOMMAND_MAP = { 'setup': 'mediagoblin.gmg_commands.wipealldata:wipe_parser_setup', 'func': 'mediagoblin.gmg_commands.wipealldata:wipe', 'help': 'Wipes **all** the data for this MediaGoblin instance'}, + 'env_export': { + 'setup': 'mediagoblin.gmg_commands.import_export:import_export_parse_setup', + 'func': 'mediagoblin.gmg_commands.import_export:env_export', + 'help': 'Exports the data for this MediaGoblin instance'}, + 'env_import': { + 'setup': 'mediagoblin.gmg_commands.import_export:import_export_parse_setup', + 'func': 'mediagoblin.gmg_commands.import_export:env_import', + 'help': 'Exports the data for this MediaGoblin instance'}, } diff --git a/mediagoblin/gmg_commands/import_export.py b/mediagoblin/gmg_commands/import_export.py new file mode 100644 index 00000000..c0ac444b --- /dev/null +++ b/mediagoblin/gmg_commands/import_export.py @@ -0,0 +1,87 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011 Free Software Foundation, Inc +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from mediagoblin.gmg_commands import util as commands_util +from mediagoblin import mg_globals +from mediagoblin.db import util as db_util +from mediagoblin.db.open import setup_connection_and_db_from_config +from mediagoblin.init.config import read_mediagoblin_config + +import shlex +import tarfile +import subprocess +import os.path + +def import_export_parse_setup(subparser): + # TODO: Add default + subparser.add_argument( + 'tar_file') + subparser.add_argument( + '-cf', '--conf_file', default='mediagoblin.ini', + help='Config file used to set up environment') + subparser.add_argument( + '--mongodump_cache', default='mongodump', + help='mongodump cache directory') + subparser.add_argument( + '--mongodump_path', default='mongodump', + help='mongodump binary') + +def _export_database(db, args): + print "\n== Exporting database ==\n" + + command = '{mongodump_path} -d {database} -o {mongodump_cache}'.format( + mongodump_path=args.mongodump_path, + database=db.name, + mongodump_cache=args.mongodump_cache) + + p = subprocess.Popen( + shlex.split(command)) + + p.wait() + + print "\n== Database exported ==\n" + +def _import_database(db, args): + pass + +def env_import(args): + config, validation_result = read_mediagoblin_config(args.conf_file) + connection, db = setup_connection_and_db_from_config( + config['mediagoblin'], use_pymongo=True) + +def env_export(args): + config, validation_result = read_mediagoblin_config(args.conf_file) + connection, db = setup_connection_and_db_from_config( + config['mediagoblin'], use_pymongo=True) + + if os.path.exists(args.tar_file): + overwrite = raw_input( + 'The output file already exists. ' + 'Are you **SURE** you want to overwrite it? ' + '(yes/no)> ') + if not overwrite == 'yes': + print "Aborting." + return + + tf = tarfile.open( + args.tar_file, + mode='w|gz') + + + _export_database(db, args) + + tf.add(args.mongodump_cache) + From 224813d28cc60aa7ec1ead34ea028713a2d61c74 Mon Sep 17 00:00:00 2001 From: Joar Wandborg Date: Sun, 7 Aug 2011 05:00:46 +0200 Subject: [PATCH 02/12] Feature #298 * Added some minor things to gmg_commands.import_export --- mediagoblin/gmg_commands/import_export.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/mediagoblin/gmg_commands/import_export.py b/mediagoblin/gmg_commands/import_export.py index c0ac444b..aa3fe1a1 100644 --- a/mediagoblin/gmg_commands/import_export.py +++ b/mediagoblin/gmg_commands/import_export.py @@ -38,6 +38,12 @@ def import_export_parse_setup(subparser): subparser.add_argument( '--mongodump_path', default='mongodump', help='mongodump binary') + subparser.add_argument( + '--mongorestore_path', default='mongorestore', + help='mongorestore binary') + subparser.add_argument( + '--extract_path', default='/tmp/mediagoblin-import', + help='the directory to which the tarball should be extracted temporarily') def _export_database(db, args): print "\n== Exporting database ==\n" @@ -55,13 +61,22 @@ def _export_database(db, args): print "\n== Database exported ==\n" def _import_database(db, args): - pass + command = '{mongorestore_path} -d {database} -o {mongodump_cache}'.format( + mongorestore_path=args.mongorestore_path, + database=db.name, + mongodump_cache=args.mongodump_cache) def env_import(args): config, validation_result = read_mediagoblin_config(args.conf_file) connection, db = setup_connection_and_db_from_config( config['mediagoblin'], use_pymongo=True) + tf = tarfile.open( + args.tar_file, + mode='r|gz') + + tf.extractall(args.extract_path) + def env_export(args): config, validation_result = read_mediagoblin_config(args.conf_file) connection, db = setup_connection_and_db_from_config( From 9188f3afe24a52fff3262360a0896cc521546fa8 Mon Sep 17 00:00:00 2001 From: Joar Wandborg Date: Mon, 8 Aug 2011 03:47:17 +0200 Subject: [PATCH 03/12] Feature #298 - Changed some defaults in gmg_commands.import_export --- mediagoblin/gmg_commands/import_export.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/mediagoblin/gmg_commands/import_export.py b/mediagoblin/gmg_commands/import_export.py index aa3fe1a1..c05a48d9 100644 --- a/mediagoblin/gmg_commands/import_export.py +++ b/mediagoblin/gmg_commands/import_export.py @@ -19,6 +19,7 @@ from mediagoblin import mg_globals from mediagoblin.db import util as db_util from mediagoblin.db.open import setup_connection_and_db_from_config from mediagoblin.init.config import read_mediagoblin_config +from mediagoblin import util as mg_util import shlex import tarfile @@ -33,7 +34,7 @@ def import_export_parse_setup(subparser): '-cf', '--conf_file', default='mediagoblin.ini', help='Config file used to set up environment') subparser.add_argument( - '--mongodump_cache', default='mongodump', + '--mongodump_cache', default='/tmp/mediagoblin/mongodump', help='mongodump cache directory') subparser.add_argument( '--mongodump_path', default='mongodump', @@ -42,12 +43,15 @@ def import_export_parse_setup(subparser): '--mongorestore_path', default='mongorestore', help='mongorestore binary') subparser.add_argument( - '--extract_path', default='/tmp/mediagoblin-import', + '--extract_path', default='/tmp/mediagoblin/import', help='the directory to which the tarball should be extracted temporarily') + subparser.add_argument( + '--media_cache_path', default='/tmp/mediagoblin/mediaentries', + help='') def _export_database(db, args): print "\n== Exporting database ==\n" - + command = '{mongodump_path} -d {database} -o {mongodump_cache}'.format( mongodump_path=args.mongodump_path, database=db.name, @@ -60,6 +64,13 @@ def _export_database(db, args): print "\n== Database exported ==\n" +def _export_media(db, args): + for entry in db.media_entries.find(): + storage = mg_util.import_component( + 'mediagoblin.storage:BasicFileStorage')() + print(storage.get_file(entry['media_files']['medium'])) + print(entry) + def _import_database(db, args): command = '{mongorestore_path} -d {database} -o {mongodump_cache}'.format( mongorestore_path=args.mongorestore_path, @@ -74,7 +85,7 @@ def env_import(args): tf = tarfile.open( args.tar_file, mode='r|gz') - + tf.extractall(args.extract_path) def env_export(args): @@ -91,6 +102,8 @@ def env_export(args): print "Aborting." return + _export_media(db, args) + tf = tarfile.open( args.tar_file, mode='w|gz') From 2a233ae33f9ed370df4177d4abb2ddd345d85cbc Mon Sep 17 00:00:00 2001 From: Joar Wandborg Date: Tue, 9 Aug 2011 03:09:42 +0200 Subject: [PATCH 04/12] Feature 298 - Create environment tarball Saving changes. --- mediagoblin/gmg_commands/import_export.py | 109 +++++++++++++++++----- 1 file changed, 88 insertions(+), 21 deletions(-) diff --git a/mediagoblin/gmg_commands/import_export.py b/mediagoblin/gmg_commands/import_export.py index c05a48d9..2c626da1 100644 --- a/mediagoblin/gmg_commands/import_export.py +++ b/mediagoblin/gmg_commands/import_export.py @@ -20,11 +20,15 @@ from mediagoblin.db import util as db_util from mediagoblin.db.open import setup_connection_and_db_from_config from mediagoblin.init.config import read_mediagoblin_config from mediagoblin import util as mg_util +from mediagoblin.storage import BasicFileStorage +from mediagoblin.init import setup_storage, setup_global_and_app_config import shlex import tarfile import subprocess import os.path +import os +import re def import_export_parse_setup(subparser): # TODO: Add default @@ -33,9 +37,6 @@ def import_export_parse_setup(subparser): subparser.add_argument( '-cf', '--conf_file', default='mediagoblin.ini', help='Config file used to set up environment') - subparser.add_argument( - '--mongodump_cache', default='/tmp/mediagoblin/mongodump', - help='mongodump cache directory') subparser.add_argument( '--mongodump_path', default='mongodump', help='mongodump binary') @@ -43,10 +44,7 @@ def import_export_parse_setup(subparser): '--mongorestore_path', default='mongorestore', help='mongorestore binary') subparser.add_argument( - '--extract_path', default='/tmp/mediagoblin/import', - help='the directory to which the tarball should be extracted temporarily') - subparser.add_argument( - '--media_cache_path', default='/tmp/mediagoblin/mediaentries', + '--cache_path', default='/tmp/mediagoblin/', help='') def _export_database(db, args): @@ -55,7 +53,7 @@ def _export_database(db, args): command = '{mongodump_path} -d {database} -o {mongodump_cache}'.format( mongodump_path=args.mongodump_path, database=db.name, - mongodump_cache=args.mongodump_cache) + mongodump_cache=args._cache_path['database']) p = subprocess.Popen( shlex.split(command)) @@ -65,19 +63,38 @@ def _export_database(db, args): print "\n== Database exported ==\n" def _export_media(db, args): + + print "\n== Exporting media ==\n" + + media_cache = BasicFileStorage( + args._cache_path['media']) + for entry in db.media_entries.find(): - storage = mg_util.import_component( - 'mediagoblin.storage:BasicFileStorage')() - print(storage.get_file(entry['media_files']['medium'])) + for name, path in entry['media_files'].items(): + mc_file = media_cache.get_file(path, mode='wb') + mc_file.write( + mg_globals.public_store.get_file(path, mode='rb').read()) + + print(mc_file) print(entry) + queue_cache = BasicFileStorage( + args._cache_path['queue']) + + qc_file = queue_cache.get_file(entry['queued_media_file'], mode='wb') + qc_file.write( + mg_globals.queue_store.get_file(entry['queued_media_file'], mode='rb').read()) + print(qc_file) + + print "\n== Media exported ==\n" + def _import_database(db, args): command = '{mongorestore_path} -d {database} -o {mongodump_cache}'.format( mongorestore_path=args.mongorestore_path, database=db.name, mongodump_cache=args.mongodump_cache) -def env_import(args): +def env_import(args): config, validation_result = read_mediagoblin_config(args.conf_file) connection, db = setup_connection_and_db_from_config( config['mediagoblin'], use_pymongo=True) @@ -88,7 +105,63 @@ def env_import(args): tf.extractall(args.extract_path) +def _setup_paths(args): + args._cache_path = dict() + PATH_MAP = { + 'media': 'media', + 'queue': 'queue', + 'database': 'database'} + + for key, val in PATH_MAP.items(): + args._cache_path[key] = os.path.join(args.cache_path, val) + + return args + +def _create_archive(args): + print "\n== Compressing to archive ==\n" + tf = tarfile.open( + args.tar_file, + mode='w|gz') + with tf: + for root, dirs, files in os.walk(args.cache_path): + print root, dirs, files + + everything = [] + everything.extend(dirs) + everything.extend(files) + + print everything + + for d in everything: + directory_path = os.path.join(root, d) + virtual_path = os.path.join( + root.replace(args.cache_path, 'mediagoblin-data/'), d) + + # print 'dir', directory_path, '\n', 'vir', virtual_path + + tarinfo = tf.gettarinfo( + directory_path, + arcname=virtual_path) + + tf.addfile(tarinfo) + + print 'added ', d + + ''' + mg_data = tf.gettarinfo( + args.cache_path, + arcname='mediagoblin-data') + + tf.addfile(mg_data) + ''' + print "\n== Archiving done ==\n" + def env_export(args): + args = _setup_paths(args) + + setup_global_and_app_config(args.conf_file) + setup_storage() + config, validation_result = read_mediagoblin_config(args.conf_file) connection, db = setup_connection_and_db_from_config( config['mediagoblin'], use_pymongo=True) @@ -102,14 +175,8 @@ def env_export(args): print "Aborting." return + _export_database(db, args) + _export_media(db, args) - tf = tarfile.open( - args.tar_file, - mode='w|gz') - - - _export_database(db, args) - - tf.add(args.mongodump_cache) - + _create_archive(args) From 7219983f8e017fe9163cbe5e28bf6314d91aa7a8 Mon Sep 17 00:00:00 2001 From: Joar Wandborg Date: Fri, 12 Aug 2011 02:13:58 +0200 Subject: [PATCH 05/12] Feature #298 - Create environment tarball * It's now possible to import/export your environment from/to a tarball. ./bin/gmg env_export [ -c mediagoblin_local.ini ] test.tar and ./bin/gmg env_import [ -c mediagoblin_local.ini ] test.tar --- mediagoblin/gmg_commands/import_export.py | 196 +++++++++++++--------- 1 file changed, 121 insertions(+), 75 deletions(-) diff --git a/mediagoblin/gmg_commands/import_export.py b/mediagoblin/gmg_commands/import_export.py index 2c626da1..56b3913d 100644 --- a/mediagoblin/gmg_commands/import_export.py +++ b/mediagoblin/gmg_commands/import_export.py @@ -24,11 +24,14 @@ from mediagoblin.storage import BasicFileStorage from mediagoblin.init import setup_storage, setup_global_and_app_config import shlex +import shutil import tarfile import subprocess import os.path import os import re +import sys + def import_export_parse_setup(subparser): # TODO: Add default @@ -47,28 +50,33 @@ def import_export_parse_setup(subparser): '--cache_path', default='/tmp/mediagoblin/', help='') + def _export_database(db, args): print "\n== Exporting database ==\n" - + command = '{mongodump_path} -d {database} -o {mongodump_cache}'.format( mongodump_path=args.mongodump_path, database=db.name, mongodump_cache=args._cache_path['database']) - + p = subprocess.Popen( shlex.split(command)) - + p.wait() print "\n== Database exported ==\n" + def _export_media(db, args): - print "\n== Exporting media ==\n" - + media_cache = BasicFileStorage( args._cache_path['media']) + # TODO: Add export of queue files + queue_cache = BasicFileStorage( + args._cache_path['queue']) + for entry in db.media_entries.find(): for name, path in entry['media_files'].items(): mc_file = media_cache.get_file(path, mode='wb') @@ -78,85 +86,54 @@ def _export_media(db, args): print(mc_file) print(entry) + print "\n== Media exported ==\n" + + +def _import_media(db, args): + """ + Import media files + + Must be called after _import_database() + """ + print "\n== Importing media ==\n" + + media_cache = BasicFileStorage( + args._cache_path['media']) + + # TODO: Add import of queue files queue_cache = BasicFileStorage( args._cache_path['queue']) - qc_file = queue_cache.get_file(entry['queued_media_file'], mode='wb') - qc_file.write( - mg_globals.queue_store.get_file(entry['queued_media_file'], mode='rb').read()) - print(qc_file) + for entry in db.media_entries.find(): + for name, path in entry['media_files'].items(): + media_file = mg_globals.public_store.get_file(path, mode='wb') + media_file.write( + media_cache.get_file(path, mode='rb').read()) + + print(media_file) + print(entry) + + print "\n== Media imported ==\n" - print "\n== Media exported ==\n" def _import_database(db, args): - command = '{mongorestore_path} -d {database} -o {mongodump_cache}'.format( + print "\n== Importing database ==\n" + command = '{mongorestore_path} -d {database}' + '{backup_dir}/{database}'.format( mongorestore_path=args.mongorestore_path, database=db.name, - mongodump_cache=args.mongodump_cache) + backup_dir=args._cache_path['database']) + + print command + + p = subprocess.Popen( + shlex.split(command)) + + p.wait() + def env_import(args): - config, validation_result = read_mediagoblin_config(args.conf_file) - connection, db = setup_connection_and_db_from_config( - config['mediagoblin'], use_pymongo=True) - - tf = tarfile.open( - args.tar_file, - mode='r|gz') - - tf.extractall(args.extract_path) - -def _setup_paths(args): - args._cache_path = dict() - PATH_MAP = { - 'media': 'media', - 'queue': 'queue', - 'database': 'database'} - - for key, val in PATH_MAP.items(): - args._cache_path[key] = os.path.join(args.cache_path, val) - - return args - -def _create_archive(args): - print "\n== Compressing to archive ==\n" - tf = tarfile.open( - args.tar_file, - mode='w|gz') - with tf: - for root, dirs, files in os.walk(args.cache_path): - print root, dirs, files - - everything = [] - everything.extend(dirs) - everything.extend(files) - - print everything - - for d in everything: - directory_path = os.path.join(root, d) - virtual_path = os.path.join( - root.replace(args.cache_path, 'mediagoblin-data/'), d) - - # print 'dir', directory_path, '\n', 'vir', virtual_path - - tarinfo = tf.gettarinfo( - directory_path, - arcname=virtual_path) - - tf.addfile(tarinfo) - - print 'added ', d - - ''' - mg_data = tf.gettarinfo( - args.cache_path, - arcname='mediagoblin-data') - - tf.addfile(mg_data) - ''' - print "\n== Archiving done ==\n" - -def env_export(args): + args.cache_path += 'mediagoblin-data' args = _setup_paths(args) setup_global_and_app_config(args.conf_file) @@ -166,6 +143,49 @@ def env_export(args): connection, db = setup_connection_and_db_from_config( config['mediagoblin'], use_pymongo=True) + tf = tarfile.open( + args.tar_file, + mode='r|gz') + + tf.extractall(args.cache_path) + + # Import database from extracted data + _import_database(db, args) + + _import_media(db, args) + + +def _setup_paths(args): + args._cache_path = dict() + PATH_MAP = { + 'media': 'media', + 'queue': 'queue', + 'database': 'database'} + + for key, val in PATH_MAP.items(): + args._cache_path[key] = os.path.join(args.cache_path, val) + + return args + + +def _create_archive(args): + print "\n== Compressing to archive ==\n" + + tf = tarfile.open( + args.tar_file, + mode='w|gz') + + with tf: + tf.add(args.cache_path, 'mediagoblin-data/') + + print "\n== Archiving done ==\n" + + +def _clean(args): + shutil.rmtree(args.cache_path) + + +def _check(args): if os.path.exists(args.tar_file): overwrite = raw_input( 'The output file already exists. ' @@ -173,10 +193,36 @@ def env_export(args): '(yes/no)> ') if not overwrite == 'yes': print "Aborting." - return + + return False + + if os.path.exists(args.cache_path): + print 'The cache directory must not exist before you run this script' + print 'Cache directory: ', args.cache_path + + return False + + return True + + +def env_export(args): + args = _setup_paths(args) + + if not _check(args): + print "\n== Checks did not pass, exiting ==\n" + sys.exit(0) + + setup_global_and_app_config(args.conf_file) + setup_storage() + + config, validation_result = read_mediagoblin_config(args.conf_file) + connection, db = setup_connection_and_db_from_config( + config['mediagoblin'], use_pymongo=True) _export_database(db, args) _export_media(db, args) _create_archive(args) + + _clean(args) From 8f12c9b24cb6fef5d7cc332e5e5e8d5587ba38e0 Mon Sep 17 00:00:00 2001 From: Joar Wandborg Date: Fri, 12 Aug 2011 17:04:34 +0200 Subject: [PATCH 06/12] Feature #298 - Create environment tarball * Reviewed the code and fixed some bugs --- mediagoblin/gmg_commands/import_export.py | 130 +++++++++++++--------- 1 file changed, 78 insertions(+), 52 deletions(-) diff --git a/mediagoblin/gmg_commands/import_export.py b/mediagoblin/gmg_commands/import_export.py index 56b3913d..f6651327 100644 --- a/mediagoblin/gmg_commands/import_export.py +++ b/mediagoblin/gmg_commands/import_export.py @@ -51,44 +51,6 @@ def import_export_parse_setup(subparser): help='') -def _export_database(db, args): - print "\n== Exporting database ==\n" - - command = '{mongodump_path} -d {database} -o {mongodump_cache}'.format( - mongodump_path=args.mongodump_path, - database=db.name, - mongodump_cache=args._cache_path['database']) - - p = subprocess.Popen( - shlex.split(command)) - - p.wait() - - print "\n== Database exported ==\n" - - -def _export_media(db, args): - print "\n== Exporting media ==\n" - - media_cache = BasicFileStorage( - args._cache_path['media']) - - # TODO: Add export of queue files - queue_cache = BasicFileStorage( - args._cache_path['queue']) - - for entry in db.media_entries.find(): - for name, path in entry['media_files'].items(): - mc_file = media_cache.get_file(path, mode='wb') - mc_file.write( - mg_globals.public_store.get_file(path, mode='rb').read()) - - print(mc_file) - print(entry) - - print "\n== Media exported ==\n" - - def _import_media(db, args): """ Import media files @@ -117,26 +79,31 @@ def _import_media(db, args): def _import_database(db, args): + """ + Restore mongo database from ___.bson files + """ print "\n== Importing database ==\n" - command = '{mongorestore_path} -d {database}' - '{backup_dir}/{database}'.format( - mongorestore_path=args.mongorestore_path, - database=db.name, - backup_dir=args._cache_path['database']) - print command - - p = subprocess.Popen( - shlex.split(command)) + p = subprocess.Popen([ + args.mongorestore_path, + '-d', db.name, + os.path.join(args._cache_path['database'], db.name)]) + + print p p.wait() + print "\n== Database imported ==\n" + def env_import(args): - args.cache_path += 'mediagoblin-data' - args = _setup_paths(args) - + """ + Restore mongo database and media files from a tar archive + """ + # args.cache_path += 'mediagoblin-data' setup_global_and_app_config(args.conf_file) + + # Creates mg_globals.public_store and mg_globals.queue_store setup_storage() config, validation_result = read_mediagoblin_config(args.conf_file) @@ -149,13 +116,20 @@ def env_import(args): tf.extractall(args.cache_path) + args.cache_path += 'mediagoblin-data' + args = _setup_paths(args) + # Import database from extracted data _import_database(db, args) _import_media(db, args) + # _clean(args) def _setup_paths(args): + """ + Populate ``args`` variable with cache subpaths + """ args._cache_path = dict() PATH_MAP = { 'media': 'media', @@ -169,6 +143,9 @@ def _setup_paths(args): def _create_archive(args): + """ + Create the tar archive + """ print "\n== Compressing to archive ==\n" tf = tarfile.open( @@ -182,10 +159,16 @@ def _create_archive(args): def _clean(args): + """ + Remove cache directory + """ shutil.rmtree(args.cache_path) -def _check(args): +def _export_check(args): + """ + Run security checks for export command + """ if os.path.exists(args.tar_file): overwrite = raw_input( 'The output file already exists. ' @@ -205,10 +188,53 @@ def _check(args): return True +def _export_database(db, args): + print "\n== Exporting database ==\n" + + command = '{mongodump_path} -d {database} -o {mongodump_cache}'.format( + mongodump_path=args.mongodump_path, + database=db.name, + mongodump_cache=args._cache_path['database']) + + p = subprocess.Popen([ + args.mongodump_path, + '-d', db.name, + '-o', args._cache_path['database']]) + + p.wait() + + print "\n== Database exported ==\n" + + +def _export_media(db, args): + print "\n== Exporting media ==\n" + + media_cache = BasicFileStorage( + args._cache_path['media']) + + # TODO: Add export of queue files + queue_cache = BasicFileStorage( + args._cache_path['queue']) + + for entry in db.media_entries.find(): + for name, path in entry['media_files'].items(): + mc_file = media_cache.get_file(path, mode='wb') + mc_file.write( + mg_globals.public_store.get_file(path, mode='rb').read()) + + print(mc_file) + print(entry) + + print "\n== Media exported ==\n" + + def env_export(args): + """ + Export database and media files to a tar archive + """ args = _setup_paths(args) - if not _check(args): + if not _export_check(args): print "\n== Checks did not pass, exiting ==\n" sys.exit(0) From cd8c65133e493730d9eb4c67f691d100c845a634 Mon Sep 17 00:00:00 2001 From: Christopher Allan Webber Date: Sat, 20 Aug 2011 16:48:29 -0500 Subject: [PATCH 07/12] Removing unused imports --- mediagoblin/gmg_commands/import_export.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/mediagoblin/gmg_commands/import_export.py b/mediagoblin/gmg_commands/import_export.py index f6651327..83d64313 100644 --- a/mediagoblin/gmg_commands/import_export.py +++ b/mediagoblin/gmg_commands/import_export.py @@ -14,22 +14,17 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -from mediagoblin.gmg_commands import util as commands_util from mediagoblin import mg_globals -from mediagoblin.db import util as db_util from mediagoblin.db.open import setup_connection_and_db_from_config from mediagoblin.init.config import read_mediagoblin_config -from mediagoblin import util as mg_util from mediagoblin.storage import BasicFileStorage from mediagoblin.init import setup_storage, setup_global_and_app_config -import shlex import shutil import tarfile import subprocess import os.path import os -import re import sys From c02bea6fb9a87e4cbcb8d77912a92b2226b4eceb Mon Sep 17 00:00:00 2001 From: Christopher Allan Webber Date: Sat, 20 Aug 2011 21:36:08 -0500 Subject: [PATCH 08/12] Use "with closing(tf)" since TarFile doesn't have .__exit__() --- mediagoblin/gmg_commands/import_export.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mediagoblin/gmg_commands/import_export.py b/mediagoblin/gmg_commands/import_export.py index 83d64313..fd32136c 100644 --- a/mediagoblin/gmg_commands/import_export.py +++ b/mediagoblin/gmg_commands/import_export.py @@ -26,6 +26,7 @@ import subprocess import os.path import os import sys +from contextlib import closing def import_export_parse_setup(subparser): @@ -147,7 +148,7 @@ def _create_archive(args): args.tar_file, mode='w|gz') - with tf: + with closing(tf): tf.add(args.cache_path, 'mediagoblin-data/') print "\n== Archiving done ==\n" From cc601bbd58b76ce6ee1f7b0b1e065fcd3d6ff217 Mon Sep 17 00:00:00 2001 From: Christopher Allan Webber Date: Sat, 20 Aug 2011 21:59:30 -0500 Subject: [PATCH 09/12] Removing some print debugging from import_export --- mediagoblin/gmg_commands/import_export.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/mediagoblin/gmg_commands/import_export.py b/mediagoblin/gmg_commands/import_export.py index fd32136c..b00fb1cb 100644 --- a/mediagoblin/gmg_commands/import_export.py +++ b/mediagoblin/gmg_commands/import_export.py @@ -68,9 +68,6 @@ def _import_media(db, args): media_file.write( media_cache.get_file(path, mode='rb').read()) - print(media_file) - print(entry) - print "\n== Media imported ==\n" @@ -85,8 +82,6 @@ def _import_database(db, args): '-d', db.name, os.path.join(args._cache_path['database'], db.name)]) - print p - p.wait() print "\n== Database imported ==\n" @@ -218,9 +213,6 @@ def _export_media(db, args): mc_file.write( mg_globals.public_store.get_file(path, mode='rb').read()) - print(mc_file) - print(entry) - print "\n== Media exported ==\n" From 00e381f79499b9dc5a456a24f4f012830a4c75c2 Mon Sep 17 00:00:00 2001 From: Christopher Allan Webber Date: Sat, 20 Aug 2011 22:00:21 -0500 Subject: [PATCH 10/12] Apparently we *should* _clean(args), that was commented out for debugging :) --- mediagoblin/gmg_commands/import_export.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mediagoblin/gmg_commands/import_export.py b/mediagoblin/gmg_commands/import_export.py index b00fb1cb..46a8269b 100644 --- a/mediagoblin/gmg_commands/import_export.py +++ b/mediagoblin/gmg_commands/import_export.py @@ -115,7 +115,8 @@ def env_import(args): _import_media(db, args) - # _clean(args) + _clean(args) + def _setup_paths(args): """ From 6c6009ba65b2df22b27cb6b3d83ee8c220767316 Mon Sep 17 00:00:00 2001 From: Christopher Allan Webber Date: Sat, 20 Aug 2011 22:22:54 -0500 Subject: [PATCH 11/12] Import / export to a temporary directory if cache_path not provided. --- mediagoblin/gmg_commands/import_export.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/mediagoblin/gmg_commands/import_export.py b/mediagoblin/gmg_commands/import_export.py index 46a8269b..812d1486 100644 --- a/mediagoblin/gmg_commands/import_export.py +++ b/mediagoblin/gmg_commands/import_export.py @@ -22,6 +22,7 @@ from mediagoblin.init import setup_storage, setup_global_and_app_config import shutil import tarfile +import tempfile import subprocess import os.path import os @@ -43,8 +44,8 @@ def import_export_parse_setup(subparser): '--mongorestore_path', default='mongorestore', help='mongorestore binary') subparser.add_argument( - '--cache_path', default='/tmp/mediagoblin/', - help='') + '--cache_path', + help='Temporary directory where files will be temporarily dumped') def _import_media(db, args): @@ -91,6 +92,9 @@ def env_import(args): """ Restore mongo database and media files from a tar archive """ + if not args.cache_path: + args.cache_path = tempfile.mkdtemp() + # args.cache_path += 'mediagoblin-data' setup_global_and_app_config(args.conf_file) @@ -171,12 +175,6 @@ def _export_check(args): return False - if os.path.exists(args.cache_path): - print 'The cache directory must not exist before you run this script' - print 'Cache directory: ', args.cache_path - - return False - return True @@ -221,6 +219,15 @@ def env_export(args): """ Export database and media files to a tar archive """ + if args.cache_path: + if os.path.exists(args.cache_path): + print 'The cache directory must not exist before you run this script' + print 'Cache directory: ', args.cache_path + + return False + else: + args.cache_path = tempfile.mkdtemp() + args = _setup_paths(args) if not _export_check(args): From 2db2211d96ec200471af675c55ae95c1b5f4ac0d Mon Sep 17 00:00:00 2001 From: Christopher Allan Webber Date: Sat, 20 Aug 2011 22:26:45 -0500 Subject: [PATCH 12/12] We should use os.path.join to concatenate directories. --- mediagoblin/gmg_commands/import_export.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mediagoblin/gmg_commands/import_export.py b/mediagoblin/gmg_commands/import_export.py index 812d1486..367924a5 100644 --- a/mediagoblin/gmg_commands/import_export.py +++ b/mediagoblin/gmg_commands/import_export.py @@ -95,7 +95,6 @@ def env_import(args): if not args.cache_path: args.cache_path = tempfile.mkdtemp() - # args.cache_path += 'mediagoblin-data' setup_global_and_app_config(args.conf_file) # Creates mg_globals.public_store and mg_globals.queue_store @@ -111,7 +110,8 @@ def env_import(args): tf.extractall(args.cache_path) - args.cache_path += 'mediagoblin-data' + args.cache_path = os.path.join( + args.cache_path, 'mediagoblin-data') args = _setup_paths(args) # Import database from extracted data