From 3214aeb2387cd1356685372f9abaebe35ea7f006 Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Thu, 6 Feb 2014 15:17:06 -0500 Subject: [PATCH 1/5] This branch will create a commandline bulk-upload script. So far, I have written the code to read csv files into a usable dictionary. --- mediagoblin/gmg_commands/__init__.py | 4 + mediagoblin/gmg_commands/batchaddmedia.py | 110 ++++++++++++++++++++++ 2 files changed, 114 insertions(+) create mode 100644 mediagoblin/gmg_commands/batchaddmedia.py diff --git a/mediagoblin/gmg_commands/__init__.py b/mediagoblin/gmg_commands/__init__.py index a1eb599d..1460733f 100644 --- a/mediagoblin/gmg_commands/__init__.py +++ b/mediagoblin/gmg_commands/__init__.py @@ -53,6 +53,10 @@ SUBCOMMAND_MAP = { 'setup': 'mediagoblin.gmg_commands.addmedia:parser_setup', 'func': 'mediagoblin.gmg_commands.addmedia:addmedia', 'help': 'Reprocess media entries'}, + 'batchaddmedia': { + 'setup': 'mediagoblin.gmg_commands.batchaddmedia:parser_setup', + 'func': 'mediagoblin.gmg_commands.batchaddmedia:batchaddmedia', + 'help': 'Reprocess many media entries'} # 'theme': { # 'setup': 'mediagoblin.gmg_commands.theme:theme_parser_setup', # 'func': 'mediagoblin.gmg_commands.theme:theme', diff --git a/mediagoblin/gmg_commands/batchaddmedia.py b/mediagoblin/gmg_commands/batchaddmedia.py new file mode 100644 index 00000000..1c0f6784 --- /dev/null +++ b/mediagoblin/gmg_commands/batchaddmedia.py @@ -0,0 +1,110 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import os + +from mediagoblin.gmg_commands import util as commands_util +from mediagoblin.submit.lib import ( + submit_media, get_upload_file_limits, + FileUploadLimit, UserUploadLimit, UserPastUploadLimit) + +from mediagoblin import mg_globals +import json, csv + +def parser_setup(subparser): + subparser.add_argument( + 'username', + help="Name of user this media entry belongs to") + subparser.add_argument( + 'locationfile', + help=( +"Local file on filesystem with the address of all the files to be uploaded")) + subparser.add_argument( + 'metadatafile', + help=( +"Local file on filesystem with metadata of all the files to be uploaded")) + subparser.add_argument( + "-l", "--license", + help=( + "License these media entry will be released under, if all the same" + "Should be a URL.")) + subparser.add_argument( + '--celery', + action='store_true', + help="Don't process eagerly, pass off to celery") + + +def batchaddmedia(args): + # Run eagerly unless explicetly set not to + if not args.celery: + os.environ['CELERY_ALWAYS_EAGER'] = 'true' + + app = commands_util.setup_app(args) + + # get the user + user = app.db.User.query.filter_by(username=args.username.lower()).first() + if user is None: + print "Sorry, no user by username '%s'" % args.username + return + + # check for the location file, if it exists... + location_filename = os.path.split(args.locationfile)[-1] + abs_location_filename = os.path.abspath(args.locationfile) + if not os.path.exists(abs_location_filename): + print "Can't find a file with filename '%s'" % args.locationfile + return + + # check for the location file, if it exists... + metadata_filename = os.path.split(args.metadatafile)[-1] + abs_metadata_filename = os.path.abspath(args.metadatafile) + if not os.path.exists(abs_metadata_filename): + print "Can't find a file with filename '%s'" % args.metadatafile + return + + upload_limit, max_file_size = get_upload_file_limits(user) + + def maybe_unicodeify(some_string): + # this is kinda terrible + if some_string is None: + return None + else: + return unicode(some_string) + + with file(abs_location_filename, 'r') as all_locations: + contents = all_locations.read() + media_locations = parse_csv_file(contents) + + with file(abs_metadata_filename, 'r') as all_metadata: + contents = all_metadata.read() + media_metadata = parse_csv_file(contents) + +def parse_csv_file(file_contents): + list_of_contents = file_contents.split('\n') + key, lines = (list_of_contents[0].split(','), + list_of_contents[1:]) + list_of_objects = [] + + # Build a dictionary + for line in lines: + if line.isspace() or line == '': continue + values = csv.reader([line]).next() + new_dict = dict([(key[i], val) + for i, val in enumerate(values)]) + list_of_objects.append(new_dict) + + return list_of_objects + + From 714c4cb7d7a1918d3b4cf5cbe9145078cd330b5b Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Wed, 12 Feb 2014 14:37:00 -0500 Subject: [PATCH 2/5] The script now officially works! It works in many different situations, whether the media is to be uploaded is stored locally or on the web. Still have to clean up the code and look for errors. I may also refactor some of this into a functi- on to be used with a GUI frontend in another project. Lastly, I need to merge this with the metadata branch I've been working on, and convert the metadata.csv information into the proper format for the new metadata column. --- mediagoblin/gmg_commands/batchaddmedia.py | 130 ++++++++++++++++++---- 1 file changed, 111 insertions(+), 19 deletions(-) diff --git a/mediagoblin/gmg_commands/batchaddmedia.py b/mediagoblin/gmg_commands/batchaddmedia.py index 1c0f6784..7d7a2d4f 100644 --- a/mediagoblin/gmg_commands/batchaddmedia.py +++ b/mediagoblin/gmg_commands/batchaddmedia.py @@ -15,6 +15,10 @@ # along with this program. If not, see . import os +import json, tempfile, urllib, tarfile, subprocess +from csv import reader as csv_reader +from urlparse import urlparse +from pyld import jsonld from mediagoblin.gmg_commands import util as commands_util from mediagoblin.submit.lib import ( @@ -22,20 +26,26 @@ from mediagoblin.submit.lib import ( FileUploadLimit, UserUploadLimit, UserPastUploadLimit) from mediagoblin import mg_globals -import json, csv def parser_setup(subparser): subparser.add_argument( 'username', help="Name of user this media entry belongs to") - subparser.add_argument( - 'locationfile', + target_type = subparser.add_mutually_exclusive_group() + target_type.add_argument('-d', + '--directory', action='store_const', + const='directory', dest='target_type', + default='directory', help=( +"Target is a directory")) + target_type.add_argument('-a', + '--archive', action='store_const', + const='archive', dest='target_type', help=( -"Local file on filesystem with the address of all the files to be uploaded")) +"Target is an archive.")) subparser.add_argument( - 'metadatafile', + 'target_path', help=( -"Local file on filesystem with metadata of all the files to be uploaded")) +"Path to a local archive or directory containing a location.csv and metadata.csv file")) subparser.add_argument( "-l", "--license", help=( @@ -59,19 +69,36 @@ def batchaddmedia(args): if user is None: print "Sorry, no user by username '%s'" % args.username return + + upload_limit, max_file_size = get_upload_file_limits(user) + temp_files = [] + + if args.target_type == 'archive': + dir_path = tempfile.mkdtemp() + temp_files.append(dir_path) + tar = tarfile.open(args.target_path) + tar.extractall(path=dir_path) + + elif args.target_type == 'directory': + dir_path = args.target_path + + location_file_path = "{dir_path}/location.csv".format( + dir_path=dir_path) + metadata_file_path = "{dir_path}/metadata.csv".format( + dir_path=dir_path) # check for the location file, if it exists... - location_filename = os.path.split(args.locationfile)[-1] - abs_location_filename = os.path.abspath(args.locationfile) + location_filename = os.path.split(location_file_path)[-1] + abs_location_filename = os.path.abspath(location_file_path) if not os.path.exists(abs_location_filename): - print "Can't find a file with filename '%s'" % args.locationfile + print "Can't find a file with filename '%s'" % location_file_path return - # check for the location file, if it exists... - metadata_filename = os.path.split(args.metadatafile)[-1] - abs_metadata_filename = os.path.abspath(args.metadatafile) + # check for the metadata file, if it exists... + metadata_filename = os.path.split(metadata_file_path)[-1] + abs_metadata_filename = os.path.abspath(metadata_file_path) if not os.path.exists(abs_metadata_filename): - print "Can't find a file with filename '%s'" % args.metadatafile + print "Can't find a file with filename '%s'" % metadata_file_path return upload_limit, max_file_size = get_upload_file_limits(user) @@ -91,20 +118,85 @@ def batchaddmedia(args): contents = all_metadata.read() media_metadata = parse_csv_file(contents) + dcterms_context = { 'dcterms':'http://purl.org/dc/terms/' } + + for media_id in media_locations.keys(): + file_metadata = media_metadata[media_id] + json_ld_metadata = jsonld.compact(file_metadata, dcterms_context) + original_location = media_locations[media_id]['media:original'] + url = urlparse(original_location) + + title = file_metadata.get('dcterms:title') + description = file_metadata.get('dcterms:description') + license = file_metadata.get('dcterms:license') + filename = url.path.split()[-1] + print "Working with {filename}".format(filename=filename) + + if url.scheme == 'http': + print "Downloading {filename}...".format( + filename=filename) + media_file = tempfile.TemporaryFile() + res = urllib.urlopen(url.geturl()) + media_file.write(res.read()) + media_file.seek(0) + + elif url.scheme == '': + path = url.path + if os.path.isabs(path): + file_abs_path = os.path.abspath(path) + else: + file_path = "{dir_path}/{local_path}".format( + dir_path=dir_path, + local_path=path) + file_abs_path = os.path.abspath(file_path) + try: + media_file = file(file_abs_path, 'r') + except IOError: + print "Local file {filename} could not be accessed.".format( + filename=filename) + print "Skipping it." + continue + print "Submitting {filename}...".format(filename=filename) + try: + submit_media( + mg_app=app, + user=user, + submitted_file=media_file, + filename=filename, + title=maybe_unicodeify(title), + description=maybe_unicodeify(description), + license=maybe_unicodeify(license), + tags_string=u"", + upload_limit=upload_limit, max_file_size=max_file_size) + print "Successfully uploading {filename}!".format(filename=filename) + print "" + except FileUploadLimit: + print "This file is larger than the upload limits for this site." + except UserUploadLimit: + print "This file will put this user past their upload limits." + except UserPastUploadLimit: + print "This user is already past their upload limits." + teardown(temp_files) + + + def parse_csv_file(file_contents): list_of_contents = file_contents.split('\n') key, lines = (list_of_contents[0].split(','), list_of_contents[1:]) - list_of_objects = [] + objects_dict = {} # Build a dictionary for line in lines: if line.isspace() or line == '': continue - values = csv.reader([line]).next() - new_dict = dict([(key[i], val) + values = csv_reader([line]).next() + line_dict = dict([(key[i], val) for i, val in enumerate(values)]) - list_of_objects.append(new_dict) + media_id = line_dict['media:id'] + objects_dict[media_id] = (line_dict) - return list_of_objects + return objects_dict - +def teardown(temp_files): + for temp_file in temp_files: + subprocess.call(['rm','-r',temp_file]) From 27b7d94896cd3cede2050b62af1321ad69cd3fa1 Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Thu, 13 Feb 2014 13:57:10 -0500 Subject: [PATCH 3/5] Minor change in the wording of argparsing. --- mediagoblin/gmg_commands/batchaddmedia.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mediagoblin/gmg_commands/batchaddmedia.py b/mediagoblin/gmg_commands/batchaddmedia.py index 7d7a2d4f..2fd36dfb 100644 --- a/mediagoblin/gmg_commands/batchaddmedia.py +++ b/mediagoblin/gmg_commands/batchaddmedia.py @@ -36,12 +36,12 @@ def parser_setup(subparser): '--directory', action='store_const', const='directory', dest='target_type', default='directory', help=( -"Target is a directory")) +"Choose this option is the target is a directory.")) target_type.add_argument('-a', '--archive', action='store_const', const='archive', dest='target_type', help=( -"Target is an archive.")) +"Choose this option if the target is an archive.")) subparser.add_argument( 'target_path', help=( From 579a6b574f402c23d3b09d22e4ab4c9f71b0e7aa Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Wed, 19 Feb 2014 14:27:14 -0500 Subject: [PATCH 4/5] I made it so the command no longer requires the "Target type" to be provided, it now recognizes whether the target is a directory or an archive on its own. I added in a help message, which is still incomplete, but should make it easier for admins to know how to use this new command. I believe we should also provi- -de an example of the location.csv and metadata.csv files, so there is no conf- -usion. Also, I made it possible for the command to recognize zip files as a valid archive. I also made some minor changes to the commands description w/i the larger gmg command help menu. --- mediagoblin/gmg_commands/__init__.py | 2 +- mediagoblin/gmg_commands/batchaddmedia.py | 55 ++++++++++++++--------- 2 files changed, 34 insertions(+), 23 deletions(-) diff --git a/mediagoblin/gmg_commands/__init__.py b/mediagoblin/gmg_commands/__init__.py index 1460733f..55e85116 100644 --- a/mediagoblin/gmg_commands/__init__.py +++ b/mediagoblin/gmg_commands/__init__.py @@ -56,7 +56,7 @@ SUBCOMMAND_MAP = { 'batchaddmedia': { 'setup': 'mediagoblin.gmg_commands.batchaddmedia:parser_setup', 'func': 'mediagoblin.gmg_commands.batchaddmedia:batchaddmedia', - 'help': 'Reprocess many media entries'} + 'help': 'Add many media entries at once'} # 'theme': { # 'setup': 'mediagoblin.gmg_commands.theme:theme_parser_setup', # 'func': 'mediagoblin.gmg_commands.theme:theme', diff --git a/mediagoblin/gmg_commands/batchaddmedia.py b/mediagoblin/gmg_commands/batchaddmedia.py index 2fd36dfb..d3ab7733 100644 --- a/mediagoblin/gmg_commands/batchaddmedia.py +++ b/mediagoblin/gmg_commands/batchaddmedia.py @@ -15,7 +15,7 @@ # along with this program. If not, see . import os -import json, tempfile, urllib, tarfile, subprocess +import json, tempfile, urllib, tarfile, zipfile, subprocess from csv import reader as csv_reader from urlparse import urlparse from pyld import jsonld @@ -28,29 +28,28 @@ from mediagoblin.submit.lib import ( from mediagoblin import mg_globals def parser_setup(subparser): + subparser.description = """\ +This command allows the administrator to upload many media files at once.""" subparser.add_argument( 'username', - help="Name of user this media entry belongs to") - target_type = subparser.add_mutually_exclusive_group() - target_type.add_argument('-d', - '--directory', action='store_const', - const='directory', dest='target_type', - default='directory', help=( -"Choose this option is the target is a directory.")) - target_type.add_argument('-a', - '--archive', action='store_const', - const='archive', dest='target_type', - help=( -"Choose this option if the target is an archive.")) + help="Name of user these media entries belong to") subparser.add_argument( 'target_path', - help=( -"Path to a local archive or directory containing a location.csv and metadata.csv file")) + help=("""\ +Path to a local archive or directory containing a "location.csv" and a +"metadata.csv" file. These are csv (comma seperated value) files with the +locations and metadata of the files to be uploaded. The location must be listed +with either the URL of the remote media file or the filesystem path of a local +file. The metadata should be provided with one column for each of the 15 Dublin +Core properties (http://dublincore.org/documents/dces/). Both "location.csv" and +"metadata.csv" must begin with a row demonstrating the order of the columns. We +have provided an example of these files at +""")) subparser.add_argument( "-l", "--license", help=( - "License these media entry will be released under, if all the same" - "Should be a URL.")) + "License these media entry will be released under, if all the same. " + "Should be a URL.")) subparser.add_argument( '--celery', action='store_true', @@ -67,26 +66,38 @@ def batchaddmedia(args): # get the user user = app.db.User.query.filter_by(username=args.username.lower()).first() if user is None: - print "Sorry, no user by username '%s'" % args.username + print "Sorry, no user by username '%s' exists" % args.username return upload_limit, max_file_size = get_upload_file_limits(user) temp_files = [] - if args.target_type == 'archive': + if tarfile.is_tarfile(args.target_path): dir_path = tempfile.mkdtemp() temp_files.append(dir_path) tar = tarfile.open(args.target_path) tar.extractall(path=dir_path) - elif args.target_type == 'directory': + elif zipfile.is_zipfile(args.target_path): + dir_path = tempfile.mkdtemp() + temp_files.append(dir_path) + zipped_file = zipfile.ZipFile(args.target_path) + zipped_file.extractall(path=dir_path) + + elif os.path.isdir(args.target_path): dir_path = args.target_path + else: + print "Couldn't recognize the file. This script only accepts tar files,\ +zip files and directories" + if dir_path.endswith('/'): + dir_path = dir_path[:-1] + location_file_path = "{dir_path}/location.csv".format( dir_path=dir_path) metadata_file_path = "{dir_path}/metadata.csv".format( dir_path=dir_path) - + # check for the location file, if it exists... location_filename = os.path.split(location_file_path)[-1] abs_location_filename = os.path.abspath(location_file_path) @@ -178,7 +189,7 @@ def batchaddmedia(args): print "This user is already past their upload limits." teardown(temp_files) - + def parse_csv_file(file_contents): list_of_contents = file_contents.split('\n') From 9d4e9de76b29f8cc602a0db6334e7d36bb3e0fb0 Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Fri, 21 Feb 2014 12:38:02 -0500 Subject: [PATCH 5/5] Changed some of the print messages as well as tweaked the order of the commands attempts to figure out what type of file the target file is. --- mediagoblin/gmg_commands/batchaddmedia.py | 30 +++++++++++++---------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/mediagoblin/gmg_commands/batchaddmedia.py b/mediagoblin/gmg_commands/batchaddmedia.py index d3ab7733..678c8ab4 100644 --- a/mediagoblin/gmg_commands/batchaddmedia.py +++ b/mediagoblin/gmg_commands/batchaddmedia.py @@ -63,6 +63,8 @@ def batchaddmedia(args): app = commands_util.setup_app(args) + files_uploaded, files_attempted = 0, 0 + # get the user user = app.db.User.query.filter_by(username=args.username.lower()).first() if user is None: @@ -72,7 +74,10 @@ def batchaddmedia(args): upload_limit, max_file_size = get_upload_file_limits(user) temp_files = [] - if tarfile.is_tarfile(args.target_path): + if os.path.isdir(args.target_path): + dir_path = args.target_path + + elif tarfile.is_tarfile(args.target_path): dir_path = tempfile.mkdtemp() temp_files.append(dir_path) tar = tarfile.open(args.target_path) @@ -84,9 +89,6 @@ def batchaddmedia(args): zipped_file = zipfile.ZipFile(args.target_path) zipped_file.extractall(path=dir_path) - elif os.path.isdir(args.target_path): - dir_path = args.target_path - else: print "Couldn't recognize the file. This script only accepts tar files,\ zip files and directories" @@ -141,11 +143,9 @@ zip files and directories" description = file_metadata.get('dcterms:description') license = file_metadata.get('dcterms:license') filename = url.path.split()[-1] - print "Working with {filename}".format(filename=filename) + files_attempted += 1 if url.scheme == 'http': - print "Downloading {filename}...".format( - filename=filename) media_file = tempfile.TemporaryFile() res = urllib.urlopen(url.geturl()) media_file.write(res.read()) @@ -163,11 +163,10 @@ zip files and directories" try: media_file = file(file_abs_path, 'r') except IOError: - print "Local file {filename} could not be accessed.".format( - filename=filename) + print "\ +FAIL: Local file {filename} could not be accessed.".format(filename=filename) print "Skipping it." continue - print "Submitting {filename}...".format(filename=filename) try: submit_media( mg_app=app, @@ -181,12 +180,17 @@ zip files and directories" upload_limit=upload_limit, max_file_size=max_file_size) print "Successfully uploading {filename}!".format(filename=filename) print "" + files_uploaded += 1 except FileUploadLimit: - print "This file is larger than the upload limits for this site." + print "FAIL: This file is larger than the upload limits for this site." except UserUploadLimit: - print "This file will put this user past their upload limits." + print "FAIL: This file will put this user past their upload limits." except UserPastUploadLimit: - print "This user is already past their upload limits." + print "FAIL: This user is already past their upload limits." + print "\ +{files_uploaded} out of {files_attempted} files successfully uploaded".format( + files_uploaded=files_uploaded, + files_attempted=files_attempted) teardown(temp_files)