heckyel/mediagoblin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Modified the batchaddmedia script so that it only looks for one csv file instead

Browse Source 
of the previous method which looked for two files.
This commit is contained in:
tilly-Q 2014-05-12 13:59:28 -04:00
parent e8d64d453b
commit 7d52eb7705
1 changed files with 46 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

123
mediagoblin/gmg_commands/batchaddmedia.py
View File

@ -25,7 +25,7 @@ from mediagoblin.submit.lib import (
submit_media, get_upload_file_limits, submit_media, get_upload_file_limits,
FileUploadLimit, UserUploadLimit, UserPastUploadLimit) FileUploadLimit, UserUploadLimit, UserPastUploadLimit)
from mediagoblin.tools.metadata import compact_and_validate from mediagoblin.tools.metadata import compact_and_validate
from mediagoblin.tools.translate import pass_to_ugettext as _
from jsonschema.exceptions import ValidationError from jsonschema.exceptions import ValidationError
@ -36,17 +36,7 @@ This command allows the administrator to upload many media files at once."""
'username', 'username',
help="Name of user these media entries belong to") help="Name of user these media entries belong to")
subparser.add_argument( subparser.add_argument(
'target_path', 'metadata_path')
help=("""\
Path to a local archive or directory containing a "location.csv" and a
"metadata.csv" file. These are csv (comma seperated value) files with the
locations and metadata of the files to be uploaded. The location must be listed
with either the URL of the remote media file or the filesystem path of a local
file. The metadata should be provided with one column for each of the 15 Dublin
Core properties (http://dublincore.org/documents/dces/). Both "location.csv" and
"metadata.csv" must begin with a row demonstrating the order of the columns. We
have provided an example of these files at <url to be added>
"""))
subparser.add_argument( subparser.add_argument(
'--celery', '--celery',
action='store_true', action='store_true',
@ -65,48 +55,24 @@ def batchaddmedia(args):
# get the user # get the user
user = app.db.User.query.filter_by(username=args.username.lower()).first() user = app.db.User.query.filter_by(username=args.username.lower()).first()
if user is None: if user is None:
print "Sorry, no user by username '%s' exists" % args.username print _(u"Sorry, no user by username '{username}' exists".format(
username=args.username))
return return
upload_limit, max_file_size = get_upload_file_limits(user) upload_limit, max_file_size = get_upload_file_limits(user)
temp_files = [] temp_files = []
if os.path.isdir(args.target_path): if os.path.isfile(args.metadata_path):
dir_path = args.target_path metadata_path = args.metadata_path
elif tarfile.is_tarfile(args.target_path):
dir_path = tempfile.mkdtemp()
temp_files.append(dir_path)
tar = tarfile.open(args.target_path)
tar.extractall(path=dir_path)
elif zipfile.is_zipfile(args.target_path):
dir_path = tempfile.mkdtemp()
temp_files.append(dir_path)
zipped_file = zipfile.ZipFile(args.target_path)
zipped_file.extractall(path=dir_path)
else: else:
print "Couldn't recognize the file. This script only accepts tar files,\ error = _(u'File at {path} not found, use -h flag for help'.format(
zip files and directories" path=args.metadata_path))
if dir_path.endswith('/'): print error
dir_path = dir_path[:-1]
location_file_path = os.path.join(dir_path,"location.csv")
metadata_file_path = os.path.join(dir_path, "metadata.csv")
# check for the location file, if it exists...
abs_location_filename = os.path.abspath(location_file_path)
if not os.path.exists(abs_location_filename):
print "Can't find a file with filename '%s'" % location_file_path
return
# check for the metadata file, if it exists...
abs_metadata_filename = os.path.abspath(metadata_file_path)
if not os.path.exists(abs_metadata_filename):
print "Can't find a file with filename '%s'" % metadata_file_path
return return
abs_metadata_filename = os.path.abspath(metadata_path)
abs_metadata_dir = os.path.dirname(abs_metadata_filename)
upload_limit, max_file_size = get_upload_file_limits(user) upload_limit, max_file_size = get_upload_file_limits(user)
def maybe_unicodeify(some_string): def maybe_unicodeify(some_string):
@ -116,36 +82,36 @@ zip files and directories"
else: else:
return unicode(some_string) return unicode(some_string)
with file(abs_location_filename, 'r') as all_locations:
contents = all_locations.read()
media_locations = parse_csv_file(contents)
with file(abs_metadata_filename, 'r') as all_metadata: with file(abs_metadata_filename, 'r') as all_metadata:
contents = all_metadata.read() contents = all_metadata.read()
media_metadata = parse_csv_file(contents) media_metadata = parse_csv_file(contents)
for media_id in media_locations.keys(): for media_id, file_metadata in media_metadata.iteritems():
files_attempted += 1 files_attempted += 1
# In case the metadata was not uploaded initialize an empty dictionary.
json_ld_metadata = compact_and_validate({})
file_metadata = media_metadata[media_id] # Get all metadata entries starting with 'media' as variables and then
# delete them because those are for internal use only.
### Remove all metadata entries starting with 'media' because we are ### original_location = file_metadata['media:location']
### only using those for internal use. ###
file_metadata = dict([(key, value) file_metadata = dict([(key, value)
for key, value in file_metadata.iteritems() if for key, value in file_metadata.iteritems() if
key.split(":")[0] != 'media']) key.split(":")[0] != 'media'])
try: try:
json_ld_metadata = compact_and_validate(file_metadata) json_ld_metadata = compact_and_validate(file_metadata)
except ValidationError, exc: except ValidationError, exc:
print "Error with '%s' value '%s': %s" % ( error = _(u"""Error with media '{media_id}' value '{error_path}': {error_msg}
media_id, exc.path[0], exc.message) Metadata was not uploaded.""".format(
media_id=media_id,
error_path=exc.path[0],
error_msg=exc.message))
print error
continue continue
original_location = media_locations[media_id]['media:original']
url = urlparse(original_location) url = urlparse(original_location)
### Pull the important media information for mediagoblin from the ### ### Pull the important media information for mediagoblin from the
### metadata, if it is provided. ### ### metadata, if it is provided.
title = json_ld_metadata.get('dc:title') title = json_ld_metadata.get('dc:title')
description = json_ld_metadata.get('dc:description') description = json_ld_metadata.get('dc:description')
@ -161,14 +127,14 @@ zip files and directories"
if os.path.isabs(path): if os.path.isabs(path):
file_abs_path = os.path.abspath(path) file_abs_path = os.path.abspath(path)
else: else:
file_path = os.path.join(dir_path, path) file_path = os.path.join(abs_metadata_dir, path)
file_abs_path = os.path.abspath(file_path) file_abs_path = os.path.abspath(file_path)
try: try:
media_file = file(file_abs_path, 'r') media_file = file(file_abs_path, 'r')
except IOError: except IOError:
print "\ print _(u"""\
FAIL: Local file {filename} could not be accessed.".format(filename=filename) FAIL: Local file {filename} could not be accessed.
print "Skipping it." {filename} will not be uploaded.""".format(filename=filename))
continue continue
try: try:
submit_media( submit_media(
@ -182,29 +148,36 @@ FAIL: Local file {filename} could not be accessed.".format(filename=filename)
metadata=json_ld_metadata, metadata=json_ld_metadata,
tags_string=u"", tags_string=u"",
upload_limit=upload_limit, max_file_size=max_file_size) upload_limit=upload_limit, max_file_size=max_file_size)
print "Successfully uploading {filename}!".format(filename=filename) print _(u"""Successfully submitted {filename}!
print "" Be sure to look at the Media Processing Panel on your website to be sure it
uploaded successfully.""".format(filename=filename))
files_uploaded += 1 files_uploaded += 1
except FileUploadLimit: except FileUploadLimit:
print "FAIL: This file is larger than the upload limits for this site." print _(
u"FAIL: This file is larger than the upload limits for this site.")
except UserUploadLimit: except UserUploadLimit:
print "FAIL: This file will put this user past their upload limits." print _(
"FAIL: This file will put this user past their upload limits.")
except UserPastUploadLimit: except UserPastUploadLimit:
print "FAIL: This user is already past their upload limits." print _("FAIL: This user is already past their upload limits.")
print "\ print _(
{files_uploaded} out of {files_attempted} files successfully uploaded".format( "{files_uploaded} out of {files_attempted} files successfully submitted".format(
files_uploaded=files_uploaded, files_uploaded=files_uploaded,
files_attempted=files_attempted) files_attempted=files_attempted))
teardown(temp_files)
def parse_csv_file(file_contents): def parse_csv_file(file_contents):
"""
The helper function which converts the csv file into a dictionary where each
item's key is the provided value 'media:id' and each item's value is another
dictionary.
"""
list_of_contents = file_contents.split('\n') list_of_contents = file_contents.split('\n')
key, lines = (list_of_contents[0].split(','), key, lines = (list_of_contents[0].split(','),
list_of_contents[1:]) list_of_contents[1:])
objects_dict = {} objects_dict = {}
# Build a dictionaryfrom mediagoblin.tools.translate import lazy_pass_to_ugettext as _ # Build a dictionary
for line in lines: for line in lines:
if line.isspace() or line == '': continue if line.isspace() or line == '': continue
values = csv_reader([line]).next() values = csv_reader([line]).next()
@ -215,7 +188,3 @@ def parse_csv_file(file_contents):
return objects_dict return objects_dict
def teardown(temp_files):
for temp_file in temp_files:
subprocess.call(['rm','-r',temp_file])