218 lines
7.9 KiB
Python
218 lines
7.9 KiB
Python
# GNU MediaGoblin -- federated, autonomous media hosting
|
|
# Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS.
|
|
#
|
|
# This program is free software: you can redistribute it and/or modify
|
|
# it under the terms of the GNU Affero General Public License as published by
|
|
# the Free Software Foundation, either version 3 of the License, or
|
|
# (at your option) any later version.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU Affero General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU Affero General Public License
|
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
from __future__ import print_function
|
|
|
|
import codecs
|
|
import csv
|
|
import os
|
|
import sys
|
|
|
|
import requests
|
|
import six
|
|
|
|
from six.moves.urllib.parse import urlparse
|
|
|
|
from mediagoblin.db.models import LocalUser
|
|
from mediagoblin.gmg_commands import util as commands_util
|
|
from mediagoblin.submit.lib import (
|
|
submit_media, FileUploadLimit, UserUploadLimit, UserPastUploadLimit)
|
|
from mediagoblin.tools.metadata import compact_and_validate
|
|
from mediagoblin.tools.translate import pass_to_ugettext as _
|
|
from jsonschema.exceptions import ValidationError
|
|
|
|
|
|
def parser_setup(subparser):
|
|
subparser.description = """\
|
|
This command allows the administrator to upload many media files at once."""
|
|
subparser.epilog = _(u"""For more information about how to properly run this
|
|
script (and how to format the metadata csv file), read the MediaGoblin
|
|
documentation page on command line uploading
|
|
<http://docs.mediagoblin.org/siteadmin/commandline-upload.html>""")
|
|
subparser.add_argument(
|
|
'username',
|
|
help=_(u"Name of user these media entries belong to"))
|
|
subparser.add_argument(
|
|
'metadata_path',
|
|
help=_(
|
|
u"""Path to the csv file containing metadata information."""))
|
|
subparser.add_argument(
|
|
'--celery',
|
|
action='store_true',
|
|
help=_(u"Don't process eagerly, pass off to celery"))
|
|
|
|
|
|
def batchaddmedia(args):
|
|
# Run eagerly unless explicetly set not to
|
|
if not args.celery:
|
|
os.environ['CELERY_ALWAYS_EAGER'] = 'true'
|
|
|
|
app = commands_util.setup_app(args)
|
|
|
|
files_uploaded, files_attempted = 0, 0
|
|
|
|
# get the user
|
|
user = app.db.LocalUser.query.filter(
|
|
LocalUser.username==args.username.lower()
|
|
).first()
|
|
if user is None:
|
|
print(_(u"Sorry, no user by username '{username}' exists".format(
|
|
username=args.username)))
|
|
return
|
|
|
|
if os.path.isfile(args.metadata_path):
|
|
metadata_path = args.metadata_path
|
|
|
|
else:
|
|
error = _(u'File at {path} not found, use -h flag for help'.format(
|
|
path=args.metadata_path))
|
|
print(error)
|
|
return
|
|
|
|
abs_metadata_filename = os.path.abspath(metadata_path)
|
|
abs_metadata_dir = os.path.dirname(abs_metadata_filename)
|
|
|
|
def maybe_unicodeify(some_string):
|
|
# this is kinda terrible
|
|
if some_string is None:
|
|
return None
|
|
else:
|
|
return six.text_type(some_string)
|
|
|
|
with codecs.open(
|
|
abs_metadata_filename, 'r', encoding='utf-8') as all_metadata:
|
|
contents = all_metadata.read()
|
|
media_metadata = parse_csv_file(contents)
|
|
|
|
for media_id, file_metadata in media_metadata.items():
|
|
files_attempted += 1
|
|
# In case the metadata was not uploaded initialize an empty dictionary.
|
|
json_ld_metadata = compact_and_validate({})
|
|
|
|
# Get all metadata entries starting with 'media' as variables and then
|
|
# delete them because those are for internal use only.
|
|
original_location = file_metadata['location']
|
|
|
|
### Pull the important media information for mediagoblin from the
|
|
### metadata, if it is provided.
|
|
title = file_metadata.get('title') or file_metadata.get('dc:title')
|
|
description = (file_metadata.get('description') or
|
|
file_metadata.get('dc:description'))
|
|
collection_slug = file_metadata.get('collection-slug')
|
|
|
|
license = file_metadata.get('license')
|
|
try:
|
|
json_ld_metadata = compact_and_validate(file_metadata)
|
|
except ValidationError as exc:
|
|
error = _(u"""Error with media '{media_id}' value '{error_path}': {error_msg}
|
|
Metadata was not uploaded.""".format(
|
|
media_id=media_id,
|
|
error_path=exc.path[0],
|
|
error_msg=exc.message))
|
|
print(error)
|
|
continue
|
|
|
|
url = urlparse(original_location)
|
|
filename = url.path.split()[-1]
|
|
|
|
if url.scheme == 'http':
|
|
res = requests.get(url.geturl(), stream=True)
|
|
media_file = res.raw
|
|
|
|
elif url.scheme == '':
|
|
path = url.path
|
|
if os.path.isabs(path):
|
|
file_abs_path = os.path.abspath(path)
|
|
else:
|
|
file_path = os.path.join(abs_metadata_dir, path)
|
|
file_abs_path = os.path.abspath(file_path)
|
|
try:
|
|
media_file = open(file_abs_path, 'rb')
|
|
except IOError:
|
|
print(_(u"""\
|
|
FAIL: Local file {filename} could not be accessed.
|
|
{filename} will not be uploaded.""".format(filename=filename)))
|
|
continue
|
|
try:
|
|
submit_media(
|
|
mg_app=app,
|
|
user=user,
|
|
submitted_file=media_file,
|
|
filename=filename,
|
|
title=maybe_unicodeify(title),
|
|
description=maybe_unicodeify(description),
|
|
collection_slug=maybe_unicodeify(collection_slug),
|
|
license=maybe_unicodeify(license),
|
|
metadata=json_ld_metadata,
|
|
tags_string=u"")
|
|
print(_(u"""Successfully submitted {filename}!
|
|
Be sure to look at the Media Processing Panel on your website to be sure it
|
|
uploaded successfully.""".format(filename=filename)))
|
|
files_uploaded += 1
|
|
except FileUploadLimit:
|
|
print(_(
|
|
u"FAIL: This file is larger than the upload limits for this site."))
|
|
except UserUploadLimit:
|
|
print(_(
|
|
"FAIL: This file will put this user past their upload limits."))
|
|
except UserPastUploadLimit:
|
|
print(_("FAIL: This user is already past their upload limits."))
|
|
print(_(
|
|
"{files_uploaded} out of {files_attempted} files successfully submitted".format(
|
|
files_uploaded=files_uploaded,
|
|
files_attempted=files_attempted)))
|
|
|
|
|
|
def unicode_csv_reader(unicode_csv_data, dialect=csv.excel, **kwargs):
|
|
# csv.py doesn't do Unicode; encode temporarily as UTF-8:
|
|
# TODO: this probably won't be necessary in Python 3
|
|
csv_reader = csv.reader(utf_8_encoder(unicode_csv_data),
|
|
dialect=dialect, **kwargs)
|
|
for row in csv_reader:
|
|
# decode UTF-8 back to Unicode, cell by cell:
|
|
yield [six.text_type(cell, 'utf-8') for cell in row]
|
|
|
|
def utf_8_encoder(unicode_csv_data):
|
|
for line in unicode_csv_data:
|
|
yield line.encode('utf-8')
|
|
|
|
def parse_csv_file(file_contents):
|
|
"""
|
|
The helper function which converts the csv file into a dictionary where each
|
|
item's key is the provided value 'id' and each item's value is another
|
|
dictionary.
|
|
"""
|
|
list_of_contents = file_contents.split('\n')
|
|
key, lines = (list_of_contents[0].split(','),
|
|
list_of_contents[1:])
|
|
objects_dict = {}
|
|
|
|
# Build a dictionary
|
|
for index, line in enumerate(lines):
|
|
if line.isspace() or line == u'': continue
|
|
if (sys.version_info[0] == 3):
|
|
# Python 3's csv.py supports Unicode out of the box.
|
|
reader = csv.reader([line])
|
|
else:
|
|
reader = unicode_csv_reader([line])
|
|
values = next(reader)
|
|
line_dict = dict([(key[i], val)
|
|
for i, val in enumerate(values)])
|
|
media_id = line_dict.get('id') or index
|
|
objects_dict[media_id] = (line_dict)
|
|
|
|
return objects_dict
|