Merge branch 'master' into merge-python3-port
Has some issues, will iteratively fix! Conflicts: mediagoblin/gmg_commands/__init__.py mediagoblin/gmg_commands/deletemedia.py mediagoblin/gmg_commands/users.py mediagoblin/oauth/views.py mediagoblin/plugins/api/views.py mediagoblin/tests/test_api.py mediagoblin/tests/test_edit.py mediagoblin/tests/test_oauth1.py mediagoblin/tests/test_util.py mediagoblin/tools/mail.py mediagoblin/webfinger/views.py setup.py
This commit is contained in:
Christopher Allan Webber
@@ -39,6 +39,10 @@ SUBCOMMAND_MAP = {
|
||||
'setup': 'mediagoblin.gmg_commands.users:changepw_parser_setup',
|
||||
'func': 'mediagoblin.gmg_commands.users:changepw',
|
||||
'help': 'Changes a user\'s password'},
|
||||
'deleteuser': {
|
||||
'setup': 'mediagoblin.gmg_commands.users:deleteuser_parser_setup',
|
||||
'func': 'mediagoblin.gmg_commands.users:deleteuser',
|
||||
'help': 'Deletes a user'},
|
||||
'dbupdate': {
|
||||
'setup': 'mediagoblin.gmg_commands.dbupdate:dbupdate_parse_setup',
|
||||
'func': 'mediagoblin.gmg_commands.dbupdate:dbupdate',
|
||||
@@ -63,6 +67,10 @@ SUBCOMMAND_MAP = {
|
||||
'setup': 'mediagoblin.gmg_commands.serve:parser_setup',
|
||||
'func': 'mediagoblin.gmg_commands.serve:serve',
|
||||
'help': 'PasteScript replacement'},
|
||||
'batchaddmedia': {
|
||||
'setup': 'mediagoblin.gmg_commands.batchaddmedia:parser_setup',
|
||||
'func': 'mediagoblin.gmg_commands.batchaddmedia:batchaddmedia',
|
||||
'help': 'Add many media entries at once'},
|
||||
# 'theme': {
|
||||
# 'setup': 'mediagoblin.gmg_commands.theme:theme_parser_setup',
|
||||
# 'func': 'mediagoblin.gmg_commands.theme:theme',
|
||||
|
||||
206
mediagoblin/gmg_commands/batchaddmedia.py
Normal file
206
mediagoblin/gmg_commands/batchaddmedia.py
Normal file
@@ -0,0 +1,206 @@
|
||||
# GNU MediaGoblin -- federated, autonomous media hosting
|
||||
# Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS.
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
import os
|
||||
import requests, codecs
|
||||
import csv
|
||||
from urlparse import urlparse
|
||||
|
||||
from mediagoblin.gmg_commands import util as commands_util
|
||||
from mediagoblin.submit.lib import (
|
||||
submit_media, get_upload_file_limits,
|
||||
FileUploadLimit, UserUploadLimit, UserPastUploadLimit)
|
||||
from mediagoblin.tools.metadata import compact_and_validate
|
||||
from mediagoblin.tools.translate import pass_to_ugettext as _
|
||||
from jsonschema.exceptions import ValidationError
|
||||
|
||||
|
||||
def parser_setup(subparser):
|
||||
subparser.description = """\
|
||||
This command allows the administrator to upload many media files at once."""
|
||||
subparser.epilog = _(u"""For more information about how to properly run this
|
||||
script (and how to format the metadata csv file), read the MediaGoblin
|
||||
documentation page on command line uploading
|
||||
<http://docs.mediagoblin.org/siteadmin/commandline-upload.html>""")
|
||||
subparser.add_argument(
|
||||
'username',
|
||||
help=_(u"Name of user these media entries belong to"))
|
||||
subparser.add_argument(
|
||||
'metadata_path',
|
||||
help=_(
|
||||
u"""Path to the csv file containing metadata information."""))
|
||||
subparser.add_argument(
|
||||
'--celery',
|
||||
action='store_true',
|
||||
help=_(u"Don't process eagerly, pass off to celery"))
|
||||
|
||||
|
||||
def batchaddmedia(args):
|
||||
# Run eagerly unless explicetly set not to
|
||||
if not args.celery:
|
||||
os.environ['CELERY_ALWAYS_EAGER'] = 'true'
|
||||
|
||||
app = commands_util.setup_app(args)
|
||||
|
||||
files_uploaded, files_attempted = 0, 0
|
||||
|
||||
# get the user
|
||||
user = app.db.User.query.filter_by(username=args.username.lower()).first()
|
||||
if user is None:
|
||||
print _(u"Sorry, no user by username '{username}' exists".format(
|
||||
username=args.username))
|
||||
return
|
||||
|
||||
upload_limit, max_file_size = get_upload_file_limits(user)
|
||||
temp_files = []
|
||||
|
||||
if os.path.isfile(args.metadata_path):
|
||||
metadata_path = args.metadata_path
|
||||
|
||||
else:
|
||||
error = _(u'File at {path} not found, use -h flag for help'.format(
|
||||
path=args.metadata_path))
|
||||
print error
|
||||
return
|
||||
|
||||
abs_metadata_filename = os.path.abspath(metadata_path)
|
||||
abs_metadata_dir = os.path.dirname(abs_metadata_filename)
|
||||
upload_limit, max_file_size = get_upload_file_limits(user)
|
||||
|
||||
def maybe_unicodeify(some_string):
|
||||
# this is kinda terrible
|
||||
if some_string is None:
|
||||
return None
|
||||
else:
|
||||
return unicode(some_string)
|
||||
|
||||
with codecs.open(
|
||||
abs_metadata_filename, 'r', encoding='utf-8') as all_metadata:
|
||||
contents = all_metadata.read()
|
||||
media_metadata = parse_csv_file(contents)
|
||||
|
||||
for media_id, file_metadata in media_metadata.iteritems():
|
||||
files_attempted += 1
|
||||
# In case the metadata was not uploaded initialize an empty dictionary.
|
||||
json_ld_metadata = compact_and_validate({})
|
||||
|
||||
# Get all metadata entries starting with 'media' as variables and then
|
||||
# delete them because those are for internal use only.
|
||||
original_location = file_metadata['location']
|
||||
|
||||
### Pull the important media information for mediagoblin from the
|
||||
### metadata, if it is provided.
|
||||
title = file_metadata.get('title') or file_metadata.get('dc:title')
|
||||
description = (file_metadata.get('description') or
|
||||
file_metadata.get('dc:description'))
|
||||
|
||||
license = file_metadata.get('license')
|
||||
try:
|
||||
json_ld_metadata = compact_and_validate(file_metadata)
|
||||
except ValidationError, exc:
|
||||
error = _(u"""Error with media '{media_id}' value '{error_path}': {error_msg}
|
||||
Metadata was not uploaded.""".format(
|
||||
media_id=media_id,
|
||||
error_path=exc.path[0],
|
||||
error_msg=exc.message))
|
||||
print error
|
||||
continue
|
||||
|
||||
url = urlparse(original_location)
|
||||
filename = url.path.split()[-1]
|
||||
|
||||
if url.scheme == 'http':
|
||||
res = requests.get(url.geturl(), stream=True)
|
||||
media_file = res.raw
|
||||
|
||||
elif url.scheme == '':
|
||||
path = url.path
|
||||
if os.path.isabs(path):
|
||||
file_abs_path = os.path.abspath(path)
|
||||
else:
|
||||
file_path = os.path.join(abs_metadata_dir, path)
|
||||
file_abs_path = os.path.abspath(file_path)
|
||||
try:
|
||||
media_file = file(file_abs_path, 'r')
|
||||
except IOError:
|
||||
print _(u"""\
|
||||
FAIL: Local file {filename} could not be accessed.
|
||||
{filename} will not be uploaded.""".format(filename=filename))
|
||||
continue
|
||||
try:
|
||||
submit_media(
|
||||
mg_app=app,
|
||||
user=user,
|
||||
submitted_file=media_file,
|
||||
filename=filename,
|
||||
title=maybe_unicodeify(title),
|
||||
description=maybe_unicodeify(description),
|
||||
license=maybe_unicodeify(license),
|
||||
metadata=json_ld_metadata,
|
||||
tags_string=u"",
|
||||
upload_limit=upload_limit, max_file_size=max_file_size)
|
||||
print _(u"""Successfully submitted {filename}!
|
||||
Be sure to look at the Media Processing Panel on your website to be sure it
|
||||
uploaded successfully.""".format(filename=filename))
|
||||
files_uploaded += 1
|
||||
except FileUploadLimit:
|
||||
print _(
|
||||
u"FAIL: This file is larger than the upload limits for this site.")
|
||||
except UserUploadLimit:
|
||||
print _(
|
||||
"FAIL: This file will put this user past their upload limits.")
|
||||
except UserPastUploadLimit:
|
||||
print _("FAIL: This user is already past their upload limits.")
|
||||
print _(
|
||||
"{files_uploaded} out of {files_attempted} files successfully submitted".format(
|
||||
files_uploaded=files_uploaded,
|
||||
files_attempted=files_attempted))
|
||||
|
||||
|
||||
def unicode_csv_reader(unicode_csv_data, dialect=csv.excel, **kwargs):
|
||||
# csv.py doesn't do Unicode; encode temporarily as UTF-8:
|
||||
csv_reader = csv.reader(utf_8_encoder(unicode_csv_data),
|
||||
dialect=dialect, **kwargs)
|
||||
for row in csv_reader:
|
||||
# decode UTF-8 back to Unicode, cell by cell:
|
||||
yield [unicode(cell, 'utf-8') for cell in row]
|
||||
|
||||
def utf_8_encoder(unicode_csv_data):
|
||||
for line in unicode_csv_data:
|
||||
yield line.encode('utf-8')
|
||||
|
||||
def parse_csv_file(file_contents):
|
||||
"""
|
||||
The helper function which converts the csv file into a dictionary where each
|
||||
item's key is the provided value 'id' and each item's value is another
|
||||
dictionary.
|
||||
"""
|
||||
list_of_contents = file_contents.split('\n')
|
||||
key, lines = (list_of_contents[0].split(','),
|
||||
list_of_contents[1:])
|
||||
objects_dict = {}
|
||||
|
||||
# Build a dictionary
|
||||
for index, line in enumerate(lines):
|
||||
if line.isspace() or line == u'': continue
|
||||
values = unicode_csv_reader([line]).next()
|
||||
line_dict = dict([(key[i], val)
|
||||
for i, val in enumerate(values)])
|
||||
media_id = line_dict.get('id') or index
|
||||
objects_dict[media_id] = (line_dict)
|
||||
|
||||
return objects_dict
|
||||
|
||||
@@ -15,19 +15,23 @@
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
from __future__ import print_function
|
||||
import sys
|
||||
|
||||
from mediagoblin.gmg_commands import util as commands_util
|
||||
|
||||
|
||||
def parser_setup(subparser):
|
||||
subparser.add_argument('media_ids',
|
||||
help='Comma separated list of media IDs to will be deleted.')
|
||||
help='Comma separated list of media IDs will be deleted.')
|
||||
|
||||
|
||||
def deletemedia(args):
|
||||
app = commands_util.setup_app(args)
|
||||
|
||||
media_ids = set(map(int, args.media_ids.split(',')))
|
||||
media_ids = set([int(mid) for mid in args.media_ids.split(',') if mid.isdigit()])
|
||||
if not media_ids:
|
||||
print 'Can\'t find any valid media ID(s).'
|
||||
sys.exit(1)
|
||||
found_medias = set()
|
||||
filter_ids = app.db.MediaEntry.id.in_(media_ids)
|
||||
medias = app.db.MediaEntry.query.filter(filter_ids).all()
|
||||
@@ -38,3 +42,4 @@ def deletemedia(args):
|
||||
for media in media_ids - found_medias:
|
||||
print('Can\'t find a media with ID %d.' % media)
|
||||
print('Done.')
|
||||
sys.exit(0)
|
||||
|
||||
@@ -38,7 +38,7 @@ def adduser(args):
|
||||
#TODO: Lets trust admins this do not validate Emails :)
|
||||
commands_util.setup_app(args)
|
||||
|
||||
args.username = commands_util.prompt_if_not_set(args.username, "Username:")
|
||||
args.username = unicode(commands_util.prompt_if_not_set(args.username, "Username:"))
|
||||
args.password = commands_util.prompt_if_not_set(args.password, "Password:",True)
|
||||
args.email = commands_util.prompt_if_not_set(args.email, "Email:")
|
||||
|
||||
@@ -119,3 +119,23 @@ def changepw(args):
|
||||
print(u'Password successfully changed')
|
||||
else:
|
||||
print(u'The user doesn\'t exist')
|
||||
|
||||
|
||||
def deleteuser_parser_setup(subparser):
|
||||
subparser.add_argument(
|
||||
'username',
|
||||
help="Username to delete")
|
||||
|
||||
|
||||
def deleteuser(args):
|
||||
commands_util.setup_app(args)
|
||||
|
||||
db = mg_globals.database
|
||||
|
||||
user = db.User.query.filter_by(
|
||||
username=unicode(args.username.lower())).one()
|
||||
if user:
|
||||
user.delete()
|
||||
print('The user %s has been deleted' % args.username)
|
||||
else:
|
||||
print('The user %s doesn\'t exist' % args.username)
|
||||
|
||||
Reference in New Issue
Block a user