From 7a29c67bf92830427e30590fe5a7b720da7520d4 Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Thu, 6 Feb 2014 15:15:57 -0500 Subject: [PATCH 01/78] In this commit, I added a new column which will be used for RDFa metadata of media. --- mediagoblin/db/migrations.py | 12 ++++++++++++ mediagoblin/db/models.py | 1 + 2 files changed, 13 insertions(+) diff --git a/mediagoblin/db/migrations.py b/mediagoblin/db/migrations.py index 426080a2..a7400bf0 100644 --- a/mediagoblin/db/migrations.py +++ b/mediagoblin/db/migrations.py @@ -31,6 +31,7 @@ from mediagoblin.db.migration_tools import ( RegisterMigration, inspect_table, replace_table_hack) from mediagoblin.db.models import (MediaEntry, Collection, MediaComment, User, Privilege) +from mediagoblin.db.extratypes import JSONEncoded, MutationDict MIGRATIONS = {} @@ -720,3 +721,14 @@ def drop_MediaEntry_collected(db): media_collected.drop() db.commit() + +@RegisterMigration(20, MIGRATIONS) +def add_work_metadata_column(db): + metadata = MetaData(bind=db.bind) + + media_file = inspect_table(metadata, 'core__mediafiles') + + col = Column('work_metadata', MutationDict.as_mutable(JSONEncoded)) + col.create(media_file) + + db.commit() diff --git a/mediagoblin/db/models.py b/mediagoblin/db/models.py index b750375d..ac69d040 100644 --- a/mediagoblin/db/models.py +++ b/mediagoblin/db/models.py @@ -420,6 +420,7 @@ class MediaFile(Base): name_id = Column(SmallInteger, ForeignKey(FileKeynames.id), nullable=False) file_path = Column(PathTupleWithSlashes) file_metadata = Column(MutationDict.as_mutable(JSONEncoded)) + work_metadata = Column(MutationDict.as_mutable(JSONEncoded)) __table_args__ = ( PrimaryKeyConstraint('media_entry', 'name_id'), From 74d7ff96142c2da375e12df91e23fb50c2b2af88 Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Tue, 18 Mar 2014 16:49:48 -0400 Subject: [PATCH 02/78] Big update. I added in a json-ld context file which will be used in all our metadata columns in the future. The context describes the dublin core elements. It still has not been finalized however. --- mediagoblin/routing.py | 4 +- .../mediagoblin/metadata_contexts/v1 | 70 +++++++++++++++++++ mediagoblin/views.py | 6 ++ setup.py | 1 + 4 files changed, 80 insertions(+), 1 deletion(-) create mode 100644 mediagoblin/templates/mediagoblin/metadata_contexts/v1 diff --git a/mediagoblin/routing.py b/mediagoblin/routing.py index 1393f01c..a6b2a543 100644 --- a/mediagoblin/routing.py +++ b/mediagoblin/routing.py @@ -28,7 +28,9 @@ _log = logging.getLogger(__name__) def get_url_map(): add_route('index', '/', 'mediagoblin.views:root_view') add_route('terms_of_service','/terms_of_service', - 'mediagoblin.views:terms_of_service') + 'mediagoblin.views:terms_of_service'), + add_route('metadata_context','/metadata_context/v/', + 'mediagoblin.views:metadata_context_view'), mount('/auth', auth_routes) mount('/mod', moderation_routes) diff --git a/mediagoblin/templates/mediagoblin/metadata_contexts/v1 b/mediagoblin/templates/mediagoblin/metadata_contexts/v1 new file mode 100644 index 00000000..1325d920 --- /dev/null +++ b/mediagoblin/templates/mediagoblin/metadata_contexts/v1 @@ -0,0 +1,70 @@ +{ + "@context": { + "dc": "http://purl.org/dc/elements/1.1/", + "xsd": "http://www.w3.org/2001/XMLSchema#", + "contributor":{ + "@id":"dc:title", + "@type":"xsd:string" + }, + "coverage":{ + "@id":"dc:coverage", + "@type":"xsd:string" + }, + "created":{ + "@id":"dc:created", + "@type":"xsd:date" + }, + "creator":{ + "@id":"dc:created", + "@type":"xsd:date" + }, + "date":{ + "@id":"dc:date", + "@type":"xsd:date" + }, + "description":{ + "@id":"dc:description", + "@type":"xsd:string" + }, + "format":{ + "@id":"dc:format", + "@type":"xsd:string" + }, + "identifier":{ + "@id":"dc:identifier", + "@type":"xsd:string" + }, + "language":{ + "@id":"dc:language", + "@type":"xsd:string" + }, + "publisher":{ + "@id":"dc:publisher", + "@type":"xsd:string" + }, + "relation":{ + "@id":"dc:relation", + "@type":"xsd:string" + }, + "rights":{ + "@id":"dc:rights", + "@type":"xsd:anyURI" + }, + "source":{ + "@id":"dc:source", + "@type":"xsd:string" + }, + "subject":{ + "@id":"dc:subject", + "@type":"xsd:string" + }, + "title": { + "@id":"dc:title", + "@type":"xsd:string" + }, + "type":{ + "@id":"dc:type", + "@type":"xsd:string" + } + } +} diff --git a/mediagoblin/views.py b/mediagoblin/views.py index 009e48e4..1ed71473 100644 --- a/mediagoblin/views.py +++ b/mediagoblin/views.py @@ -62,3 +62,9 @@ def terms_of_service(request): return render_to_response(request, 'mediagoblin/terms_of_service.html', {}) + +def metadata_context_view(request): + version = request.matchdict['version_number'] + return render_to_response(request, + 'mediagoblin/metadata_contexts/v{version}'.format( + version=version), {}) diff --git a/setup.py b/setup.py index d3f91686..93873d73 100644 --- a/setup.py +++ b/setup.py @@ -66,6 +66,7 @@ try: 'six>=1.4.1', 'oauthlib==0.5.0', 'unidecode', + 'jsonschema', ## Annoying. Please remove once we can! We only indirectly ## use pbr, and currently it breaks things, presumably till From 3214aeb2387cd1356685372f9abaebe35ea7f006 Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Thu, 6 Feb 2014 15:17:06 -0500 Subject: [PATCH 03/78] This branch will create a commandline bulk-upload script. So far, I have written the code to read csv files into a usable dictionary. --- mediagoblin/gmg_commands/__init__.py | 4 + mediagoblin/gmg_commands/batchaddmedia.py | 110 ++++++++++++++++++++++ 2 files changed, 114 insertions(+) create mode 100644 mediagoblin/gmg_commands/batchaddmedia.py diff --git a/mediagoblin/gmg_commands/__init__.py b/mediagoblin/gmg_commands/__init__.py index a1eb599d..1460733f 100644 --- a/mediagoblin/gmg_commands/__init__.py +++ b/mediagoblin/gmg_commands/__init__.py @@ -53,6 +53,10 @@ SUBCOMMAND_MAP = { 'setup': 'mediagoblin.gmg_commands.addmedia:parser_setup', 'func': 'mediagoblin.gmg_commands.addmedia:addmedia', 'help': 'Reprocess media entries'}, + 'batchaddmedia': { + 'setup': 'mediagoblin.gmg_commands.batchaddmedia:parser_setup', + 'func': 'mediagoblin.gmg_commands.batchaddmedia:batchaddmedia', + 'help': 'Reprocess many media entries'} # 'theme': { # 'setup': 'mediagoblin.gmg_commands.theme:theme_parser_setup', # 'func': 'mediagoblin.gmg_commands.theme:theme', diff --git a/mediagoblin/gmg_commands/batchaddmedia.py b/mediagoblin/gmg_commands/batchaddmedia.py new file mode 100644 index 00000000..1c0f6784 --- /dev/null +++ b/mediagoblin/gmg_commands/batchaddmedia.py @@ -0,0 +1,110 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import os + +from mediagoblin.gmg_commands import util as commands_util +from mediagoblin.submit.lib import ( + submit_media, get_upload_file_limits, + FileUploadLimit, UserUploadLimit, UserPastUploadLimit) + +from mediagoblin import mg_globals +import json, csv + +def parser_setup(subparser): + subparser.add_argument( + 'username', + help="Name of user this media entry belongs to") + subparser.add_argument( + 'locationfile', + help=( +"Local file on filesystem with the address of all the files to be uploaded")) + subparser.add_argument( + 'metadatafile', + help=( +"Local file on filesystem with metadata of all the files to be uploaded")) + subparser.add_argument( + "-l", "--license", + help=( + "License these media entry will be released under, if all the same" + "Should be a URL.")) + subparser.add_argument( + '--celery', + action='store_true', + help="Don't process eagerly, pass off to celery") + + +def batchaddmedia(args): + # Run eagerly unless explicetly set not to + if not args.celery: + os.environ['CELERY_ALWAYS_EAGER'] = 'true' + + app = commands_util.setup_app(args) + + # get the user + user = app.db.User.query.filter_by(username=args.username.lower()).first() + if user is None: + print "Sorry, no user by username '%s'" % args.username + return + + # check for the location file, if it exists... + location_filename = os.path.split(args.locationfile)[-1] + abs_location_filename = os.path.abspath(args.locationfile) + if not os.path.exists(abs_location_filename): + print "Can't find a file with filename '%s'" % args.locationfile + return + + # check for the location file, if it exists... + metadata_filename = os.path.split(args.metadatafile)[-1] + abs_metadata_filename = os.path.abspath(args.metadatafile) + if not os.path.exists(abs_metadata_filename): + print "Can't find a file with filename '%s'" % args.metadatafile + return + + upload_limit, max_file_size = get_upload_file_limits(user) + + def maybe_unicodeify(some_string): + # this is kinda terrible + if some_string is None: + return None + else: + return unicode(some_string) + + with file(abs_location_filename, 'r') as all_locations: + contents = all_locations.read() + media_locations = parse_csv_file(contents) + + with file(abs_metadata_filename, 'r') as all_metadata: + contents = all_metadata.read() + media_metadata = parse_csv_file(contents) + +def parse_csv_file(file_contents): + list_of_contents = file_contents.split('\n') + key, lines = (list_of_contents[0].split(','), + list_of_contents[1:]) + list_of_objects = [] + + # Build a dictionary + for line in lines: + if line.isspace() or line == '': continue + values = csv.reader([line]).next() + new_dict = dict([(key[i], val) + for i, val in enumerate(values)]) + list_of_objects.append(new_dict) + + return list_of_objects + + From 714c4cb7d7a1918d3b4cf5cbe9145078cd330b5b Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Wed, 12 Feb 2014 14:37:00 -0500 Subject: [PATCH 04/78] The script now officially works! It works in many different situations, whether the media is to be uploaded is stored locally or on the web. Still have to clean up the code and look for errors. I may also refactor some of this into a functi- on to be used with a GUI frontend in another project. Lastly, I need to merge this with the metadata branch I've been working on, and convert the metadata.csv information into the proper format for the new metadata column. --- mediagoblin/gmg_commands/batchaddmedia.py | 130 ++++++++++++++++++---- 1 file changed, 111 insertions(+), 19 deletions(-) diff --git a/mediagoblin/gmg_commands/batchaddmedia.py b/mediagoblin/gmg_commands/batchaddmedia.py index 1c0f6784..7d7a2d4f 100644 --- a/mediagoblin/gmg_commands/batchaddmedia.py +++ b/mediagoblin/gmg_commands/batchaddmedia.py @@ -15,6 +15,10 @@ # along with this program. If not, see . import os +import json, tempfile, urllib, tarfile, subprocess +from csv import reader as csv_reader +from urlparse import urlparse +from pyld import jsonld from mediagoblin.gmg_commands import util as commands_util from mediagoblin.submit.lib import ( @@ -22,20 +26,26 @@ from mediagoblin.submit.lib import ( FileUploadLimit, UserUploadLimit, UserPastUploadLimit) from mediagoblin import mg_globals -import json, csv def parser_setup(subparser): subparser.add_argument( 'username', help="Name of user this media entry belongs to") - subparser.add_argument( - 'locationfile', + target_type = subparser.add_mutually_exclusive_group() + target_type.add_argument('-d', + '--directory', action='store_const', + const='directory', dest='target_type', + default='directory', help=( +"Target is a directory")) + target_type.add_argument('-a', + '--archive', action='store_const', + const='archive', dest='target_type', help=( -"Local file on filesystem with the address of all the files to be uploaded")) +"Target is an archive.")) subparser.add_argument( - 'metadatafile', + 'target_path', help=( -"Local file on filesystem with metadata of all the files to be uploaded")) +"Path to a local archive or directory containing a location.csv and metadata.csv file")) subparser.add_argument( "-l", "--license", help=( @@ -59,19 +69,36 @@ def batchaddmedia(args): if user is None: print "Sorry, no user by username '%s'" % args.username return + + upload_limit, max_file_size = get_upload_file_limits(user) + temp_files = [] + + if args.target_type == 'archive': + dir_path = tempfile.mkdtemp() + temp_files.append(dir_path) + tar = tarfile.open(args.target_path) + tar.extractall(path=dir_path) + + elif args.target_type == 'directory': + dir_path = args.target_path + + location_file_path = "{dir_path}/location.csv".format( + dir_path=dir_path) + metadata_file_path = "{dir_path}/metadata.csv".format( + dir_path=dir_path) # check for the location file, if it exists... - location_filename = os.path.split(args.locationfile)[-1] - abs_location_filename = os.path.abspath(args.locationfile) + location_filename = os.path.split(location_file_path)[-1] + abs_location_filename = os.path.abspath(location_file_path) if not os.path.exists(abs_location_filename): - print "Can't find a file with filename '%s'" % args.locationfile + print "Can't find a file with filename '%s'" % location_file_path return - # check for the location file, if it exists... - metadata_filename = os.path.split(args.metadatafile)[-1] - abs_metadata_filename = os.path.abspath(args.metadatafile) + # check for the metadata file, if it exists... + metadata_filename = os.path.split(metadata_file_path)[-1] + abs_metadata_filename = os.path.abspath(metadata_file_path) if not os.path.exists(abs_metadata_filename): - print "Can't find a file with filename '%s'" % args.metadatafile + print "Can't find a file with filename '%s'" % metadata_file_path return upload_limit, max_file_size = get_upload_file_limits(user) @@ -91,20 +118,85 @@ def batchaddmedia(args): contents = all_metadata.read() media_metadata = parse_csv_file(contents) + dcterms_context = { 'dcterms':'http://purl.org/dc/terms/' } + + for media_id in media_locations.keys(): + file_metadata = media_metadata[media_id] + json_ld_metadata = jsonld.compact(file_metadata, dcterms_context) + original_location = media_locations[media_id]['media:original'] + url = urlparse(original_location) + + title = file_metadata.get('dcterms:title') + description = file_metadata.get('dcterms:description') + license = file_metadata.get('dcterms:license') + filename = url.path.split()[-1] + print "Working with {filename}".format(filename=filename) + + if url.scheme == 'http': + print "Downloading {filename}...".format( + filename=filename) + media_file = tempfile.TemporaryFile() + res = urllib.urlopen(url.geturl()) + media_file.write(res.read()) + media_file.seek(0) + + elif url.scheme == '': + path = url.path + if os.path.isabs(path): + file_abs_path = os.path.abspath(path) + else: + file_path = "{dir_path}/{local_path}".format( + dir_path=dir_path, + local_path=path) + file_abs_path = os.path.abspath(file_path) + try: + media_file = file(file_abs_path, 'r') + except IOError: + print "Local file {filename} could not be accessed.".format( + filename=filename) + print "Skipping it." + continue + print "Submitting {filename}...".format(filename=filename) + try: + submit_media( + mg_app=app, + user=user, + submitted_file=media_file, + filename=filename, + title=maybe_unicodeify(title), + description=maybe_unicodeify(description), + license=maybe_unicodeify(license), + tags_string=u"", + upload_limit=upload_limit, max_file_size=max_file_size) + print "Successfully uploading {filename}!".format(filename=filename) + print "" + except FileUploadLimit: + print "This file is larger than the upload limits for this site." + except UserUploadLimit: + print "This file will put this user past their upload limits." + except UserPastUploadLimit: + print "This user is already past their upload limits." + teardown(temp_files) + + + def parse_csv_file(file_contents): list_of_contents = file_contents.split('\n') key, lines = (list_of_contents[0].split(','), list_of_contents[1:]) - list_of_objects = [] + objects_dict = {} # Build a dictionary for line in lines: if line.isspace() or line == '': continue - values = csv.reader([line]).next() - new_dict = dict([(key[i], val) + values = csv_reader([line]).next() + line_dict = dict([(key[i], val) for i, val in enumerate(values)]) - list_of_objects.append(new_dict) + media_id = line_dict['media:id'] + objects_dict[media_id] = (line_dict) - return list_of_objects + return objects_dict - +def teardown(temp_files): + for temp_file in temp_files: + subprocess.call(['rm','-r',temp_file]) From 27b7d94896cd3cede2050b62af1321ad69cd3fa1 Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Thu, 13 Feb 2014 13:57:10 -0500 Subject: [PATCH 05/78] Minor change in the wording of argparsing. --- mediagoblin/gmg_commands/batchaddmedia.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mediagoblin/gmg_commands/batchaddmedia.py b/mediagoblin/gmg_commands/batchaddmedia.py index 7d7a2d4f..2fd36dfb 100644 --- a/mediagoblin/gmg_commands/batchaddmedia.py +++ b/mediagoblin/gmg_commands/batchaddmedia.py @@ -36,12 +36,12 @@ def parser_setup(subparser): '--directory', action='store_const', const='directory', dest='target_type', default='directory', help=( -"Target is a directory")) +"Choose this option is the target is a directory.")) target_type.add_argument('-a', '--archive', action='store_const', const='archive', dest='target_type', help=( -"Target is an archive.")) +"Choose this option if the target is an archive.")) subparser.add_argument( 'target_path', help=( From 579a6b574f402c23d3b09d22e4ab4c9f71b0e7aa Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Wed, 19 Feb 2014 14:27:14 -0500 Subject: [PATCH 06/78] I made it so the command no longer requires the "Target type" to be provided, it now recognizes whether the target is a directory or an archive on its own. I added in a help message, which is still incomplete, but should make it easier for admins to know how to use this new command. I believe we should also provi- -de an example of the location.csv and metadata.csv files, so there is no conf- -usion. Also, I made it possible for the command to recognize zip files as a valid archive. I also made some minor changes to the commands description w/i the larger gmg command help menu. --- mediagoblin/gmg_commands/__init__.py | 2 +- mediagoblin/gmg_commands/batchaddmedia.py | 55 ++++++++++++++--------- 2 files changed, 34 insertions(+), 23 deletions(-) diff --git a/mediagoblin/gmg_commands/__init__.py b/mediagoblin/gmg_commands/__init__.py index 1460733f..55e85116 100644 --- a/mediagoblin/gmg_commands/__init__.py +++ b/mediagoblin/gmg_commands/__init__.py @@ -56,7 +56,7 @@ SUBCOMMAND_MAP = { 'batchaddmedia': { 'setup': 'mediagoblin.gmg_commands.batchaddmedia:parser_setup', 'func': 'mediagoblin.gmg_commands.batchaddmedia:batchaddmedia', - 'help': 'Reprocess many media entries'} + 'help': 'Add many media entries at once'} # 'theme': { # 'setup': 'mediagoblin.gmg_commands.theme:theme_parser_setup', # 'func': 'mediagoblin.gmg_commands.theme:theme', diff --git a/mediagoblin/gmg_commands/batchaddmedia.py b/mediagoblin/gmg_commands/batchaddmedia.py index 2fd36dfb..d3ab7733 100644 --- a/mediagoblin/gmg_commands/batchaddmedia.py +++ b/mediagoblin/gmg_commands/batchaddmedia.py @@ -15,7 +15,7 @@ # along with this program. If not, see . import os -import json, tempfile, urllib, tarfile, subprocess +import json, tempfile, urllib, tarfile, zipfile, subprocess from csv import reader as csv_reader from urlparse import urlparse from pyld import jsonld @@ -28,29 +28,28 @@ from mediagoblin.submit.lib import ( from mediagoblin import mg_globals def parser_setup(subparser): + subparser.description = """\ +This command allows the administrator to upload many media files at once.""" subparser.add_argument( 'username', - help="Name of user this media entry belongs to") - target_type = subparser.add_mutually_exclusive_group() - target_type.add_argument('-d', - '--directory', action='store_const', - const='directory', dest='target_type', - default='directory', help=( -"Choose this option is the target is a directory.")) - target_type.add_argument('-a', - '--archive', action='store_const', - const='archive', dest='target_type', - help=( -"Choose this option if the target is an archive.")) + help="Name of user these media entries belong to") subparser.add_argument( 'target_path', - help=( -"Path to a local archive or directory containing a location.csv and metadata.csv file")) + help=("""\ +Path to a local archive or directory containing a "location.csv" and a +"metadata.csv" file. These are csv (comma seperated value) files with the +locations and metadata of the files to be uploaded. The location must be listed +with either the URL of the remote media file or the filesystem path of a local +file. The metadata should be provided with one column for each of the 15 Dublin +Core properties (http://dublincore.org/documents/dces/). Both "location.csv" and +"metadata.csv" must begin with a row demonstrating the order of the columns. We +have provided an example of these files at +""")) subparser.add_argument( "-l", "--license", help=( - "License these media entry will be released under, if all the same" - "Should be a URL.")) + "License these media entry will be released under, if all the same. " + "Should be a URL.")) subparser.add_argument( '--celery', action='store_true', @@ -67,26 +66,38 @@ def batchaddmedia(args): # get the user user = app.db.User.query.filter_by(username=args.username.lower()).first() if user is None: - print "Sorry, no user by username '%s'" % args.username + print "Sorry, no user by username '%s' exists" % args.username return upload_limit, max_file_size = get_upload_file_limits(user) temp_files = [] - if args.target_type == 'archive': + if tarfile.is_tarfile(args.target_path): dir_path = tempfile.mkdtemp() temp_files.append(dir_path) tar = tarfile.open(args.target_path) tar.extractall(path=dir_path) - elif args.target_type == 'directory': + elif zipfile.is_zipfile(args.target_path): + dir_path = tempfile.mkdtemp() + temp_files.append(dir_path) + zipped_file = zipfile.ZipFile(args.target_path) + zipped_file.extractall(path=dir_path) + + elif os.path.isdir(args.target_path): dir_path = args.target_path + else: + print "Couldn't recognize the file. This script only accepts tar files,\ +zip files and directories" + if dir_path.endswith('/'): + dir_path = dir_path[:-1] + location_file_path = "{dir_path}/location.csv".format( dir_path=dir_path) metadata_file_path = "{dir_path}/metadata.csv".format( dir_path=dir_path) - + # check for the location file, if it exists... location_filename = os.path.split(location_file_path)[-1] abs_location_filename = os.path.abspath(location_file_path) @@ -178,7 +189,7 @@ def batchaddmedia(args): print "This user is already past their upload limits." teardown(temp_files) - + def parse_csv_file(file_contents): list_of_contents = file_contents.split('\n') From 9d4e9de76b29f8cc602a0db6334e7d36bb3e0fb0 Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Fri, 21 Feb 2014 12:38:02 -0500 Subject: [PATCH 07/78] Changed some of the print messages as well as tweaked the order of the commands attempts to figure out what type of file the target file is. --- mediagoblin/gmg_commands/batchaddmedia.py | 30 +++++++++++++---------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/mediagoblin/gmg_commands/batchaddmedia.py b/mediagoblin/gmg_commands/batchaddmedia.py index d3ab7733..678c8ab4 100644 --- a/mediagoblin/gmg_commands/batchaddmedia.py +++ b/mediagoblin/gmg_commands/batchaddmedia.py @@ -63,6 +63,8 @@ def batchaddmedia(args): app = commands_util.setup_app(args) + files_uploaded, files_attempted = 0, 0 + # get the user user = app.db.User.query.filter_by(username=args.username.lower()).first() if user is None: @@ -72,7 +74,10 @@ def batchaddmedia(args): upload_limit, max_file_size = get_upload_file_limits(user) temp_files = [] - if tarfile.is_tarfile(args.target_path): + if os.path.isdir(args.target_path): + dir_path = args.target_path + + elif tarfile.is_tarfile(args.target_path): dir_path = tempfile.mkdtemp() temp_files.append(dir_path) tar = tarfile.open(args.target_path) @@ -84,9 +89,6 @@ def batchaddmedia(args): zipped_file = zipfile.ZipFile(args.target_path) zipped_file.extractall(path=dir_path) - elif os.path.isdir(args.target_path): - dir_path = args.target_path - else: print "Couldn't recognize the file. This script only accepts tar files,\ zip files and directories" @@ -141,11 +143,9 @@ zip files and directories" description = file_metadata.get('dcterms:description') license = file_metadata.get('dcterms:license') filename = url.path.split()[-1] - print "Working with {filename}".format(filename=filename) + files_attempted += 1 if url.scheme == 'http': - print "Downloading {filename}...".format( - filename=filename) media_file = tempfile.TemporaryFile() res = urllib.urlopen(url.geturl()) media_file.write(res.read()) @@ -163,11 +163,10 @@ zip files and directories" try: media_file = file(file_abs_path, 'r') except IOError: - print "Local file {filename} could not be accessed.".format( - filename=filename) + print "\ +FAIL: Local file {filename} could not be accessed.".format(filename=filename) print "Skipping it." continue - print "Submitting {filename}...".format(filename=filename) try: submit_media( mg_app=app, @@ -181,12 +180,17 @@ zip files and directories" upload_limit=upload_limit, max_file_size=max_file_size) print "Successfully uploading {filename}!".format(filename=filename) print "" + files_uploaded += 1 except FileUploadLimit: - print "This file is larger than the upload limits for this site." + print "FAIL: This file is larger than the upload limits for this site." except UserUploadLimit: - print "This file will put this user past their upload limits." + print "FAIL: This file will put this user past their upload limits." except UserPastUploadLimit: - print "This user is already past their upload limits." + print "FAIL: This user is already past their upload limits." + print "\ +{files_uploaded} out of {files_attempted} files successfully uploaded".format( + files_uploaded=files_uploaded, + files_attempted=files_attempted) teardown(temp_files) From 6b43a6f432b57c0f54427d65de361adc63388799 Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Thu, 27 Mar 2014 13:31:04 -0400 Subject: [PATCH 08/78] Began work on metadata validation --- mediagoblin/gmg_commands/batchaddmedia.py | 33 +++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/mediagoblin/gmg_commands/batchaddmedia.py b/mediagoblin/gmg_commands/batchaddmedia.py index 678c8ab4..83aea7b7 100644 --- a/mediagoblin/gmg_commands/batchaddmedia.py +++ b/mediagoblin/gmg_commands/batchaddmedia.py @@ -26,6 +26,7 @@ from mediagoblin.submit.lib import ( FileUploadLimit, UserUploadLimit, UserPastUploadLimit) from mediagoblin import mg_globals +from jsonschema import validate def parser_setup(subparser): subparser.description = """\ @@ -215,3 +216,35 @@ def parse_csv_file(file_contents): def teardown(temp_files): for temp_file in temp_files: subprocess.call(['rm','-r',temp_file]) + +def check_metadata_format(metadata_dict): + schema = json.loads(""" +{ + "$schema":"http://json-schema.org/schema#", + "properties":{ + "@context":{}, + "contributor":{}, + "coverage":{}, + "created":{}, + "creator":{}, + "date":{}, + "description":{}, + "format":{}, + "identifier":{}, + "language":{}, + "publisher":{}, + "relation":{}, + "rights" : { + "format":"uri", + "type":"string" + }, + "source":{}, + "subject":{}, + "title":{}, + "type":{} + }, + "additionalProperties": false, + "required":["title","@context"] +}""") + try: + validate(metadata_dict, schema) From 680faaaa855a5fa60178d6b2a7e562619d3a4c4b Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Thu, 27 Mar 2014 13:55:15 -0400 Subject: [PATCH 09/78] Added exception handling into the metadata format checking function. --- mediagoblin/gmg_commands/batchaddmedia.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/mediagoblin/gmg_commands/batchaddmedia.py b/mediagoblin/gmg_commands/batchaddmedia.py index 83aea7b7..f06bc2e8 100644 --- a/mediagoblin/gmg_commands/batchaddmedia.py +++ b/mediagoblin/gmg_commands/batchaddmedia.py @@ -24,9 +24,11 @@ from mediagoblin.gmg_commands import util as commands_util from mediagoblin.submit.lib import ( submit_media, get_upload_file_limits, FileUploadLimit, UserUploadLimit, UserPastUploadLimit) +from mediagoblin.tools.translate import lazy_pass_to_ugettext as _ from mediagoblin import mg_globals -from jsonschema import validate +from jsonschema import validate +from jsonschema.exceptions import ValidationError def parser_setup(subparser): subparser.description = """\ @@ -135,7 +137,10 @@ zip files and directories" dcterms_context = { 'dcterms':'http://purl.org/dc/terms/' } for media_id in media_locations.keys(): - file_metadata = media_metadata[media_id] + file_metadata = media_metadata[media_id] + santized_metadata = check_metadata_format(file_metadata) + if sanitized_metadata == {}: continue + json_ld_metadata = jsonld.compact(file_metadata, dcterms_context) original_location = media_locations[media_id]['media:original'] url = urlparse(original_location) @@ -248,3 +253,14 @@ def check_metadata_format(metadata_dict): }""") try: validate(metadata_dict, schema) + output_dict = metadata_dict + except ValidationError, exc: + title = metadata_dict.get('title') or metadata_dict.get('media:id') or \ + _(u'UNKNOWN FILE') + print _( +u"""WARN: Could not find appropriate metadata for file {title}. File will be +skipped""".format(title=title)) + output_dict = {} + except: + raise + return output_dict From 8f054a6b99a594da36a859f7bb5f11464c1602bd Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Thu, 27 Mar 2014 14:11:12 -0400 Subject: [PATCH 10/78] Fixed up some fatal errors. Is still not ready. --- mediagoblin/gmg_commands/batchaddmedia.py | 46 ++++++++++++----------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/mediagoblin/gmg_commands/batchaddmedia.py b/mediagoblin/gmg_commands/batchaddmedia.py index f06bc2e8..414e969c 100644 --- a/mediagoblin/gmg_commands/batchaddmedia.py +++ b/mediagoblin/gmg_commands/batchaddmedia.py @@ -138,7 +138,7 @@ zip files and directories" for media_id in media_locations.keys(): file_metadata = media_metadata[media_id] - santized_metadata = check_metadata_format(file_metadata) + sanitized_metadata = check_metadata_format(file_metadata) if sanitized_metadata == {}: continue json_ld_metadata = jsonld.compact(file_metadata, dcterms_context) @@ -207,7 +207,7 @@ def parse_csv_file(file_contents): list_of_contents[1:]) objects_dict = {} - # Build a dictionary + # Build a dictionaryfrom mediagoblin.tools.translate import lazy_pass_to_ugettext as _ for line in lines: if line.isspace() or line == '': continue values = csv_reader([line]).next() @@ -228,38 +228,40 @@ def check_metadata_format(metadata_dict): "$schema":"http://json-schema.org/schema#", "properties":{ "@context":{}, - "contributor":{}, - "coverage":{}, - "created":{}, - "creator":{}, - "date":{}, - "description":{}, - "format":{}, - "identifier":{}, - "language":{}, - "publisher":{}, - "relation":{}, - "rights" : { + "dcterms:contributor":{}, + "dcterms:coverage":{}, + "dcterms:created":{}, + "dcterms:creator":{}, + "dcterms:date":{}, + "dcterms:description":{}, + "dcterms:format":{}, + "dcterms:identifier":{}, + "dcterms:language":{}, + "dcterms:publisher":{}, + "dcterms:relation":{}, + "dcterms:rights" : { "format":"uri", "type":"string" }, - "source":{}, - "subject":{}, - "title":{}, - "type":{} + "dcterms:source":{}, + "dcterms:subject":{}, + "dcterms:title":{}, + "dcterms:type":{}, + "media:id":{} }, "additionalProperties": false, - "required":["title","@context"] + "required":["dcterms:title","@context","media:id"] }""") + metadata_dict["@context"] = u"http://127.0.0.1:6543/metadata_context/v1/" try: validate(metadata_dict, schema) output_dict = metadata_dict except ValidationError, exc: - title = metadata_dict.get('title') or metadata_dict.get('media:id') or \ + title = metadata_dict.get('dcterms:title') or metadata_dict.get('media:id') or \ _(u'UNKNOWN FILE') print _( -u"""WARN: Could not find appropriate metadata for file {title}. File will be -skipped""".format(title=title)) +u"""WARN: Could not find appropriate metadata for file {title}. +File will be skipped""".format(title=title)) output_dict = {} except: raise From 32aec1e533e9de8b843e54d5a08b55d26e81f87e Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Thu, 27 Mar 2014 17:10:31 -0400 Subject: [PATCH 11/78] Fixed a minor error in the batch upload script and modified the json-ld context. --- mediagoblin/gmg_commands/batchaddmedia.py | 9 ++--- .../mediagoblin/metadata_contexts/v1 | 34 +++++++++---------- 2 files changed, 22 insertions(+), 21 deletions(-) diff --git a/mediagoblin/gmg_commands/batchaddmedia.py b/mediagoblin/gmg_commands/batchaddmedia.py index 414e969c..012a5ee4 100644 --- a/mediagoblin/gmg_commands/batchaddmedia.py +++ b/mediagoblin/gmg_commands/batchaddmedia.py @@ -137,6 +137,8 @@ zip files and directories" dcterms_context = { 'dcterms':'http://purl.org/dc/terms/' } for media_id in media_locations.keys(): + files_attempted += 1 + file_metadata = media_metadata[media_id] sanitized_metadata = check_metadata_format(file_metadata) if sanitized_metadata == {}: continue @@ -149,7 +151,6 @@ zip files and directories" description = file_metadata.get('dcterms:description') license = file_metadata.get('dcterms:license') filename = url.path.split()[-1] - files_attempted += 1 if url.scheme == 'http': media_file = tempfile.TemporaryFile() @@ -228,6 +229,7 @@ def check_metadata_format(metadata_dict): "$schema":"http://json-schema.org/schema#", "properties":{ "@context":{}, + "dcterms:contributor":{}, "dcterms:coverage":{}, "dcterms:created":{}, @@ -246,8 +248,7 @@ def check_metadata_format(metadata_dict): "dcterms:source":{}, "dcterms:subject":{}, "dcterms:title":{}, - "dcterms:type":{}, - "media:id":{} + "dcterms:type":{} }, "additionalProperties": false, "required":["dcterms:title","@context","media:id"] @@ -260,7 +261,7 @@ def check_metadata_format(metadata_dict): title = metadata_dict.get('dcterms:title') or metadata_dict.get('media:id') or \ _(u'UNKNOWN FILE') print _( -u"""WARN: Could not find appropriate metadata for file {title}. +u"""WARN: Could not find appropriate metadata for file "{title}". File will be skipped""".format(title=title)) output_dict = {} except: diff --git a/mediagoblin/templates/mediagoblin/metadata_contexts/v1 b/mediagoblin/templates/mediagoblin/metadata_contexts/v1 index 1325d920..99882de2 100644 --- a/mediagoblin/templates/mediagoblin/metadata_contexts/v1 +++ b/mediagoblin/templates/mediagoblin/metadata_contexts/v1 @@ -1,69 +1,69 @@ { "@context": { - "dc": "http://purl.org/dc/elements/1.1/", + "dcterms": "http://purl.org/dc/elements/1.1/", "xsd": "http://www.w3.org/2001/XMLSchema#", "contributor":{ - "@id":"dc:title", + "@id":"dcterms:title", "@type":"xsd:string" }, "coverage":{ - "@id":"dc:coverage", + "@id":"dcterms:coverage", "@type":"xsd:string" }, "created":{ - "@id":"dc:created", + "@id":"dcterms:created", "@type":"xsd:date" }, "creator":{ - "@id":"dc:created", + "@id":"dcterms:created", "@type":"xsd:date" }, "date":{ - "@id":"dc:date", + "@id":"dcterms:date", "@type":"xsd:date" }, "description":{ - "@id":"dc:description", + "@id":"dcterms:description", "@type":"xsd:string" }, "format":{ - "@id":"dc:format", + "@id":"dcterms:format", "@type":"xsd:string" }, "identifier":{ - "@id":"dc:identifier", + "@id":"dcterms:identifier", "@type":"xsd:string" }, "language":{ - "@id":"dc:language", + "@id":"dcterms:language", "@type":"xsd:string" }, "publisher":{ - "@id":"dc:publisher", + "@id":"dcterms:publisher", "@type":"xsd:string" }, "relation":{ - "@id":"dc:relation", + "@id":"dcterms:relation", "@type":"xsd:string" }, "rights":{ - "@id":"dc:rights", + "@id":"dcterms:rights", "@type":"xsd:anyURI" }, "source":{ - "@id":"dc:source", + "@id":"dcterms:source", "@type":"xsd:string" }, "subject":{ - "@id":"dc:subject", + "@id":"dcterms:subject", "@type":"xsd:string" }, "title": { - "@id":"dc:title", + "@id":"dcterms:title", "@type":"xsd:string" }, "type":{ - "@id":"dc:type", + "@id":"dcterms:type", "@type":"xsd:string" } } From 0e4144abaf2dc6a18bc2750808e4561caf4e5e9c Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Thu, 27 Mar 2014 17:29:34 -0400 Subject: [PATCH 12/78] Wrote more comprehensive error messages. --- mediagoblin/gmg_commands/batchaddmedia.py | 28 +++++++++++++++++------ 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/mediagoblin/gmg_commands/batchaddmedia.py b/mediagoblin/gmg_commands/batchaddmedia.py index 012a5ee4..fe345d5f 100644 --- a/mediagoblin/gmg_commands/batchaddmedia.py +++ b/mediagoblin/gmg_commands/batchaddmedia.py @@ -229,7 +229,7 @@ def check_metadata_format(metadata_dict): "$schema":"http://json-schema.org/schema#", "properties":{ "@context":{}, - + "media:id":{}, "dcterms:contributor":{}, "dcterms:coverage":{}, "dcterms:created":{}, @@ -251,18 +251,32 @@ def check_metadata_format(metadata_dict): "dcterms:type":{} }, "additionalProperties": false, - "required":["dcterms:title","@context","media:id"] + "required":["dcterms:title","@context","media:id","bell"] }""") metadata_dict["@context"] = u"http://127.0.0.1:6543/metadata_context/v1/" try: validate(metadata_dict, schema) output_dict = metadata_dict except ValidationError, exc: - title = metadata_dict.get('dcterms:title') or metadata_dict.get('media:id') or \ - _(u'UNKNOWN FILE') - print _( -u"""WARN: Could not find appropriate metadata for file "{title}". -File will be skipped""".format(title=title)) + title = (metadata_dict.get('dcterms:title') or + metadata_dict.get('media:id') or _(u'UNKNOWN FILE')) + + if exc.validator == "additionalProperties": + message = _(u'Invalid metadata provided for file "{title}". This \ +script only accepts the Dublin Core metadata terms.'.format(title=title)) + + elif exc.validator == "required": + message = _( +u'All necessary metadata was not provided for file "{title}", you must include \ +a "dcterms:title" column for each media file'.format(title=title)) + + else: + message = _(u'Could not find appropriate metadata for file \ +"{title}".'.format(title=title)) + + print _(u"""WARN: {message} \nSkipping File...\n""".format( + message=message)) + output_dict = {} except: raise From 8e33666813b7ab5f46746a1c294c8e5baa6b08ef Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Thu, 3 Apr 2014 12:18:17 -0400 Subject: [PATCH 13/78] Fixed a bad get of 'dcterms:rights' and am throwing away the idea of an external context file for the json-ld because it feels unnecessary seeing as we are just using the dc core terms --- mediagoblin/gmg_commands/batchaddmedia.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/mediagoblin/gmg_commands/batchaddmedia.py b/mediagoblin/gmg_commands/batchaddmedia.py index fe345d5f..68993aa2 100644 --- a/mediagoblin/gmg_commands/batchaddmedia.py +++ b/mediagoblin/gmg_commands/batchaddmedia.py @@ -15,7 +15,7 @@ # along with this program. If not, see . import os -import json, tempfile, urllib, tarfile, zipfile, subprocess +import tempfile, urllib, tarfile, zipfile, subprocess from csv import reader as csv_reader from urlparse import urlparse from pyld import jsonld @@ -149,7 +149,7 @@ zip files and directories" title = file_metadata.get('dcterms:title') description = file_metadata.get('dcterms:description') - license = file_metadata.get('dcterms:license') + license = file_metadata.get('dcterms:rights') filename = url.path.split()[-1] if url.scheme == 'http': @@ -201,7 +201,6 @@ FAIL: Local file {filename} could not be accessed.".format(filename=filename) teardown(temp_files) - def parse_csv_file(file_contents): list_of_contents = file_contents.split('\n') key, lines = (list_of_contents[0].split(','), @@ -219,16 +218,16 @@ def parse_csv_file(file_contents): return objects_dict + def teardown(temp_files): for temp_file in temp_files: subprocess.call(['rm','-r',temp_file]) + def check_metadata_format(metadata_dict): - schema = json.loads(""" -{ + schema = { "$schema":"http://json-schema.org/schema#", "properties":{ - "@context":{}, "media:id":{}, "dcterms:contributor":{}, "dcterms:coverage":{}, @@ -250,13 +249,14 @@ def check_metadata_format(metadata_dict): "dcterms:title":{}, "dcterms:type":{} }, - "additionalProperties": false, - "required":["dcterms:title","@context","media:id","bell"] -}""") - metadata_dict["@context"] = u"http://127.0.0.1:6543/metadata_context/v1/" + "additionalProperties": False, + "required":["dcterms:title","media:id"] +} try: validate(metadata_dict, schema) output_dict = metadata_dict + del output_dict['media:id'] + except ValidationError, exc: title = (metadata_dict.get('dcterms:title') or metadata_dict.get('media:id') or _(u'UNKNOWN FILE')) @@ -280,4 +280,5 @@ a "dcterms:title" column for each media file'.format(title=title)) output_dict = {} except: raise + return output_dict From fb60426ed1263de092ebc27afb96175d55ae7095 Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Thu, 3 Apr 2014 12:20:30 -0400 Subject: [PATCH 14/78] Took out all of the references to the temporary url I was using /metadata_context/v1 --- mediagoblin/routing.py | 2 - .../mediagoblin/metadata_contexts/v1 | 70 ------------------- mediagoblin/views.py | 6 -- 3 files changed, 78 deletions(-) delete mode 100644 mediagoblin/templates/mediagoblin/metadata_contexts/v1 diff --git a/mediagoblin/routing.py b/mediagoblin/routing.py index a6b2a543..9f2584d3 100644 --- a/mediagoblin/routing.py +++ b/mediagoblin/routing.py @@ -29,8 +29,6 @@ def get_url_map(): add_route('index', '/', 'mediagoblin.views:root_view') add_route('terms_of_service','/terms_of_service', 'mediagoblin.views:terms_of_service'), - add_route('metadata_context','/metadata_context/v/', - 'mediagoblin.views:metadata_context_view'), mount('/auth', auth_routes) mount('/mod', moderation_routes) diff --git a/mediagoblin/templates/mediagoblin/metadata_contexts/v1 b/mediagoblin/templates/mediagoblin/metadata_contexts/v1 deleted file mode 100644 index 99882de2..00000000 --- a/mediagoblin/templates/mediagoblin/metadata_contexts/v1 +++ /dev/null @@ -1,70 +0,0 @@ -{ - "@context": { - "dcterms": "http://purl.org/dc/elements/1.1/", - "xsd": "http://www.w3.org/2001/XMLSchema#", - "contributor":{ - "@id":"dcterms:title", - "@type":"xsd:string" - }, - "coverage":{ - "@id":"dcterms:coverage", - "@type":"xsd:string" - }, - "created":{ - "@id":"dcterms:created", - "@type":"xsd:date" - }, - "creator":{ - "@id":"dcterms:created", - "@type":"xsd:date" - }, - "date":{ - "@id":"dcterms:date", - "@type":"xsd:date" - }, - "description":{ - "@id":"dcterms:description", - "@type":"xsd:string" - }, - "format":{ - "@id":"dcterms:format", - "@type":"xsd:string" - }, - "identifier":{ - "@id":"dcterms:identifier", - "@type":"xsd:string" - }, - "language":{ - "@id":"dcterms:language", - "@type":"xsd:string" - }, - "publisher":{ - "@id":"dcterms:publisher", - "@type":"xsd:string" - }, - "relation":{ - "@id":"dcterms:relation", - "@type":"xsd:string" - }, - "rights":{ - "@id":"dcterms:rights", - "@type":"xsd:anyURI" - }, - "source":{ - "@id":"dcterms:source", - "@type":"xsd:string" - }, - "subject":{ - "@id":"dcterms:subject", - "@type":"xsd:string" - }, - "title": { - "@id":"dcterms:title", - "@type":"xsd:string" - }, - "type":{ - "@id":"dcterms:type", - "@type":"xsd:string" - } - } -} diff --git a/mediagoblin/views.py b/mediagoblin/views.py index 1ed71473..009e48e4 100644 --- a/mediagoblin/views.py +++ b/mediagoblin/views.py @@ -62,9 +62,3 @@ def terms_of_service(request): return render_to_response(request, 'mediagoblin/terms_of_service.html', {}) - -def metadata_context_view(request): - version = request.matchdict['version_number'] - return render_to_response(request, - 'mediagoblin/metadata_contexts/v{version}'.format( - version=version), {}) From 6fa9b06f9a7d9f33b2e891ff615395dfbb20c18e Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Fri, 11 Apr 2014 13:06:09 -0400 Subject: [PATCH 15/78] Fixed incorrectly coded references to filesystem paths --- mediagoblin/gmg_commands/batchaddmedia.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/mediagoblin/gmg_commands/batchaddmedia.py b/mediagoblin/gmg_commands/batchaddmedia.py index 68993aa2..b058a47e 100644 --- a/mediagoblin/gmg_commands/batchaddmedia.py +++ b/mediagoblin/gmg_commands/batchaddmedia.py @@ -98,10 +98,8 @@ zip files and directories" if dir_path.endswith('/'): dir_path = dir_path[:-1] - location_file_path = "{dir_path}/location.csv".format( - dir_path=dir_path) - metadata_file_path = "{dir_path}/metadata.csv".format( - dir_path=dir_path) + location_file_path = os.path.join(dir_path,"location.csv") + metadata_file_path = os.path.join(dir_path, "metadata.csv") # check for the location file, if it exists... location_filename = os.path.split(location_file_path)[-1] @@ -163,9 +161,7 @@ zip files and directories" if os.path.isabs(path): file_abs_path = os.path.abspath(path) else: - file_path = "{dir_path}/{local_path}".format( - dir_path=dir_path, - local_path=path) + file_path = os.path.join(dir_path, path) file_abs_path = os.path.abspath(file_path) try: media_file = file(file_abs_path, 'r') From fbb13abe9a22d08c3a2b86245cf02c1363c36d86 Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Tue, 15 Apr 2014 13:35:22 -0400 Subject: [PATCH 16/78] Added the 'requests' library as a dependency and switched over to using it to fetch remote pieces of media in the batchupload script --- mediagoblin/gmg_commands/batchaddmedia.py | 9 ++++----- setup.py | 1 + 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/mediagoblin/gmg_commands/batchaddmedia.py b/mediagoblin/gmg_commands/batchaddmedia.py index b058a47e..deb6c5bd 100644 --- a/mediagoblin/gmg_commands/batchaddmedia.py +++ b/mediagoblin/gmg_commands/batchaddmedia.py @@ -15,9 +15,10 @@ # along with this program. If not, see . import os -import tempfile, urllib, tarfile, zipfile, subprocess +import tempfile, tarfile, zipfile, subprocess, requests from csv import reader as csv_reader from urlparse import urlparse +import requests from pyld import jsonld from mediagoblin.gmg_commands import util as commands_util @@ -151,10 +152,8 @@ zip files and directories" filename = url.path.split()[-1] if url.scheme == 'http': - media_file = tempfile.TemporaryFile() - res = urllib.urlopen(url.geturl()) - media_file.write(res.read()) - media_file.seek(0) + res = requests.get(url.geturl()) + media_file = res.raw elif url.scheme == '': path = url.path diff --git a/setup.py b/setup.py index 93873d73..12739ffd 100644 --- a/setup.py +++ b/setup.py @@ -67,6 +67,7 @@ try: 'oauthlib==0.5.0', 'unidecode', 'jsonschema', + 'requests', ## Annoying. Please remove once we can! We only indirectly ## use pbr, and currently it breaks things, presumably till From b91df79041f59ec87ad2e6f48ca6aa2a78de3c1d Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Tue, 15 Apr 2014 13:51:27 -0400 Subject: [PATCH 17/78] Moved the metadata column to MediaEntry rather than MediaFile --- mediagoblin/db/migrations.py | 8 ++++---- mediagoblin/db/models.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/mediagoblin/db/migrations.py b/mediagoblin/db/migrations.py index a7400bf0..294ab43b 100644 --- a/mediagoblin/db/migrations.py +++ b/mediagoblin/db/migrations.py @@ -723,12 +723,12 @@ def drop_MediaEntry_collected(db): db.commit() @RegisterMigration(20, MIGRATIONS) -def add_work_metadata_column(db): +def add_metadata_column(db): metadata = MetaData(bind=db.bind) - media_file = inspect_table(metadata, 'core__mediafiles') + media_entry = inspect_table(metadata, 'core__media_entries') - col = Column('work_metadata', MutationDict.as_mutable(JSONEncoded)) - col.create(media_file) + col = Column('metadata', MutationDict.as_mutable(JSONEncoded)) + col.create(media_entry) db.commit() diff --git a/mediagoblin/db/models.py b/mediagoblin/db/models.py index ac69d040..7c0f0bf3 100644 --- a/mediagoblin/db/models.py +++ b/mediagoblin/db/models.py @@ -264,6 +264,7 @@ class MediaEntry(Base, MediaEntryMixin): cascade="all, delete-orphan" ) collections = association_proxy("collections_helper", "in_collection") + metadata = Column(MutationDict.as_mutable(JSONEncoded)) ## TODO # fail_error @@ -420,7 +421,6 @@ class MediaFile(Base): name_id = Column(SmallInteger, ForeignKey(FileKeynames.id), nullable=False) file_path = Column(PathTupleWithSlashes) file_metadata = Column(MutationDict.as_mutable(JSONEncoded)) - work_metadata = Column(MutationDict.as_mutable(JSONEncoded)) __table_args__ = ( PrimaryKeyConstraint('media_entry', 'name_id'), From 89b6b55766f71466ec001398b2537569543dc175 Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Tue, 15 Apr 2014 14:17:43 -0400 Subject: [PATCH 18/78] Changed the name of the metadata column --- mediagoblin/db/migrations.py | 2 +- mediagoblin/db/models.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mediagoblin/db/migrations.py b/mediagoblin/db/migrations.py index 294ab43b..8dac3214 100644 --- a/mediagoblin/db/migrations.py +++ b/mediagoblin/db/migrations.py @@ -728,7 +728,7 @@ def add_metadata_column(db): media_entry = inspect_table(metadata, 'core__media_entries') - col = Column('metadata', MutationDict.as_mutable(JSONEncoded)) + col = Column('media_metadata', MutationDict.as_mutable(JSONEncoded)) col.create(media_entry) db.commit() diff --git a/mediagoblin/db/models.py b/mediagoblin/db/models.py index 7c0f0bf3..defa0849 100644 --- a/mediagoblin/db/models.py +++ b/mediagoblin/db/models.py @@ -264,7 +264,7 @@ class MediaEntry(Base, MediaEntryMixin): cascade="all, delete-orphan" ) collections = association_proxy("collections_helper", "in_collection") - metadata = Column(MutationDict.as_mutable(JSONEncoded)) + media_metadata = Column(MutationDict.as_mutable(JSONEncoded)) ## TODO # fail_error From 2dd966b5e2c6c406d153e2d4cdf886e30198a1d3 Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Thu, 6 Feb 2014 15:15:57 -0500 Subject: [PATCH 19/78] In this commit, I added a new column which will be used for RDFa metadata of media. --- mediagoblin/db/migrations.py | 12 ++++++++++++ mediagoblin/db/models.py | 1 + 2 files changed, 13 insertions(+) diff --git a/mediagoblin/db/migrations.py b/mediagoblin/db/migrations.py index 426080a2..a7400bf0 100644 --- a/mediagoblin/db/migrations.py +++ b/mediagoblin/db/migrations.py @@ -31,6 +31,7 @@ from mediagoblin.db.migration_tools import ( RegisterMigration, inspect_table, replace_table_hack) from mediagoblin.db.models import (MediaEntry, Collection, MediaComment, User, Privilege) +from mediagoblin.db.extratypes import JSONEncoded, MutationDict MIGRATIONS = {} @@ -720,3 +721,14 @@ def drop_MediaEntry_collected(db): media_collected.drop() db.commit() + +@RegisterMigration(20, MIGRATIONS) +def add_work_metadata_column(db): + metadata = MetaData(bind=db.bind) + + media_file = inspect_table(metadata, 'core__mediafiles') + + col = Column('work_metadata', MutationDict.as_mutable(JSONEncoded)) + col.create(media_file) + + db.commit() diff --git a/mediagoblin/db/models.py b/mediagoblin/db/models.py index b750375d..ac69d040 100644 --- a/mediagoblin/db/models.py +++ b/mediagoblin/db/models.py @@ -420,6 +420,7 @@ class MediaFile(Base): name_id = Column(SmallInteger, ForeignKey(FileKeynames.id), nullable=False) file_path = Column(PathTupleWithSlashes) file_metadata = Column(MutationDict.as_mutable(JSONEncoded)) + work_metadata = Column(MutationDict.as_mutable(JSONEncoded)) __table_args__ = ( PrimaryKeyConstraint('media_entry', 'name_id'), From ec5a385ada3401d6aab65c334b44ce5492ad032a Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Tue, 18 Mar 2014 16:49:48 -0400 Subject: [PATCH 20/78] Big update. I added in a json-ld context file which will be used in all our metadata columns in the future. The context describes the dublin core elements. It still has not been finalized however. --- mediagoblin/routing.py | 4 +- .../mediagoblin/metadata_contexts/v1 | 70 +++++++++++++++++++ mediagoblin/views.py | 6 ++ setup.py | 2 + 4 files changed, 81 insertions(+), 1 deletion(-) create mode 100644 mediagoblin/templates/mediagoblin/metadata_contexts/v1 diff --git a/mediagoblin/routing.py b/mediagoblin/routing.py index 1393f01c..a6b2a543 100644 --- a/mediagoblin/routing.py +++ b/mediagoblin/routing.py @@ -28,7 +28,9 @@ _log = logging.getLogger(__name__) def get_url_map(): add_route('index', '/', 'mediagoblin.views:root_view') add_route('terms_of_service','/terms_of_service', - 'mediagoblin.views:terms_of_service') + 'mediagoblin.views:terms_of_service'), + add_route('metadata_context','/metadata_context/v/', + 'mediagoblin.views:metadata_context_view'), mount('/auth', auth_routes) mount('/mod', moderation_routes) diff --git a/mediagoblin/templates/mediagoblin/metadata_contexts/v1 b/mediagoblin/templates/mediagoblin/metadata_contexts/v1 new file mode 100644 index 00000000..1325d920 --- /dev/null +++ b/mediagoblin/templates/mediagoblin/metadata_contexts/v1 @@ -0,0 +1,70 @@ +{ + "@context": { + "dc": "http://purl.org/dc/elements/1.1/", + "xsd": "http://www.w3.org/2001/XMLSchema#", + "contributor":{ + "@id":"dc:title", + "@type":"xsd:string" + }, + "coverage":{ + "@id":"dc:coverage", + "@type":"xsd:string" + }, + "created":{ + "@id":"dc:created", + "@type":"xsd:date" + }, + "creator":{ + "@id":"dc:created", + "@type":"xsd:date" + }, + "date":{ + "@id":"dc:date", + "@type":"xsd:date" + }, + "description":{ + "@id":"dc:description", + "@type":"xsd:string" + }, + "format":{ + "@id":"dc:format", + "@type":"xsd:string" + }, + "identifier":{ + "@id":"dc:identifier", + "@type":"xsd:string" + }, + "language":{ + "@id":"dc:language", + "@type":"xsd:string" + }, + "publisher":{ + "@id":"dc:publisher", + "@type":"xsd:string" + }, + "relation":{ + "@id":"dc:relation", + "@type":"xsd:string" + }, + "rights":{ + "@id":"dc:rights", + "@type":"xsd:anyURI" + }, + "source":{ + "@id":"dc:source", + "@type":"xsd:string" + }, + "subject":{ + "@id":"dc:subject", + "@type":"xsd:string" + }, + "title": { + "@id":"dc:title", + "@type":"xsd:string" + }, + "type":{ + "@id":"dc:type", + "@type":"xsd:string" + } + } +} diff --git a/mediagoblin/views.py b/mediagoblin/views.py index 009e48e4..1ed71473 100644 --- a/mediagoblin/views.py +++ b/mediagoblin/views.py @@ -62,3 +62,9 @@ def terms_of_service(request): return render_to_response(request, 'mediagoblin/terms_of_service.html', {}) + +def metadata_context_view(request): + version = request.matchdict['version_number'] + return render_to_response(request, + 'mediagoblin/metadata_contexts/v{version}'.format( + version=version), {}) diff --git a/setup.py b/setup.py index 59f0ab8f..15fd913c 100644 --- a/setup.py +++ b/setup.py @@ -66,8 +66,10 @@ try: 'mock', 'itsdangerous', 'pytz', + 'six>=1.4.1', 'oauthlib==0.5.0', 'unidecode', + 'jsonschema', 'ExifRead', # PLEASE change this when we can; a dependency is forcing us to set this From 8aa015978c970d26992b3b405d3e58401b4b81e2 Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Thu, 6 Feb 2014 15:17:06 -0500 Subject: [PATCH 21/78] This branch will create a commandline bulk-upload script. So far, I have written the code to read csv files into a usable dictionary. --- mediagoblin/gmg_commands/__init__.py | 4 + mediagoblin/gmg_commands/batchaddmedia.py | 110 ++++++++++++++++++++++ 2 files changed, 114 insertions(+) create mode 100644 mediagoblin/gmg_commands/batchaddmedia.py diff --git a/mediagoblin/gmg_commands/__init__.py b/mediagoblin/gmg_commands/__init__.py index a1eb599d..1460733f 100644 --- a/mediagoblin/gmg_commands/__init__.py +++ b/mediagoblin/gmg_commands/__init__.py @@ -53,6 +53,10 @@ SUBCOMMAND_MAP = { 'setup': 'mediagoblin.gmg_commands.addmedia:parser_setup', 'func': 'mediagoblin.gmg_commands.addmedia:addmedia', 'help': 'Reprocess media entries'}, + 'batchaddmedia': { + 'setup': 'mediagoblin.gmg_commands.batchaddmedia:parser_setup', + 'func': 'mediagoblin.gmg_commands.batchaddmedia:batchaddmedia', + 'help': 'Reprocess many media entries'} # 'theme': { # 'setup': 'mediagoblin.gmg_commands.theme:theme_parser_setup', # 'func': 'mediagoblin.gmg_commands.theme:theme', diff --git a/mediagoblin/gmg_commands/batchaddmedia.py b/mediagoblin/gmg_commands/batchaddmedia.py new file mode 100644 index 00000000..1c0f6784 --- /dev/null +++ b/mediagoblin/gmg_commands/batchaddmedia.py @@ -0,0 +1,110 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import os + +from mediagoblin.gmg_commands import util as commands_util +from mediagoblin.submit.lib import ( + submit_media, get_upload_file_limits, + FileUploadLimit, UserUploadLimit, UserPastUploadLimit) + +from mediagoblin import mg_globals +import json, csv + +def parser_setup(subparser): + subparser.add_argument( + 'username', + help="Name of user this media entry belongs to") + subparser.add_argument( + 'locationfile', + help=( +"Local file on filesystem with the address of all the files to be uploaded")) + subparser.add_argument( + 'metadatafile', + help=( +"Local file on filesystem with metadata of all the files to be uploaded")) + subparser.add_argument( + "-l", "--license", + help=( + "License these media entry will be released under, if all the same" + "Should be a URL.")) + subparser.add_argument( + '--celery', + action='store_true', + help="Don't process eagerly, pass off to celery") + + +def batchaddmedia(args): + # Run eagerly unless explicetly set not to + if not args.celery: + os.environ['CELERY_ALWAYS_EAGER'] = 'true' + + app = commands_util.setup_app(args) + + # get the user + user = app.db.User.query.filter_by(username=args.username.lower()).first() + if user is None: + print "Sorry, no user by username '%s'" % args.username + return + + # check for the location file, if it exists... + location_filename = os.path.split(args.locationfile)[-1] + abs_location_filename = os.path.abspath(args.locationfile) + if not os.path.exists(abs_location_filename): + print "Can't find a file with filename '%s'" % args.locationfile + return + + # check for the location file, if it exists... + metadata_filename = os.path.split(args.metadatafile)[-1] + abs_metadata_filename = os.path.abspath(args.metadatafile) + if not os.path.exists(abs_metadata_filename): + print "Can't find a file with filename '%s'" % args.metadatafile + return + + upload_limit, max_file_size = get_upload_file_limits(user) + + def maybe_unicodeify(some_string): + # this is kinda terrible + if some_string is None: + return None + else: + return unicode(some_string) + + with file(abs_location_filename, 'r') as all_locations: + contents = all_locations.read() + media_locations = parse_csv_file(contents) + + with file(abs_metadata_filename, 'r') as all_metadata: + contents = all_metadata.read() + media_metadata = parse_csv_file(contents) + +def parse_csv_file(file_contents): + list_of_contents = file_contents.split('\n') + key, lines = (list_of_contents[0].split(','), + list_of_contents[1:]) + list_of_objects = [] + + # Build a dictionary + for line in lines: + if line.isspace() or line == '': continue + values = csv.reader([line]).next() + new_dict = dict([(key[i], val) + for i, val in enumerate(values)]) + list_of_objects.append(new_dict) + + return list_of_objects + + From 268f243074fd4cd97017062e4a9e9afd0a860b32 Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Wed, 12 Feb 2014 14:37:00 -0500 Subject: [PATCH 22/78] The script now officially works! It works in many different situations, whether the media is to be uploaded is stored locally or on the web. Still have to clean up the code and look for errors. I may also refactor some of this into a functi- on to be used with a GUI frontend in another project. Lastly, I need to merge this with the metadata branch I've been working on, and convert the metadata.csv information into the proper format for the new metadata column. --- mediagoblin/gmg_commands/batchaddmedia.py | 130 ++++++++++++++++++---- 1 file changed, 111 insertions(+), 19 deletions(-) diff --git a/mediagoblin/gmg_commands/batchaddmedia.py b/mediagoblin/gmg_commands/batchaddmedia.py index 1c0f6784..7d7a2d4f 100644 --- a/mediagoblin/gmg_commands/batchaddmedia.py +++ b/mediagoblin/gmg_commands/batchaddmedia.py @@ -15,6 +15,10 @@ # along with this program. If not, see . import os +import json, tempfile, urllib, tarfile, subprocess +from csv import reader as csv_reader +from urlparse import urlparse +from pyld import jsonld from mediagoblin.gmg_commands import util as commands_util from mediagoblin.submit.lib import ( @@ -22,20 +26,26 @@ from mediagoblin.submit.lib import ( FileUploadLimit, UserUploadLimit, UserPastUploadLimit) from mediagoblin import mg_globals -import json, csv def parser_setup(subparser): subparser.add_argument( 'username', help="Name of user this media entry belongs to") - subparser.add_argument( - 'locationfile', + target_type = subparser.add_mutually_exclusive_group() + target_type.add_argument('-d', + '--directory', action='store_const', + const='directory', dest='target_type', + default='directory', help=( +"Target is a directory")) + target_type.add_argument('-a', + '--archive', action='store_const', + const='archive', dest='target_type', help=( -"Local file on filesystem with the address of all the files to be uploaded")) +"Target is an archive.")) subparser.add_argument( - 'metadatafile', + 'target_path', help=( -"Local file on filesystem with metadata of all the files to be uploaded")) +"Path to a local archive or directory containing a location.csv and metadata.csv file")) subparser.add_argument( "-l", "--license", help=( @@ -59,19 +69,36 @@ def batchaddmedia(args): if user is None: print "Sorry, no user by username '%s'" % args.username return + + upload_limit, max_file_size = get_upload_file_limits(user) + temp_files = [] + + if args.target_type == 'archive': + dir_path = tempfile.mkdtemp() + temp_files.append(dir_path) + tar = tarfile.open(args.target_path) + tar.extractall(path=dir_path) + + elif args.target_type == 'directory': + dir_path = args.target_path + + location_file_path = "{dir_path}/location.csv".format( + dir_path=dir_path) + metadata_file_path = "{dir_path}/metadata.csv".format( + dir_path=dir_path) # check for the location file, if it exists... - location_filename = os.path.split(args.locationfile)[-1] - abs_location_filename = os.path.abspath(args.locationfile) + location_filename = os.path.split(location_file_path)[-1] + abs_location_filename = os.path.abspath(location_file_path) if not os.path.exists(abs_location_filename): - print "Can't find a file with filename '%s'" % args.locationfile + print "Can't find a file with filename '%s'" % location_file_path return - # check for the location file, if it exists... - metadata_filename = os.path.split(args.metadatafile)[-1] - abs_metadata_filename = os.path.abspath(args.metadatafile) + # check for the metadata file, if it exists... + metadata_filename = os.path.split(metadata_file_path)[-1] + abs_metadata_filename = os.path.abspath(metadata_file_path) if not os.path.exists(abs_metadata_filename): - print "Can't find a file with filename '%s'" % args.metadatafile + print "Can't find a file with filename '%s'" % metadata_file_path return upload_limit, max_file_size = get_upload_file_limits(user) @@ -91,20 +118,85 @@ def batchaddmedia(args): contents = all_metadata.read() media_metadata = parse_csv_file(contents) + dcterms_context = { 'dcterms':'http://purl.org/dc/terms/' } + + for media_id in media_locations.keys(): + file_metadata = media_metadata[media_id] + json_ld_metadata = jsonld.compact(file_metadata, dcterms_context) + original_location = media_locations[media_id]['media:original'] + url = urlparse(original_location) + + title = file_metadata.get('dcterms:title') + description = file_metadata.get('dcterms:description') + license = file_metadata.get('dcterms:license') + filename = url.path.split()[-1] + print "Working with {filename}".format(filename=filename) + + if url.scheme == 'http': + print "Downloading {filename}...".format( + filename=filename) + media_file = tempfile.TemporaryFile() + res = urllib.urlopen(url.geturl()) + media_file.write(res.read()) + media_file.seek(0) + + elif url.scheme == '': + path = url.path + if os.path.isabs(path): + file_abs_path = os.path.abspath(path) + else: + file_path = "{dir_path}/{local_path}".format( + dir_path=dir_path, + local_path=path) + file_abs_path = os.path.abspath(file_path) + try: + media_file = file(file_abs_path, 'r') + except IOError: + print "Local file {filename} could not be accessed.".format( + filename=filename) + print "Skipping it." + continue + print "Submitting {filename}...".format(filename=filename) + try: + submit_media( + mg_app=app, + user=user, + submitted_file=media_file, + filename=filename, + title=maybe_unicodeify(title), + description=maybe_unicodeify(description), + license=maybe_unicodeify(license), + tags_string=u"", + upload_limit=upload_limit, max_file_size=max_file_size) + print "Successfully uploading {filename}!".format(filename=filename) + print "" + except FileUploadLimit: + print "This file is larger than the upload limits for this site." + except UserUploadLimit: + print "This file will put this user past their upload limits." + except UserPastUploadLimit: + print "This user is already past their upload limits." + teardown(temp_files) + + + def parse_csv_file(file_contents): list_of_contents = file_contents.split('\n') key, lines = (list_of_contents[0].split(','), list_of_contents[1:]) - list_of_objects = [] + objects_dict = {} # Build a dictionary for line in lines: if line.isspace() or line == '': continue - values = csv.reader([line]).next() - new_dict = dict([(key[i], val) + values = csv_reader([line]).next() + line_dict = dict([(key[i], val) for i, val in enumerate(values)]) - list_of_objects.append(new_dict) + media_id = line_dict['media:id'] + objects_dict[media_id] = (line_dict) - return list_of_objects + return objects_dict - +def teardown(temp_files): + for temp_file in temp_files: + subprocess.call(['rm','-r',temp_file]) From 6c37aeaa33aa21d6937e392185df4d0f6bffef7a Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Thu, 13 Feb 2014 13:57:10 -0500 Subject: [PATCH 23/78] Minor change in the wording of argparsing. --- mediagoblin/gmg_commands/batchaddmedia.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mediagoblin/gmg_commands/batchaddmedia.py b/mediagoblin/gmg_commands/batchaddmedia.py index 7d7a2d4f..2fd36dfb 100644 --- a/mediagoblin/gmg_commands/batchaddmedia.py +++ b/mediagoblin/gmg_commands/batchaddmedia.py @@ -36,12 +36,12 @@ def parser_setup(subparser): '--directory', action='store_const', const='directory', dest='target_type', default='directory', help=( -"Target is a directory")) +"Choose this option is the target is a directory.")) target_type.add_argument('-a', '--archive', action='store_const', const='archive', dest='target_type', help=( -"Target is an archive.")) +"Choose this option if the target is an archive.")) subparser.add_argument( 'target_path', help=( From 28ecc53a5aed89b5d567e0245e06aa5f34bd371d Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Wed, 19 Feb 2014 14:27:14 -0500 Subject: [PATCH 24/78] I made it so the command no longer requires the "Target type" to be provided, it now recognizes whether the target is a directory or an archive on its own. I added in a help message, which is still incomplete, but should make it easier for admins to know how to use this new command. I believe we should also provi- -de an example of the location.csv and metadata.csv files, so there is no conf- -usion. Also, I made it possible for the command to recognize zip files as a valid archive. I also made some minor changes to the commands description w/i the larger gmg command help menu. --- mediagoblin/gmg_commands/__init__.py | 2 +- mediagoblin/gmg_commands/batchaddmedia.py | 55 ++++++++++++++--------- 2 files changed, 34 insertions(+), 23 deletions(-) diff --git a/mediagoblin/gmg_commands/__init__.py b/mediagoblin/gmg_commands/__init__.py index 1460733f..55e85116 100644 --- a/mediagoblin/gmg_commands/__init__.py +++ b/mediagoblin/gmg_commands/__init__.py @@ -56,7 +56,7 @@ SUBCOMMAND_MAP = { 'batchaddmedia': { 'setup': 'mediagoblin.gmg_commands.batchaddmedia:parser_setup', 'func': 'mediagoblin.gmg_commands.batchaddmedia:batchaddmedia', - 'help': 'Reprocess many media entries'} + 'help': 'Add many media entries at once'} # 'theme': { # 'setup': 'mediagoblin.gmg_commands.theme:theme_parser_setup', # 'func': 'mediagoblin.gmg_commands.theme:theme', diff --git a/mediagoblin/gmg_commands/batchaddmedia.py b/mediagoblin/gmg_commands/batchaddmedia.py index 2fd36dfb..d3ab7733 100644 --- a/mediagoblin/gmg_commands/batchaddmedia.py +++ b/mediagoblin/gmg_commands/batchaddmedia.py @@ -15,7 +15,7 @@ # along with this program. If not, see . import os -import json, tempfile, urllib, tarfile, subprocess +import json, tempfile, urllib, tarfile, zipfile, subprocess from csv import reader as csv_reader from urlparse import urlparse from pyld import jsonld @@ -28,29 +28,28 @@ from mediagoblin.submit.lib import ( from mediagoblin import mg_globals def parser_setup(subparser): + subparser.description = """\ +This command allows the administrator to upload many media files at once.""" subparser.add_argument( 'username', - help="Name of user this media entry belongs to") - target_type = subparser.add_mutually_exclusive_group() - target_type.add_argument('-d', - '--directory', action='store_const', - const='directory', dest='target_type', - default='directory', help=( -"Choose this option is the target is a directory.")) - target_type.add_argument('-a', - '--archive', action='store_const', - const='archive', dest='target_type', - help=( -"Choose this option if the target is an archive.")) + help="Name of user these media entries belong to") subparser.add_argument( 'target_path', - help=( -"Path to a local archive or directory containing a location.csv and metadata.csv file")) + help=("""\ +Path to a local archive or directory containing a "location.csv" and a +"metadata.csv" file. These are csv (comma seperated value) files with the +locations and metadata of the files to be uploaded. The location must be listed +with either the URL of the remote media file or the filesystem path of a local +file. The metadata should be provided with one column for each of the 15 Dublin +Core properties (http://dublincore.org/documents/dces/). Both "location.csv" and +"metadata.csv" must begin with a row demonstrating the order of the columns. We +have provided an example of these files at +""")) subparser.add_argument( "-l", "--license", help=( - "License these media entry will be released under, if all the same" - "Should be a URL.")) + "License these media entry will be released under, if all the same. " + "Should be a URL.")) subparser.add_argument( '--celery', action='store_true', @@ -67,26 +66,38 @@ def batchaddmedia(args): # get the user user = app.db.User.query.filter_by(username=args.username.lower()).first() if user is None: - print "Sorry, no user by username '%s'" % args.username + print "Sorry, no user by username '%s' exists" % args.username return upload_limit, max_file_size = get_upload_file_limits(user) temp_files = [] - if args.target_type == 'archive': + if tarfile.is_tarfile(args.target_path): dir_path = tempfile.mkdtemp() temp_files.append(dir_path) tar = tarfile.open(args.target_path) tar.extractall(path=dir_path) - elif args.target_type == 'directory': + elif zipfile.is_zipfile(args.target_path): + dir_path = tempfile.mkdtemp() + temp_files.append(dir_path) + zipped_file = zipfile.ZipFile(args.target_path) + zipped_file.extractall(path=dir_path) + + elif os.path.isdir(args.target_path): dir_path = args.target_path + else: + print "Couldn't recognize the file. This script only accepts tar files,\ +zip files and directories" + if dir_path.endswith('/'): + dir_path = dir_path[:-1] + location_file_path = "{dir_path}/location.csv".format( dir_path=dir_path) metadata_file_path = "{dir_path}/metadata.csv".format( dir_path=dir_path) - + # check for the location file, if it exists... location_filename = os.path.split(location_file_path)[-1] abs_location_filename = os.path.abspath(location_file_path) @@ -178,7 +189,7 @@ def batchaddmedia(args): print "This user is already past their upload limits." teardown(temp_files) - + def parse_csv_file(file_contents): list_of_contents = file_contents.split('\n') From 5c14f62d19abefb99f392cf1a92b07ec0de26bc4 Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Fri, 21 Feb 2014 12:38:02 -0500 Subject: [PATCH 25/78] Changed some of the print messages as well as tweaked the order of the commands attempts to figure out what type of file the target file is. --- mediagoblin/gmg_commands/batchaddmedia.py | 30 +++++++++++++---------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/mediagoblin/gmg_commands/batchaddmedia.py b/mediagoblin/gmg_commands/batchaddmedia.py index d3ab7733..678c8ab4 100644 --- a/mediagoblin/gmg_commands/batchaddmedia.py +++ b/mediagoblin/gmg_commands/batchaddmedia.py @@ -63,6 +63,8 @@ def batchaddmedia(args): app = commands_util.setup_app(args) + files_uploaded, files_attempted = 0, 0 + # get the user user = app.db.User.query.filter_by(username=args.username.lower()).first() if user is None: @@ -72,7 +74,10 @@ def batchaddmedia(args): upload_limit, max_file_size = get_upload_file_limits(user) temp_files = [] - if tarfile.is_tarfile(args.target_path): + if os.path.isdir(args.target_path): + dir_path = args.target_path + + elif tarfile.is_tarfile(args.target_path): dir_path = tempfile.mkdtemp() temp_files.append(dir_path) tar = tarfile.open(args.target_path) @@ -84,9 +89,6 @@ def batchaddmedia(args): zipped_file = zipfile.ZipFile(args.target_path) zipped_file.extractall(path=dir_path) - elif os.path.isdir(args.target_path): - dir_path = args.target_path - else: print "Couldn't recognize the file. This script only accepts tar files,\ zip files and directories" @@ -141,11 +143,9 @@ zip files and directories" description = file_metadata.get('dcterms:description') license = file_metadata.get('dcterms:license') filename = url.path.split()[-1] - print "Working with {filename}".format(filename=filename) + files_attempted += 1 if url.scheme == 'http': - print "Downloading {filename}...".format( - filename=filename) media_file = tempfile.TemporaryFile() res = urllib.urlopen(url.geturl()) media_file.write(res.read()) @@ -163,11 +163,10 @@ zip files and directories" try: media_file = file(file_abs_path, 'r') except IOError: - print "Local file {filename} could not be accessed.".format( - filename=filename) + print "\ +FAIL: Local file {filename} could not be accessed.".format(filename=filename) print "Skipping it." continue - print "Submitting {filename}...".format(filename=filename) try: submit_media( mg_app=app, @@ -181,12 +180,17 @@ zip files and directories" upload_limit=upload_limit, max_file_size=max_file_size) print "Successfully uploading {filename}!".format(filename=filename) print "" + files_uploaded += 1 except FileUploadLimit: - print "This file is larger than the upload limits for this site." + print "FAIL: This file is larger than the upload limits for this site." except UserUploadLimit: - print "This file will put this user past their upload limits." + print "FAIL: This file will put this user past their upload limits." except UserPastUploadLimit: - print "This user is already past their upload limits." + print "FAIL: This user is already past their upload limits." + print "\ +{files_uploaded} out of {files_attempted} files successfully uploaded".format( + files_uploaded=files_uploaded, + files_attempted=files_attempted) teardown(temp_files) From 3e76b2bc7712e012bdb2f971961adf96741e0df3 Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Thu, 27 Mar 2014 13:31:04 -0400 Subject: [PATCH 26/78] Began work on metadata validation --- mediagoblin/gmg_commands/batchaddmedia.py | 33 +++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/mediagoblin/gmg_commands/batchaddmedia.py b/mediagoblin/gmg_commands/batchaddmedia.py index 678c8ab4..83aea7b7 100644 --- a/mediagoblin/gmg_commands/batchaddmedia.py +++ b/mediagoblin/gmg_commands/batchaddmedia.py @@ -26,6 +26,7 @@ from mediagoblin.submit.lib import ( FileUploadLimit, UserUploadLimit, UserPastUploadLimit) from mediagoblin import mg_globals +from jsonschema import validate def parser_setup(subparser): subparser.description = """\ @@ -215,3 +216,35 @@ def parse_csv_file(file_contents): def teardown(temp_files): for temp_file in temp_files: subprocess.call(['rm','-r',temp_file]) + +def check_metadata_format(metadata_dict): + schema = json.loads(""" +{ + "$schema":"http://json-schema.org/schema#", + "properties":{ + "@context":{}, + "contributor":{}, + "coverage":{}, + "created":{}, + "creator":{}, + "date":{}, + "description":{}, + "format":{}, + "identifier":{}, + "language":{}, + "publisher":{}, + "relation":{}, + "rights" : { + "format":"uri", + "type":"string" + }, + "source":{}, + "subject":{}, + "title":{}, + "type":{} + }, + "additionalProperties": false, + "required":["title","@context"] +}""") + try: + validate(metadata_dict, schema) From 26b3d6cf27d8653bfcbd8caf9bec4abfb5c16c5a Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Thu, 27 Mar 2014 13:55:15 -0400 Subject: [PATCH 27/78] Added exception handling into the metadata format checking function. --- mediagoblin/gmg_commands/batchaddmedia.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/mediagoblin/gmg_commands/batchaddmedia.py b/mediagoblin/gmg_commands/batchaddmedia.py index 83aea7b7..f06bc2e8 100644 --- a/mediagoblin/gmg_commands/batchaddmedia.py +++ b/mediagoblin/gmg_commands/batchaddmedia.py @@ -24,9 +24,11 @@ from mediagoblin.gmg_commands import util as commands_util from mediagoblin.submit.lib import ( submit_media, get_upload_file_limits, FileUploadLimit, UserUploadLimit, UserPastUploadLimit) +from mediagoblin.tools.translate import lazy_pass_to_ugettext as _ from mediagoblin import mg_globals -from jsonschema import validate +from jsonschema import validate +from jsonschema.exceptions import ValidationError def parser_setup(subparser): subparser.description = """\ @@ -135,7 +137,10 @@ zip files and directories" dcterms_context = { 'dcterms':'http://purl.org/dc/terms/' } for media_id in media_locations.keys(): - file_metadata = media_metadata[media_id] + file_metadata = media_metadata[media_id] + santized_metadata = check_metadata_format(file_metadata) + if sanitized_metadata == {}: continue + json_ld_metadata = jsonld.compact(file_metadata, dcterms_context) original_location = media_locations[media_id]['media:original'] url = urlparse(original_location) @@ -248,3 +253,14 @@ def check_metadata_format(metadata_dict): }""") try: validate(metadata_dict, schema) + output_dict = metadata_dict + except ValidationError, exc: + title = metadata_dict.get('title') or metadata_dict.get('media:id') or \ + _(u'UNKNOWN FILE') + print _( +u"""WARN: Could not find appropriate metadata for file {title}. File will be +skipped""".format(title=title)) + output_dict = {} + except: + raise + return output_dict From 8c7cccf6cc6d2a4f58d10a116a535854d6ae9e63 Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Thu, 27 Mar 2014 14:11:12 -0400 Subject: [PATCH 28/78] Fixed up some fatal errors. Is still not ready. --- mediagoblin/gmg_commands/batchaddmedia.py | 46 ++++++++++++----------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/mediagoblin/gmg_commands/batchaddmedia.py b/mediagoblin/gmg_commands/batchaddmedia.py index f06bc2e8..414e969c 100644 --- a/mediagoblin/gmg_commands/batchaddmedia.py +++ b/mediagoblin/gmg_commands/batchaddmedia.py @@ -138,7 +138,7 @@ zip files and directories" for media_id in media_locations.keys(): file_metadata = media_metadata[media_id] - santized_metadata = check_metadata_format(file_metadata) + sanitized_metadata = check_metadata_format(file_metadata) if sanitized_metadata == {}: continue json_ld_metadata = jsonld.compact(file_metadata, dcterms_context) @@ -207,7 +207,7 @@ def parse_csv_file(file_contents): list_of_contents[1:]) objects_dict = {} - # Build a dictionary + # Build a dictionaryfrom mediagoblin.tools.translate import lazy_pass_to_ugettext as _ for line in lines: if line.isspace() or line == '': continue values = csv_reader([line]).next() @@ -228,38 +228,40 @@ def check_metadata_format(metadata_dict): "$schema":"http://json-schema.org/schema#", "properties":{ "@context":{}, - "contributor":{}, - "coverage":{}, - "created":{}, - "creator":{}, - "date":{}, - "description":{}, - "format":{}, - "identifier":{}, - "language":{}, - "publisher":{}, - "relation":{}, - "rights" : { + "dcterms:contributor":{}, + "dcterms:coverage":{}, + "dcterms:created":{}, + "dcterms:creator":{}, + "dcterms:date":{}, + "dcterms:description":{}, + "dcterms:format":{}, + "dcterms:identifier":{}, + "dcterms:language":{}, + "dcterms:publisher":{}, + "dcterms:relation":{}, + "dcterms:rights" : { "format":"uri", "type":"string" }, - "source":{}, - "subject":{}, - "title":{}, - "type":{} + "dcterms:source":{}, + "dcterms:subject":{}, + "dcterms:title":{}, + "dcterms:type":{}, + "media:id":{} }, "additionalProperties": false, - "required":["title","@context"] + "required":["dcterms:title","@context","media:id"] }""") + metadata_dict["@context"] = u"http://127.0.0.1:6543/metadata_context/v1/" try: validate(metadata_dict, schema) output_dict = metadata_dict except ValidationError, exc: - title = metadata_dict.get('title') or metadata_dict.get('media:id') or \ + title = metadata_dict.get('dcterms:title') or metadata_dict.get('media:id') or \ _(u'UNKNOWN FILE') print _( -u"""WARN: Could not find appropriate metadata for file {title}. File will be -skipped""".format(title=title)) +u"""WARN: Could not find appropriate metadata for file {title}. +File will be skipped""".format(title=title)) output_dict = {} except: raise From e46760d3155873803fe3ee0e1d10cd0142eacae1 Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Thu, 27 Mar 2014 17:10:31 -0400 Subject: [PATCH 29/78] Fixed a minor error in the batch upload script and modified the json-ld context. --- mediagoblin/gmg_commands/batchaddmedia.py | 9 ++--- .../mediagoblin/metadata_contexts/v1 | 34 +++++++++---------- 2 files changed, 22 insertions(+), 21 deletions(-) diff --git a/mediagoblin/gmg_commands/batchaddmedia.py b/mediagoblin/gmg_commands/batchaddmedia.py index 414e969c..012a5ee4 100644 --- a/mediagoblin/gmg_commands/batchaddmedia.py +++ b/mediagoblin/gmg_commands/batchaddmedia.py @@ -137,6 +137,8 @@ zip files and directories" dcterms_context = { 'dcterms':'http://purl.org/dc/terms/' } for media_id in media_locations.keys(): + files_attempted += 1 + file_metadata = media_metadata[media_id] sanitized_metadata = check_metadata_format(file_metadata) if sanitized_metadata == {}: continue @@ -149,7 +151,6 @@ zip files and directories" description = file_metadata.get('dcterms:description') license = file_metadata.get('dcterms:license') filename = url.path.split()[-1] - files_attempted += 1 if url.scheme == 'http': media_file = tempfile.TemporaryFile() @@ -228,6 +229,7 @@ def check_metadata_format(metadata_dict): "$schema":"http://json-schema.org/schema#", "properties":{ "@context":{}, + "dcterms:contributor":{}, "dcterms:coverage":{}, "dcterms:created":{}, @@ -246,8 +248,7 @@ def check_metadata_format(metadata_dict): "dcterms:source":{}, "dcterms:subject":{}, "dcterms:title":{}, - "dcterms:type":{}, - "media:id":{} + "dcterms:type":{} }, "additionalProperties": false, "required":["dcterms:title","@context","media:id"] @@ -260,7 +261,7 @@ def check_metadata_format(metadata_dict): title = metadata_dict.get('dcterms:title') or metadata_dict.get('media:id') or \ _(u'UNKNOWN FILE') print _( -u"""WARN: Could not find appropriate metadata for file {title}. +u"""WARN: Could not find appropriate metadata for file "{title}". File will be skipped""".format(title=title)) output_dict = {} except: diff --git a/mediagoblin/templates/mediagoblin/metadata_contexts/v1 b/mediagoblin/templates/mediagoblin/metadata_contexts/v1 index 1325d920..99882de2 100644 --- a/mediagoblin/templates/mediagoblin/metadata_contexts/v1 +++ b/mediagoblin/templates/mediagoblin/metadata_contexts/v1 @@ -1,69 +1,69 @@ { "@context": { - "dc": "http://purl.org/dc/elements/1.1/", + "dcterms": "http://purl.org/dc/elements/1.1/", "xsd": "http://www.w3.org/2001/XMLSchema#", "contributor":{ - "@id":"dc:title", + "@id":"dcterms:title", "@type":"xsd:string" }, "coverage":{ - "@id":"dc:coverage", + "@id":"dcterms:coverage", "@type":"xsd:string" }, "created":{ - "@id":"dc:created", + "@id":"dcterms:created", "@type":"xsd:date" }, "creator":{ - "@id":"dc:created", + "@id":"dcterms:created", "@type":"xsd:date" }, "date":{ - "@id":"dc:date", + "@id":"dcterms:date", "@type":"xsd:date" }, "description":{ - "@id":"dc:description", + "@id":"dcterms:description", "@type":"xsd:string" }, "format":{ - "@id":"dc:format", + "@id":"dcterms:format", "@type":"xsd:string" }, "identifier":{ - "@id":"dc:identifier", + "@id":"dcterms:identifier", "@type":"xsd:string" }, "language":{ - "@id":"dc:language", + "@id":"dcterms:language", "@type":"xsd:string" }, "publisher":{ - "@id":"dc:publisher", + "@id":"dcterms:publisher", "@type":"xsd:string" }, "relation":{ - "@id":"dc:relation", + "@id":"dcterms:relation", "@type":"xsd:string" }, "rights":{ - "@id":"dc:rights", + "@id":"dcterms:rights", "@type":"xsd:anyURI" }, "source":{ - "@id":"dc:source", + "@id":"dcterms:source", "@type":"xsd:string" }, "subject":{ - "@id":"dc:subject", + "@id":"dcterms:subject", "@type":"xsd:string" }, "title": { - "@id":"dc:title", + "@id":"dcterms:title", "@type":"xsd:string" }, "type":{ - "@id":"dc:type", + "@id":"dcterms:type", "@type":"xsd:string" } } From 907d9626e356cfcbc6b3e47a92771650a8eee4e1 Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Thu, 27 Mar 2014 17:29:34 -0400 Subject: [PATCH 30/78] Wrote more comprehensive error messages. --- mediagoblin/gmg_commands/batchaddmedia.py | 28 +++++++++++++++++------ 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/mediagoblin/gmg_commands/batchaddmedia.py b/mediagoblin/gmg_commands/batchaddmedia.py index 012a5ee4..fe345d5f 100644 --- a/mediagoblin/gmg_commands/batchaddmedia.py +++ b/mediagoblin/gmg_commands/batchaddmedia.py @@ -229,7 +229,7 @@ def check_metadata_format(metadata_dict): "$schema":"http://json-schema.org/schema#", "properties":{ "@context":{}, - + "media:id":{}, "dcterms:contributor":{}, "dcterms:coverage":{}, "dcterms:created":{}, @@ -251,18 +251,32 @@ def check_metadata_format(metadata_dict): "dcterms:type":{} }, "additionalProperties": false, - "required":["dcterms:title","@context","media:id"] + "required":["dcterms:title","@context","media:id","bell"] }""") metadata_dict["@context"] = u"http://127.0.0.1:6543/metadata_context/v1/" try: validate(metadata_dict, schema) output_dict = metadata_dict except ValidationError, exc: - title = metadata_dict.get('dcterms:title') or metadata_dict.get('media:id') or \ - _(u'UNKNOWN FILE') - print _( -u"""WARN: Could not find appropriate metadata for file "{title}". -File will be skipped""".format(title=title)) + title = (metadata_dict.get('dcterms:title') or + metadata_dict.get('media:id') or _(u'UNKNOWN FILE')) + + if exc.validator == "additionalProperties": + message = _(u'Invalid metadata provided for file "{title}". This \ +script only accepts the Dublin Core metadata terms.'.format(title=title)) + + elif exc.validator == "required": + message = _( +u'All necessary metadata was not provided for file "{title}", you must include \ +a "dcterms:title" column for each media file'.format(title=title)) + + else: + message = _(u'Could not find appropriate metadata for file \ +"{title}".'.format(title=title)) + + print _(u"""WARN: {message} \nSkipping File...\n""".format( + message=message)) + output_dict = {} except: raise From 77d51d4f3378de8f3bbf54dd4c99e62399688800 Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Thu, 3 Apr 2014 12:18:17 -0400 Subject: [PATCH 31/78] Fixed a bad get of 'dcterms:rights' and am throwing away the idea of an external context file for the json-ld because it feels unnecessary seeing as we are just using the dc core terms --- mediagoblin/gmg_commands/batchaddmedia.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/mediagoblin/gmg_commands/batchaddmedia.py b/mediagoblin/gmg_commands/batchaddmedia.py index fe345d5f..68993aa2 100644 --- a/mediagoblin/gmg_commands/batchaddmedia.py +++ b/mediagoblin/gmg_commands/batchaddmedia.py @@ -15,7 +15,7 @@ # along with this program. If not, see . import os -import json, tempfile, urllib, tarfile, zipfile, subprocess +import tempfile, urllib, tarfile, zipfile, subprocess from csv import reader as csv_reader from urlparse import urlparse from pyld import jsonld @@ -149,7 +149,7 @@ zip files and directories" title = file_metadata.get('dcterms:title') description = file_metadata.get('dcterms:description') - license = file_metadata.get('dcterms:license') + license = file_metadata.get('dcterms:rights') filename = url.path.split()[-1] if url.scheme == 'http': @@ -201,7 +201,6 @@ FAIL: Local file {filename} could not be accessed.".format(filename=filename) teardown(temp_files) - def parse_csv_file(file_contents): list_of_contents = file_contents.split('\n') key, lines = (list_of_contents[0].split(','), @@ -219,16 +218,16 @@ def parse_csv_file(file_contents): return objects_dict + def teardown(temp_files): for temp_file in temp_files: subprocess.call(['rm','-r',temp_file]) + def check_metadata_format(metadata_dict): - schema = json.loads(""" -{ + schema = { "$schema":"http://json-schema.org/schema#", "properties":{ - "@context":{}, "media:id":{}, "dcterms:contributor":{}, "dcterms:coverage":{}, @@ -250,13 +249,14 @@ def check_metadata_format(metadata_dict): "dcterms:title":{}, "dcterms:type":{} }, - "additionalProperties": false, - "required":["dcterms:title","@context","media:id","bell"] -}""") - metadata_dict["@context"] = u"http://127.0.0.1:6543/metadata_context/v1/" + "additionalProperties": False, + "required":["dcterms:title","media:id"] +} try: validate(metadata_dict, schema) output_dict = metadata_dict + del output_dict['media:id'] + except ValidationError, exc: title = (metadata_dict.get('dcterms:title') or metadata_dict.get('media:id') or _(u'UNKNOWN FILE')) @@ -280,4 +280,5 @@ a "dcterms:title" column for each media file'.format(title=title)) output_dict = {} except: raise + return output_dict From 7b1ee4711c7ade0d1b4dba90b7199f969b9912b1 Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Thu, 3 Apr 2014 12:20:30 -0400 Subject: [PATCH 32/78] Took out all of the references to the temporary url I was using /metadata_context/v1 --- mediagoblin/routing.py | 2 - .../mediagoblin/metadata_contexts/v1 | 70 ------------------- mediagoblin/views.py | 6 -- 3 files changed, 78 deletions(-) delete mode 100644 mediagoblin/templates/mediagoblin/metadata_contexts/v1 diff --git a/mediagoblin/routing.py b/mediagoblin/routing.py index a6b2a543..9f2584d3 100644 --- a/mediagoblin/routing.py +++ b/mediagoblin/routing.py @@ -29,8 +29,6 @@ def get_url_map(): add_route('index', '/', 'mediagoblin.views:root_view') add_route('terms_of_service','/terms_of_service', 'mediagoblin.views:terms_of_service'), - add_route('metadata_context','/metadata_context/v/', - 'mediagoblin.views:metadata_context_view'), mount('/auth', auth_routes) mount('/mod', moderation_routes) diff --git a/mediagoblin/templates/mediagoblin/metadata_contexts/v1 b/mediagoblin/templates/mediagoblin/metadata_contexts/v1 deleted file mode 100644 index 99882de2..00000000 --- a/mediagoblin/templates/mediagoblin/metadata_contexts/v1 +++ /dev/null @@ -1,70 +0,0 @@ -{ - "@context": { - "dcterms": "http://purl.org/dc/elements/1.1/", - "xsd": "http://www.w3.org/2001/XMLSchema#", - "contributor":{ - "@id":"dcterms:title", - "@type":"xsd:string" - }, - "coverage":{ - "@id":"dcterms:coverage", - "@type":"xsd:string" - }, - "created":{ - "@id":"dcterms:created", - "@type":"xsd:date" - }, - "creator":{ - "@id":"dcterms:created", - "@type":"xsd:date" - }, - "date":{ - "@id":"dcterms:date", - "@type":"xsd:date" - }, - "description":{ - "@id":"dcterms:description", - "@type":"xsd:string" - }, - "format":{ - "@id":"dcterms:format", - "@type":"xsd:string" - }, - "identifier":{ - "@id":"dcterms:identifier", - "@type":"xsd:string" - }, - "language":{ - "@id":"dcterms:language", - "@type":"xsd:string" - }, - "publisher":{ - "@id":"dcterms:publisher", - "@type":"xsd:string" - }, - "relation":{ - "@id":"dcterms:relation", - "@type":"xsd:string" - }, - "rights":{ - "@id":"dcterms:rights", - "@type":"xsd:anyURI" - }, - "source":{ - "@id":"dcterms:source", - "@type":"xsd:string" - }, - "subject":{ - "@id":"dcterms:subject", - "@type":"xsd:string" - }, - "title": { - "@id":"dcterms:title", - "@type":"xsd:string" - }, - "type":{ - "@id":"dcterms:type", - "@type":"xsd:string" - } - } -} diff --git a/mediagoblin/views.py b/mediagoblin/views.py index 1ed71473..009e48e4 100644 --- a/mediagoblin/views.py +++ b/mediagoblin/views.py @@ -62,9 +62,3 @@ def terms_of_service(request): return render_to_response(request, 'mediagoblin/terms_of_service.html', {}) - -def metadata_context_view(request): - version = request.matchdict['version_number'] - return render_to_response(request, - 'mediagoblin/metadata_contexts/v{version}'.format( - version=version), {}) From 18a9c50db6db680070948fd42043ea0a89deaa6f Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Fri, 11 Apr 2014 13:06:09 -0400 Subject: [PATCH 33/78] Fixed incorrectly coded references to filesystem paths --- mediagoblin/gmg_commands/batchaddmedia.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/mediagoblin/gmg_commands/batchaddmedia.py b/mediagoblin/gmg_commands/batchaddmedia.py index 68993aa2..b058a47e 100644 --- a/mediagoblin/gmg_commands/batchaddmedia.py +++ b/mediagoblin/gmg_commands/batchaddmedia.py @@ -98,10 +98,8 @@ zip files and directories" if dir_path.endswith('/'): dir_path = dir_path[:-1] - location_file_path = "{dir_path}/location.csv".format( - dir_path=dir_path) - metadata_file_path = "{dir_path}/metadata.csv".format( - dir_path=dir_path) + location_file_path = os.path.join(dir_path,"location.csv") + metadata_file_path = os.path.join(dir_path, "metadata.csv") # check for the location file, if it exists... location_filename = os.path.split(location_file_path)[-1] @@ -163,9 +161,7 @@ zip files and directories" if os.path.isabs(path): file_abs_path = os.path.abspath(path) else: - file_path = "{dir_path}/{local_path}".format( - dir_path=dir_path, - local_path=path) + file_path = os.path.join(dir_path, path) file_abs_path = os.path.abspath(file_path) try: media_file = file(file_abs_path, 'r') From ecea4847e8259125dec4617c6f11c7d7e3962925 Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Tue, 15 Apr 2014 13:35:22 -0400 Subject: [PATCH 34/78] Added the 'requests' library as a dependency and switched over to using it to fetch remote pieces of media in the batchupload script --- mediagoblin/gmg_commands/batchaddmedia.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/mediagoblin/gmg_commands/batchaddmedia.py b/mediagoblin/gmg_commands/batchaddmedia.py index b058a47e..deb6c5bd 100644 --- a/mediagoblin/gmg_commands/batchaddmedia.py +++ b/mediagoblin/gmg_commands/batchaddmedia.py @@ -15,9 +15,10 @@ # along with this program. If not, see . import os -import tempfile, urllib, tarfile, zipfile, subprocess +import tempfile, tarfile, zipfile, subprocess, requests from csv import reader as csv_reader from urlparse import urlparse +import requests from pyld import jsonld from mediagoblin.gmg_commands import util as commands_util @@ -151,10 +152,8 @@ zip files and directories" filename = url.path.split()[-1] if url.scheme == 'http': - media_file = tempfile.TemporaryFile() - res = urllib.urlopen(url.geturl()) - media_file.write(res.read()) - media_file.seek(0) + res = requests.get(url.geturl()) + media_file = res.raw elif url.scheme == '': path = url.path From 9f3dc83a6cec48379972b1f99f955b9525e77c32 Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Tue, 15 Apr 2014 13:51:27 -0400 Subject: [PATCH 35/78] Moved the metadata column to MediaEntry rather than MediaFile --- mediagoblin/db/migrations.py | 8 ++++---- mediagoblin/db/models.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/mediagoblin/db/migrations.py b/mediagoblin/db/migrations.py index a7400bf0..294ab43b 100644 --- a/mediagoblin/db/migrations.py +++ b/mediagoblin/db/migrations.py @@ -723,12 +723,12 @@ def drop_MediaEntry_collected(db): db.commit() @RegisterMigration(20, MIGRATIONS) -def add_work_metadata_column(db): +def add_metadata_column(db): metadata = MetaData(bind=db.bind) - media_file = inspect_table(metadata, 'core__mediafiles') + media_entry = inspect_table(metadata, 'core__media_entries') - col = Column('work_metadata', MutationDict.as_mutable(JSONEncoded)) - col.create(media_file) + col = Column('metadata', MutationDict.as_mutable(JSONEncoded)) + col.create(media_entry) db.commit() diff --git a/mediagoblin/db/models.py b/mediagoblin/db/models.py index ac69d040..7c0f0bf3 100644 --- a/mediagoblin/db/models.py +++ b/mediagoblin/db/models.py @@ -264,6 +264,7 @@ class MediaEntry(Base, MediaEntryMixin): cascade="all, delete-orphan" ) collections = association_proxy("collections_helper", "in_collection") + metadata = Column(MutationDict.as_mutable(JSONEncoded)) ## TODO # fail_error @@ -420,7 +421,6 @@ class MediaFile(Base): name_id = Column(SmallInteger, ForeignKey(FileKeynames.id), nullable=False) file_path = Column(PathTupleWithSlashes) file_metadata = Column(MutationDict.as_mutable(JSONEncoded)) - work_metadata = Column(MutationDict.as_mutable(JSONEncoded)) __table_args__ = ( PrimaryKeyConstraint('media_entry', 'name_id'), From 0bfb4089cc187113a7808eed6643a73c86bb01cb Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Tue, 15 Apr 2014 14:17:43 -0400 Subject: [PATCH 36/78] Changed the name of the metadata column --- mediagoblin/db/migrations.py | 2 +- mediagoblin/db/models.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mediagoblin/db/migrations.py b/mediagoblin/db/migrations.py index 294ab43b..8dac3214 100644 --- a/mediagoblin/db/migrations.py +++ b/mediagoblin/db/migrations.py @@ -728,7 +728,7 @@ def add_metadata_column(db): media_entry = inspect_table(metadata, 'core__media_entries') - col = Column('metadata', MutationDict.as_mutable(JSONEncoded)) + col = Column('media_metadata', MutationDict.as_mutable(JSONEncoded)) col.create(media_entry) db.commit() diff --git a/mediagoblin/db/models.py b/mediagoblin/db/models.py index 7c0f0bf3..defa0849 100644 --- a/mediagoblin/db/models.py +++ b/mediagoblin/db/models.py @@ -264,7 +264,7 @@ class MediaEntry(Base, MediaEntryMixin): cascade="all, delete-orphan" ) collections = association_proxy("collections_helper", "in_collection") - metadata = Column(MutationDict.as_mutable(JSONEncoded)) + media_metadata = Column(MutationDict.as_mutable(JSONEncoded)) ## TODO # fail_error From 45f426ddee9900439c086a8bb3d1cfaedf3eca6f Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Mon, 21 Apr 2014 12:07:33 -0400 Subject: [PATCH 37/78] Made it possible to submit media with the metadata provided --- mediagoblin/gmg_commands/batchaddmedia.py | 1 + mediagoblin/submit/lib.py | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/mediagoblin/gmg_commands/batchaddmedia.py b/mediagoblin/gmg_commands/batchaddmedia.py index deb6c5bd..b6fd2763 100644 --- a/mediagoblin/gmg_commands/batchaddmedia.py +++ b/mediagoblin/gmg_commands/batchaddmedia.py @@ -178,6 +178,7 @@ FAIL: Local file {filename} could not be accessed.".format(filename=filename) title=maybe_unicodeify(title), description=maybe_unicodeify(description), license=maybe_unicodeify(license), + metadata=json_ld_metadata, tags_string=u"", upload_limit=upload_limit, max_file_size=max_file_size) print "Successfully uploading {filename}!".format(filename=filename) diff --git a/mediagoblin/submit/lib.py b/mediagoblin/submit/lib.py index c70e2731..df3f7b62 100644 --- a/mediagoblin/submit/lib.py +++ b/mediagoblin/submit/lib.py @@ -98,7 +98,7 @@ class UserPastUploadLimit(UploadLimitError): def submit_media(mg_app, user, submitted_file, filename, title=None, description=None, - license=None, tags_string=u"", + license=None, metadata=None, tags_string=u"", upload_limit=None, max_file_size=None, callback_url=None, # If provided we'll do the feed_url update, otherwise ignore @@ -142,6 +142,8 @@ def submit_media(mg_app, user, submitted_file, filename, entry.license = license or None + entry.media_metadata = metadata or u"" + # Process the user's folksonomy "tags" entry.tags = convert_to_tag_list_of_dicts(tags_string) From e00ce53ef63abb20524399518e260c9262cc041b Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Mon, 21 Apr 2014 12:18:29 -0400 Subject: [PATCH 38/78] I had imported requests twice --- mediagoblin/gmg_commands/batchaddmedia.py | 1 - 1 file changed, 1 deletion(-) diff --git a/mediagoblin/gmg_commands/batchaddmedia.py b/mediagoblin/gmg_commands/batchaddmedia.py index b6fd2763..f50425f3 100644 --- a/mediagoblin/gmg_commands/batchaddmedia.py +++ b/mediagoblin/gmg_commands/batchaddmedia.py @@ -18,7 +18,6 @@ import os import tempfile, tarfile, zipfile, subprocess, requests from csv import reader as csv_reader from urlparse import urlparse -import requests from pyld import jsonld from mediagoblin.gmg_commands import util as commands_util From 7ff99dabfbb3e854afe2aba17a79a0aee9062e44 Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Mon, 21 Apr 2014 12:29:00 -0400 Subject: [PATCH 39/78] Fixed a problem that was causing errors in batch uploading remote files. --- mediagoblin/gmg_commands/batchaddmedia.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediagoblin/gmg_commands/batchaddmedia.py b/mediagoblin/gmg_commands/batchaddmedia.py index f50425f3..61a068d2 100644 --- a/mediagoblin/gmg_commands/batchaddmedia.py +++ b/mediagoblin/gmg_commands/batchaddmedia.py @@ -151,7 +151,7 @@ zip files and directories" filename = url.path.split()[-1] if url.scheme == 'http': - res = requests.get(url.geturl()) + res = requests.get(url.geturl(), stream=True) media_file = res.raw elif url.scheme == '': From 8ccd560ca2c480bb55f8caf1ddc035731eee11ca Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Mon, 21 Apr 2014 15:42:57 -0400 Subject: [PATCH 40/78] Created the basic structure of the metadata display table in rdfa lite --- .../mediagoblin/user_pages/media.html | 2 + .../mediagoblin/utils/metadata_table.html | 37 +++++++++++++++++++ mediagoblin/user_pages/lib.py | 3 ++ mediagoblin/user_pages/views.py | 5 ++- 4 files changed, 45 insertions(+), 2 deletions(-) create mode 100644 mediagoblin/templates/mediagoblin/utils/metadata_table.html diff --git a/mediagoblin/templates/mediagoblin/user_pages/media.html b/mediagoblin/templates/mediagoblin/user_pages/media.html index e01cce5c..22971fec 100644 --- a/mediagoblin/templates/mediagoblin/user_pages/media.html +++ b/mediagoblin/templates/mediagoblin/user_pages/media.html @@ -18,6 +18,7 @@ {%- extends "mediagoblin/base.html" %} {% import "/mediagoblin/utils/wtforms.html" as wtforms_util %} +{% import "/mediagoblin/utils/metadata_table.html" as metadata_util %} {% from "mediagoblin/utils/pagination.html" import render_pagination %} {% block title %}{{ media.title }} — {{ super() }}{% endblock %} @@ -231,6 +232,7 @@ {% template_hook("media_sideinfo") %} {% block mediagoblin_sidebar %} + {{ metadata_util.render_table(request, media, rdfa_to_readable) }} {% endblock %} diff --git a/mediagoblin/templates/mediagoblin/utils/metadata_table.html b/mediagoblin/templates/mediagoblin/utils/metadata_table.html new file mode 100644 index 00000000..38b580d5 --- /dev/null +++ b/mediagoblin/templates/mediagoblin/utils/metadata_table.html @@ -0,0 +1,37 @@ +{# +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +#} + +{%- macro render_table(request, media_entry, format_predicate) %} + {%- set metadata=media_entry.media_metadata %} + {%- set metadata_context=metadata['@context'] %} + {%- if metadata %} + + {%- for key, value in metadata.iteritems() if not key=='@context' %} + + + {% if value -%} + + {%- else -%} + + {%- endif -%} + {%- endfor %} +
{{ format_predicate(key) }}{{ value }}
+ {% endif %} +{%- endmacro %} diff --git a/mediagoblin/user_pages/lib.py b/mediagoblin/user_pages/lib.py index e5c8defc..83a99cee 100644 --- a/mediagoblin/user_pages/lib.py +++ b/mediagoblin/user_pages/lib.py @@ -116,3 +116,6 @@ def build_report_object(report_form, media_entry=None, comment=None): report_object.reporter_id = report_form.reporter_id.data return report_object +def rdfa_to_readable(rdfa_predicate): + readable = rdfa_predicate.split(u":")[1].capitalize() + return readable diff --git a/mediagoblin/user_pages/views.py b/mediagoblin/user_pages/views.py index 78751a28..f42eae1f 100644 --- a/mediagoblin/user_pages/views.py +++ b/mediagoblin/user_pages/views.py @@ -28,7 +28,7 @@ from mediagoblin.tools.translate import pass_to_ugettext as _ from mediagoblin.tools.pagination import Pagination from mediagoblin.user_pages import forms as user_forms from mediagoblin.user_pages.lib import (send_comment_email, - add_media_to_collection, build_report_object) + add_media_to_collection, build_report_object, rdfa_to_readable) from mediagoblin.notifications import trigger_notification, \ add_comment_subscription, mark_comment_notification_seen from mediagoblin.tools.pluginapi import hook_transform @@ -152,7 +152,8 @@ def media_home(request, media, page, **kwargs): 'comments': comments, 'pagination': pagination, 'comment_form': comment_form, - 'app_config': mg_globals.app_config} + 'app_config': mg_globals.app_config, + 'rdfa_to_readable':rdfa_to_readable} # Since the media template name gets swapped out for each media # type, normal context hooks don't work if you want to affect all From 0656de67954bc33c92555cbd48d0c51ca20a0e50 Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Mon, 21 Apr 2014 15:46:02 -0400 Subject: [PATCH 41/78] Made it so that the metadata table only shows terms with filled values --- .../templates/mediagoblin/utils/metadata_table.html | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/mediagoblin/templates/mediagoblin/utils/metadata_table.html b/mediagoblin/templates/mediagoblin/utils/metadata_table.html index 38b580d5..166d5716 100644 --- a/mediagoblin/templates/mediagoblin/utils/metadata_table.html +++ b/mediagoblin/templates/mediagoblin/utils/metadata_table.html @@ -24,13 +24,12 @@ {{ prefix }} {{ metadata_context[prefix] }} {%- endfor %}"> {%- for key, value in metadata.iteritems() if not key=='@context' %} - - {{ format_predicate(key) }} - {% if value -%} - {{ value }} - {%- else -%} - - {%- endif -%} + {% if value -%} + + {{ format_predicate(key) }} + {{ value }} + + {%- endif -%} {%- endfor %} {% endif %} From 9f5d388ec01c195ffbacc4a1fd876fb507a6f62d Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Mon, 21 Apr 2014 19:07:28 -0400 Subject: [PATCH 42/78] In the middle of some major changes --- mediagoblin/gmg_commands/batchaddmedia.py | 26 +++++++++++++++---- .../mediagoblin/utils/metadata_table.html | 12 ++++----- 2 files changed, 27 insertions(+), 11 deletions(-) diff --git a/mediagoblin/gmg_commands/batchaddmedia.py b/mediagoblin/gmg_commands/batchaddmedia.py index 61a068d2..43c24f6d 100644 --- a/mediagoblin/gmg_commands/batchaddmedia.py +++ b/mediagoblin/gmg_commands/batchaddmedia.py @@ -132,7 +132,8 @@ zip files and directories" contents = all_metadata.read() media_metadata = parse_csv_file(contents) - dcterms_context = { 'dcterms':'http://purl.org/dc/terms/' } + metadata_context = { 'dcterms':'http://purl.org/dc/terms/', + 'xsd': 'http://www.w3.org/2001/XMLSchema#'} for media_id in media_locations.keys(): files_attempted += 1 @@ -141,13 +142,14 @@ zip files and directories" sanitized_metadata = check_metadata_format(file_metadata) if sanitized_metadata == {}: continue - json_ld_metadata = jsonld.compact(file_metadata, dcterms_context) + json_ld_metadata = jsonld.compact(build_json_ld_metadata(file_metadata), + metadata_context) original_location = media_locations[media_id]['media:original'] url = urlparse(original_location) - title = file_metadata.get('dcterms:title') - description = file_metadata.get('dcterms:description') - license = file_metadata.get('dcterms:rights') + title = sanitized_metadata.get('dcterms:title') + description = sanitized_metadata.get('dcterms:description') + license = sanitized_metadata.get('dcterms:rights') filename = url.path.split()[-1] if url.scheme == 'http': @@ -218,6 +220,19 @@ def teardown(temp_files): for temp_file in temp_files: subprocess.call(['rm','-r',temp_file]) +def build_json_ld_metadata(metadata_dict): + output_dict = {} + for p in metadata_dict.keys(): + if p in ["dcterms:rights", "dcterms:relation"]: + m_type = "xsd:uri" + elif p in ["dcterms:date", "dcterms:created"]: + m_type = "xsd:date" + else: + m_type = "xsd:string" + description = {"@value": metadata_dict[p], + "@type" : m_type} + output_dict[p] = description + return output_dict def check_metadata_format(metadata_dict): schema = { @@ -250,6 +265,7 @@ def check_metadata_format(metadata_dict): try: validate(metadata_dict, schema) output_dict = metadata_dict + # "media:id" is only for internal use, so we delete it for the output del output_dict['media:id'] except ValidationError, exc: diff --git a/mediagoblin/templates/mediagoblin/utils/metadata_table.html b/mediagoblin/templates/mediagoblin/utils/metadata_table.html index 166d5716..2eb57af3 100644 --- a/mediagoblin/templates/mediagoblin/utils/metadata_table.html +++ b/mediagoblin/templates/mediagoblin/utils/metadata_table.html @@ -20,14 +20,14 @@ {%- set metadata=media_entry.media_metadata %} {%- set metadata_context=metadata['@context'] %} {%- if metadata %} - - {%- for key, value in metadata.iteritems() if not key=='@context' %} - {% if value -%} +
+ {%- for key, value_dict in metadata.iteritems() if not key=='@context' %} + {% if value_dict['@value'] -%} - + {%- endif -%} {%- endfor %} From fffc5dcfe031d30551a91e668b377d443d9267db Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Tue, 6 May 2014 12:39:23 -0400 Subject: [PATCH 43/78] Created the media metadata editor page --- mediagoblin/edit/forms.py | 5 ++++ mediagoblin/edit/views.py | 13 ++++++++++- .../templates/mediagoblin/edit/metadata.html | 23 +++++++++++++++++++ mediagoblin/user_pages/routing.py | 4 ++++ 4 files changed, 44 insertions(+), 1 deletion(-) create mode 100644 mediagoblin/templates/mediagoblin/edit/metadata.html diff --git a/mediagoblin/edit/forms.py b/mediagoblin/edit/forms.py index 2c9b5e99..cff3a53f 100644 --- a/mediagoblin/edit/forms.py +++ b/mediagoblin/edit/forms.py @@ -122,3 +122,8 @@ class ChangeEmailForm(wtforms.Form): [wtforms.validators.Required()], description=_( "Enter your password to prove you own this account.")) + +class EditMetaDataForm(wtforms.Form): + media_metadata = wtforms.FieldList( + wtforms.TextField( + _(u'Value'))) diff --git a/mediagoblin/edit/views.py b/mediagoblin/edit/views.py index 80590875..e20d0ecc 100644 --- a/mediagoblin/edit/views.py +++ b/mediagoblin/edit/views.py @@ -29,7 +29,8 @@ from mediagoblin.edit import forms from mediagoblin.edit.lib import may_edit_media from mediagoblin.decorators import (require_active_login, active_user_from_url, get_media_entry_by_id, user_may_alter_collection, - get_user_collection) + get_user_collection, user_has_privilege, + user_not_banned) from mediagoblin.tools.crypto import get_timed_signer_url from mediagoblin.tools.mail import email_debug_message from mediagoblin.tools.response import (render_to_response, @@ -432,3 +433,13 @@ def change_email(request): 'mediagoblin/edit/change_email.html', {'form': form, 'user': user}) + +@user_has_privilege(u'admin') +@require_active_login +@get_media_entry_by_id +def edit_metadata(request, media): + form = forms.EditMetaDataForm() + return render_to_response( + request, + 'mediagoblin/edit/metadata.html', + {'form':form}) diff --git a/mediagoblin/templates/mediagoblin/edit/metadata.html b/mediagoblin/templates/mediagoblin/edit/metadata.html new file mode 100644 index 00000000..3f97555e --- /dev/null +++ b/mediagoblin/templates/mediagoblin/edit/metadata.html @@ -0,0 +1,23 @@ +{# +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +#} +{%- extends "mediagoblin/base.html" %} +{% import "/mediagoblin/utils/wtforms.html" as wtforms_util %} + +{% block mediagoblin_content %} + {{ wtforms_util.render_divs(form) }} +{% endblock mediagoblin_content %} diff --git a/mediagoblin/user_pages/routing.py b/mediagoblin/user_pages/routing.py index f0f4d8b7..8eb51c8d 100644 --- a/mediagoblin/user_pages/routing.py +++ b/mediagoblin/user_pages/routing.py @@ -101,3 +101,7 @@ add_route('mediagoblin.edit.edit_media', add_route('mediagoblin.edit.attachments', '/u//m//attachments/', 'mediagoblin.edit.views:edit_attachments') + +add_route('mediagoblin.edit.metadata', + '/u//m//metadata/', + 'mediagoblin.edit.views:edit_metadata') From f0cfd3396e2bcfd6a0b3eead1875efd0d29f0ff5 Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Tue, 6 May 2014 12:54:08 -0400 Subject: [PATCH 44/78] Set up the metadata editor forms --- mediagoblin/edit/forms.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/mediagoblin/edit/forms.py b/mediagoblin/edit/forms.py index cff3a53f..6cc7a9cb 100644 --- a/mediagoblin/edit/forms.py +++ b/mediagoblin/edit/forms.py @@ -123,7 +123,13 @@ class ChangeEmailForm(wtforms.Form): description=_( "Enter your password to prove you own this account.")) +class MetaDataForm(wtforms.Form): + identifier = wtforms.TextField( + _(u'Id')) + value = wtforms.TextField( + _(u'Value')) + class EditMetaDataForm(wtforms.Form): media_metadata = wtforms.FieldList( - wtforms.TextField( - _(u'Value'))) + wtforms.FormField(MetaDataForm) + ) From e80596c80eb06e6d199795e59dcc37b27d77fe55 Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Tue, 6 May 2014 17:00:25 -0400 Subject: [PATCH 45/78] Created a UI for editting a media's metadata. Had to add a new macro to wtforms.html in the process. --- mediagoblin/edit/forms.py | 11 +- mediagoblin/edit/views.py | 17 ++- mediagoblin/static/css/base.css | 14 +++ .../templates/mediagoblin/edit/metadata.html | 108 +++++++++++++++++- .../mediagoblin/utils/metadata_table.html | 6 + .../templates/mediagoblin/utils/wtforms.html | 18 +++ 6 files changed, 167 insertions(+), 7 deletions(-) diff --git a/mediagoblin/edit/forms.py b/mediagoblin/edit/forms.py index 6cc7a9cb..ce66526f 100644 --- a/mediagoblin/edit/forms.py +++ b/mediagoblin/edit/forms.py @@ -124,12 +124,13 @@ class ChangeEmailForm(wtforms.Form): "Enter your password to prove you own this account.")) class MetaDataForm(wtforms.Form): - identifier = wtforms.TextField( - _(u'Id')) - value = wtforms.TextField( - _(u'Value')) + identifier = wtforms.TextField('') + value = wtforms.TextField('') class EditMetaDataForm(wtforms.Form): media_metadata = wtforms.FieldList( - wtforms.FormField(MetaDataForm) + wtforms.FormField(MetaDataForm, label="") + ) + context = wtforms.FieldList( + wtforms.FormField(MetaDataForm, label="") ) diff --git a/mediagoblin/edit/views.py b/mediagoblin/edit/views.py index e20d0ecc..e3dd82ab 100644 --- a/mediagoblin/edit/views.py +++ b/mediagoblin/edit/views.py @@ -439,7 +439,22 @@ def change_email(request): @get_media_entry_by_id def edit_metadata(request, media): form = forms.EditMetaDataForm() + if media.media_metadata: + for row in media.media_metadata.iteritems(): + if row[0] == "@context": continue + identifier = row[0] + # TODO Will change when we revert the metadata branch + value = row[1]['@value'] + form.media_metadata.append_entry({ + 'identifier':identifier, + 'value':value}) + for row in media.media_metadata['@context'].iteritems(): + identifier, value = row[0:2] + form.context.append_entry({ + 'identifier':identifier, + 'value':value}) return render_to_response( request, 'mediagoblin/edit/metadata.html', - {'form':form}) + {'form':form, + 'media':media}) diff --git a/mediagoblin/static/css/base.css b/mediagoblin/static/css/base.css index 32c6c6cb..f0f9e3e6 100644 --- a/mediagoblin/static/css/base.css +++ b/mediagoblin/static/css/base.css @@ -938,3 +938,17 @@ p.verifier { none repeat scroll 0% 0% rgb(221, 221, 221); padding: 1em 0px; } + +/* for the media metadata editing table */ +table.metadata_editor { + + margin: 10px auto; + width: 800px; +} + +table.metadata_editor tr td { + width:350px; +} +table.metadata_editor tr td input.form_field_input { + width: 300px +} diff --git a/mediagoblin/templates/mediagoblin/edit/metadata.html b/mediagoblin/templates/mediagoblin/edit/metadata.html index 3f97555e..cbf74106 100644 --- a/mediagoblin/templates/mediagoblin/edit/metadata.html +++ b/mediagoblin/templates/mediagoblin/edit/metadata.html @@ -17,7 +17,113 @@ #} {%- extends "mediagoblin/base.html" %} {% import "/mediagoblin/utils/wtforms.html" as wtforms_util %} +{% block mediagoblin_head %} + +{% endblock %} {% block mediagoblin_content %} - {{ wtforms_util.render_divs(form) }} +

{% trans media_name=media.title -%} + Metadata for "{{ media_name }}"{% endtrans %}

+
+ +

{% trans %}Context{% endtrans %}

+
{{ format_predicate(key) }}{{ value }} + {{ value_dict['@value'] }}
+ + + + + {% for miniform in form.context -%} + {{ wtforms_util.render_table_row(miniform) }} + {% endfor -%} +
+ {% trans %}Identifier{% endtrans %}{% trans %}Value{% endtrans %}
+ + + + + + +
+ + +

{% trans %}Data{% endtrans %}

+ + + + + + {% for miniform in form.media_metadata -%} + {{ wtforms_util.render_table_row(miniform) }} + {% endfor -%} +
+ {% trans %}Identifier{% endtrans %}{% trans %}Value{% endtrans %}
+ + + + + + + + + + +
+
+ {{ csrf_token }} + + {% endblock mediagoblin_content %} diff --git a/mediagoblin/templates/mediagoblin/utils/metadata_table.html b/mediagoblin/templates/mediagoblin/utils/metadata_table.html index 2eb57af3..0c67264a 100644 --- a/mediagoblin/templates/mediagoblin/utils/metadata_table.html +++ b/mediagoblin/templates/mediagoblin/utils/metadata_table.html @@ -33,4 +33,10 @@ {%- endfor %} {% endif %} + {% if request.user and request.user.has_privilege('admin') %} + + {% trans %}Edit Metadata{% endtrans %} + {% endif %} {%- endmacro %} diff --git a/mediagoblin/templates/mediagoblin/utils/wtforms.html b/mediagoblin/templates/mediagoblin/utils/wtforms.html index e079274e..e861b674 100644 --- a/mediagoblin/templates/mediagoblin/utils/wtforms.html +++ b/mediagoblin/templates/mediagoblin/utils/wtforms.html @@ -87,6 +87,24 @@ {% endfor %} {%- endmacro %} +{% macro render_table_row(form) %} + + {%- for field in form %} + + {{field}} + {%- if field.errors -%} +
+
    + {% for error in field.errors %} +
  • {{error}}
    • + {%- endfor %} +
+ {%- endif -%} + + {%- endfor %} + +{%- endmacro %} + {# Render a boolean field #} {% macro render_bool(field) %}
From 9919fb08a4d56f5976a6fef0fc70b2ee6c751759 Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Tue, 6 May 2014 17:19:30 -0400 Subject: [PATCH 46/78] Made it so the metadata editting page is only one step away from functioning correctly. --- mediagoblin/edit/views.py | 16 +++++++++++++++- .../templates/mediagoblin/edit/metadata.html | 10 +++++----- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/mediagoblin/edit/views.py b/mediagoblin/edit/views.py index e3dd82ab..496df6b9 100644 --- a/mediagoblin/edit/views.py +++ b/mediagoblin/edit/views.py @@ -17,6 +17,7 @@ from datetime import datetime from itsdangerous import BadSignature +from pyld import jsonld from werkzeug.exceptions import Forbidden from werkzeug.utils import secure_filename @@ -438,7 +439,20 @@ def change_email(request): @require_active_login @get_media_entry_by_id def edit_metadata(request, media): - form = forms.EditMetaDataForm() + form = forms.EditMetaDataForm(request.form) + if request.method == "POST" and form.validate(): + context = dict([(row['identifier'],row['value']) + for row in form.context.data]) + metadata_dict = dict([(row['identifier'],row['value']) + for row in form.media_metadata.data]) + # TODO VALIDATE THIS BEFORE WE ENTER IT + # validate(metadata_dict) + # validate(context) + json_ld_metadata = jsonld.compact(metadata_dict, context) + # media.media_metadata = json_ld_metadata + # media.save() + return redirect_obj(request, media) + if media.media_metadata: for row in media.media_metadata.iteritems(): if row[0] == "@context": continue diff --git a/mediagoblin/templates/mediagoblin/edit/metadata.html b/mediagoblin/templates/mediagoblin/edit/metadata.html index cbf74106..364cad0d 100644 --- a/mediagoblin/templates/mediagoblin/edit/metadata.html +++ b/mediagoblin/templates/mediagoblin/edit/metadata.html @@ -41,7 +41,7 @@ $('table'+list_id+' tr').each(function(row){ id_input = $(this).find('td').find('input'); value_input = $(this).find('td').next().find('input'); - if ((value_input.attr('value') == "") && + if ((value_input.attr('value') == "") && (id_input.attr('value') == "")) { $(this).remove(); } @@ -52,13 +52,13 @@ var context_lines = {{ form.context | length }}; var metadata_lines = {{ form.media_metadata | length }}; $("#add_new_metadata_row").click(function(){ - add_new_row("#metadata_list", + add_new_row("#metadata_list", metadata_lines, 'media_metadata-'); metadata_lines += 1; }) $("#add_new_context_row").click(function(){ - add_new_row("#context_list", + add_new_row("#context_list", context_lines, 'context-'); context_lines += 1; @@ -115,7 +115,7 @@ - @@ -125,5 +125,5 @@ {{ csrf_token }} - + {% endblock mediagoblin_content %} From f73585be4739beef5c61c41de9a045518be7e311 Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Tue, 6 May 2014 17:23:22 -0400 Subject: [PATCH 47/78] Fixed a slight css error. --- mediagoblin/static/css/base.css | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediagoblin/static/css/base.css b/mediagoblin/static/css/base.css index f0f9e3e6..d0396ceb 100644 --- a/mediagoblin/static/css/base.css +++ b/mediagoblin/static/css/base.css @@ -949,6 +949,6 @@ table.metadata_editor { table.metadata_editor tr td { width:350px; } -table.metadata_editor tr td input.form_field_input { +table.metadata_editor tr td.form_field_input input { width: 300px } From d015e4a84dfcde0bef510228f5b8f23a7c895a34 Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Tue, 6 May 2014 17:45:43 -0400 Subject: [PATCH 48/78] Added in a few blank lines when a user edits the metadata of a file that has none. --- mediagoblin/edit/views.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/mediagoblin/edit/views.py b/mediagoblin/edit/views.py index 496df6b9..4ab3fe01 100644 --- a/mediagoblin/edit/views.py +++ b/mediagoblin/edit/views.py @@ -467,6 +467,16 @@ def edit_metadata(request, media): form.context.append_entry({ 'identifier':identifier, 'value':value}) + else: + form.media_metadata.append_entry({ + 'identifier':"", + 'value':""}) + form.media_metadata.append_entry({ + 'identifier':"", + 'value':""}) + form.context.append_entry({ + 'identifier':"", + 'value':""}) return render_to_response( request, 'mediagoblin/edit/metadata.html', From af3a9107a9aef453b62f8fd83e03e9a1bbe416b8 Mon Sep 17 00:00:00 2001 From: Christopher Allan Webber Date: Wed, 7 May 2014 13:36:52 -0500 Subject: [PATCH 49/78] The URL format checker now works correctly ...though it isn't checking the right thing --- mediagoblin/gmg_commands/batchaddmedia.py | 77 +++++++++++++---------- 1 file changed, 44 insertions(+), 33 deletions(-) diff --git a/mediagoblin/gmg_commands/batchaddmedia.py b/mediagoblin/gmg_commands/batchaddmedia.py index 43c24f6d..41fb86c9 100644 --- a/mediagoblin/gmg_commands/batchaddmedia.py +++ b/mediagoblin/gmg_commands/batchaddmedia.py @@ -15,7 +15,7 @@ # along with this program. If not, see . import os -import tempfile, tarfile, zipfile, subprocess, requests +import copy, tempfile, tarfile, zipfile, subprocess, re, requests from csv import reader as csv_reader from urlparse import urlparse from pyld import jsonld @@ -27,8 +27,10 @@ from mediagoblin.submit.lib import ( from mediagoblin.tools.translate import lazy_pass_to_ugettext as _ from mediagoblin import mg_globals -from jsonschema import validate +from jsonschema import validate, FormatChecker, draft4_format_checker from jsonschema.exceptions import ValidationError +from jsonschema.compat import str_types + def parser_setup(subparser): subparser.description = """\ @@ -48,11 +50,6 @@ Core properties (http://dublincore.org/documents/dces/). Both "location.csv" and "metadata.csv" must begin with a row demonstrating the order of the columns. We have provided an example of these files at """)) - subparser.add_argument( - "-l", "--license", - help=( - "License these media entry will be released under, if all the same. " - "Should be a URL.")) subparser.add_argument( '--celery', action='store_true', @@ -149,6 +146,8 @@ zip files and directories" title = sanitized_metadata.get('dcterms:title') description = sanitized_metadata.get('dcterms:description') + + # TODO: this isn't the same thing license = sanitized_metadata.get('dcterms:rights') filename = url.path.split()[-1] @@ -234,36 +233,48 @@ def build_json_ld_metadata(metadata_dict): output_dict[p] = description return output_dict + +## Set up the MediaGoblin checker +# + +URL_REGEX = re.compile( + r'^[a-z]+://([^/:]+|([0-9]{1,3}\.){3}[0-9]{1,3})(:[0-9]+)?(\/.*)?$', + re.IGNORECASE) + +def is_uri(instance): + if not isinstance(instance, str_types): + return True + + return URL_REGEX.match(instance) + + +class DefaultChecker(FormatChecker): + checkers = copy.deepcopy(draft4_format_checker.checkers) + +DefaultChecker.checkers[u"uri"] = (is_uri, ()) + +DEFAULT_CHECKER = DefaultChecker() + def check_metadata_format(metadata_dict): schema = { - "$schema":"http://json-schema.org/schema#", - "properties":{ - "media:id":{}, - "dcterms:contributor":{}, - "dcterms:coverage":{}, - "dcterms:created":{}, - "dcterms:creator":{}, - "dcterms:date":{}, - "dcterms:description":{}, - "dcterms:format":{}, - "dcterms:identifier":{}, - "dcterms:language":{}, - "dcterms:publisher":{}, - "dcterms:relation":{}, - "dcterms:rights" : { - "format":"uri", - "type":"string" + "$schema": "http://json-schema.org/schema#", + + "type": "object", + "properties": { + "dcterms:rights": { + "format": "uri", + "type": "string", + }, + "dcterms:created": { + + } }, - "dcterms:source":{}, - "dcterms:subject":{}, - "dcterms:title":{}, - "dcterms:type":{} - }, - "additionalProperties": False, - "required":["dcterms:title","media:id"] -} + # "required": ["dcterms:title", "media:id"], + } + try: - validate(metadata_dict, schema) + validate(metadata_dict, schema, + format_checker=DEFAULT_CHECKER) output_dict = metadata_dict # "media:id" is only for internal use, so we delete it for the output del output_dict['media:id'] From 2a2c534e51f47cbf7c537c9b659648f23146a64b Mon Sep 17 00:00:00 2001 From: Christopher Allan Webber Date: Wed, 7 May 2014 15:21:10 -0500 Subject: [PATCH 50/78] Removing build_json_ld_metadata --- mediagoblin/gmg_commands/batchaddmedia.py | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/mediagoblin/gmg_commands/batchaddmedia.py b/mediagoblin/gmg_commands/batchaddmedia.py index 41fb86c9..cf362d83 100644 --- a/mediagoblin/gmg_commands/batchaddmedia.py +++ b/mediagoblin/gmg_commands/batchaddmedia.py @@ -139,8 +139,7 @@ zip files and directories" sanitized_metadata = check_metadata_format(file_metadata) if sanitized_metadata == {}: continue - json_ld_metadata = jsonld.compact(build_json_ld_metadata(file_metadata), - metadata_context) + json_ld_metadata = jsonld.compact(file_metadata, metadata_context) original_location = media_locations[media_id]['media:original'] url = urlparse(original_location) @@ -219,20 +218,6 @@ def teardown(temp_files): for temp_file in temp_files: subprocess.call(['rm','-r',temp_file]) -def build_json_ld_metadata(metadata_dict): - output_dict = {} - for p in metadata_dict.keys(): - if p in ["dcterms:rights", "dcterms:relation"]: - m_type = "xsd:uri" - elif p in ["dcterms:date", "dcterms:created"]: - m_type = "xsd:date" - else: - m_type = "xsd:string" - description = {"@value": metadata_dict[p], - "@type" : m_type} - output_dict[p] = description - return output_dict - ## Set up the MediaGoblin checker # From a4486286363cca8d0ef9d1026883b13e7f84d8e0 Mon Sep 17 00:00:00 2001 From: Christopher Allan Webber Date: Wed, 7 May 2014 15:21:58 -0500 Subject: [PATCH 51/78] Removing unused variables --- mediagoblin/gmg_commands/batchaddmedia.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/mediagoblin/gmg_commands/batchaddmedia.py b/mediagoblin/gmg_commands/batchaddmedia.py index cf362d83..07c0b3fc 100644 --- a/mediagoblin/gmg_commands/batchaddmedia.py +++ b/mediagoblin/gmg_commands/batchaddmedia.py @@ -26,7 +26,6 @@ from mediagoblin.submit.lib import ( FileUploadLimit, UserUploadLimit, UserPastUploadLimit) from mediagoblin.tools.translate import lazy_pass_to_ugettext as _ -from mediagoblin import mg_globals from jsonschema import validate, FormatChecker, draft4_format_checker from jsonschema.exceptions import ValidationError from jsonschema.compat import str_types @@ -99,14 +98,12 @@ zip files and directories" metadata_file_path = os.path.join(dir_path, "metadata.csv") # check for the location file, if it exists... - location_filename = os.path.split(location_file_path)[-1] abs_location_filename = os.path.abspath(location_file_path) if not os.path.exists(abs_location_filename): print "Can't find a file with filename '%s'" % location_file_path return # check for the metadata file, if it exists... - metadata_filename = os.path.split(metadata_file_path)[-1] abs_metadata_filename = os.path.abspath(metadata_file_path) if not os.path.exists(abs_metadata_filename): print "Can't find a file with filename '%s'" % metadata_file_path From e5e2cc2f16f47fb28f5aef256652e2f2e20eb45d Mon Sep 17 00:00:00 2001 From: Christopher Allan Webber Date: Wed, 7 May 2014 18:41:34 -0500 Subject: [PATCH 52/78] Starting to add metadata tools, as well as mediagoblin's schema --- .../metadata/mediagoblin-0.1.dev.jsonld | 47 ++++++++ mediagoblin/tools/metadata.py | 106 ++++++++++++++++++ 2 files changed, 153 insertions(+) create mode 100644 mediagoblin/static/metadata/mediagoblin-0.1.dev.jsonld create mode 100644 mediagoblin/tools/metadata.py diff --git a/mediagoblin/static/metadata/mediagoblin-0.1.dev.jsonld b/mediagoblin/static/metadata/mediagoblin-0.1.dev.jsonld new file mode 100644 index 00000000..20a71b53 --- /dev/null +++ b/mediagoblin/static/metadata/mediagoblin-0.1.dev.jsonld @@ -0,0 +1,47 @@ +{ + "@context": { + "qb": "http://purl.org/linked-data/cube#", + "grddl": "http://www.w3.org/2003/g/data-view#", + "ma": "http://www.w3.org/ns/ma-ont#", + "owl": "http://www.w3.org/2002/07/owl#", + "rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#", + "rdfa": "http://www.w3.org/ns/rdfa#", + "rdfs": "http://www.w3.org/2000/01/rdf-schema#", + "rif": "http://www.w3.org/2007/rif#", + "rr": "http://www.w3.org/ns/r2rml#", + "skos": "http://www.w3.org/2004/02/skos/core#", + "skosxl": "http://www.w3.org/2008/05/skos-xl#", + "wdr": "http://www.w3.org/2007/05/powder#", + "void": "http://rdfs.org/ns/void#", + "wdrs": "http://www.w3.org/2007/05/powder-s#", + "xhv": "http://www.w3.org/1999/xhtml/vocab#", + "xml": "http://www.w3.org/XML/1998/namespace", + "xsd": "http://www.w3.org/2001/XMLSchema#", + "prov": "http://www.w3.org/ns/prov#", + "sd": "http://www.w3.org/ns/sparql-service-description#", + "org": "http://www.w3.org/ns/org#", + "gldp": "http://www.w3.org/ns/people#", + "cnt": "http://www.w3.org/2008/content#", + "dcat": "http://www.w3.org/ns/dcat#", + "earl": "http://www.w3.org/ns/earl#", + "ht": "http://www.w3.org/2006/http#", + "ptr": "http://www.w3.org/2009/pointers#", + "cc": "http://creativecommons.org/ns#", + "ctag": "http://commontag.org/ns#", + "dc": "http://purl.org/dc/terms/", + "dc11": "http://purl.org/dc/elements/1.1/", + "dcterms": "http://purl.org/dc/terms/", + "foaf": "http://xmlns.com/foaf/0.1/", + "gr": "http://purl.org/goodrelations/v1#", + "ical": "http://www.w3.org/2002/12/cal/icaltzd#", + "og": "http://ogp.me/ns#", + "rev": "http://purl.org/stuff/rev#", + "sioc": "http://rdfs.org/sioc/ns#", + "v": "http://rdf.data-vocabulary.org/#", + "vcard": "http://www.w3.org/2006/vcard/ns#", + "schema": "http://schema.org/", + "describedby": "http://www.w3.org/2007/05/powder-s#describedby", + "license": "http://www.w3.org/1999/xhtml/vocab#license", + "role": "http://www.w3.org/1999/xhtml/vocab#role" + } +} diff --git a/mediagoblin/tools/metadata.py b/mediagoblin/tools/metadata.py new file mode 100644 index 00000000..428e425c --- /dev/null +++ b/mediagoblin/tools/metadata.py @@ -0,0 +1,106 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + + +import os +import copy +import json +import re +from pkg_resources import resource_filename + +import dateutil.parser +from pyld import jsonld +from jsonschema import validate, FormatChecker, draft4_format_checker +from jsonschema.compat import str_types + + +MEDIAGOBLIN_CONTEXT_PATH = resource_filename( + "mediagoblin", + os.path.sep.join(["static", "metadata", "mediagoblin-0.1.dev.jsonld"])) +MEDIAGOBLIN_CONTEXT = json.loads(file(MEDIAGOBLIN_CONTEXT_PATH).read()) + + +######################################################## +## Set up the MediaGoblin format checker for json-schema +######################################################## + +URL_REGEX = re.compile( + r'^[a-z]+://([^/:]+|([0-9]{1,3}\.){3}[0-9]{1,3})(:[0-9]+)?(\/.*)?$', + re.IGNORECASE) + +def is_uri(instance): + """ + jsonschema uri validator + """ + if not isinstance(instance, str_types): + return True + + return URL_REGEX.match(instance) + +def is_datetime(instance): + """ + Is a date or datetime readable string. + """ + if not isinstance(instance, str_types): + return True + + return dateutil.parser.parse(instance) + + +class DefaultChecker(FormatChecker): + """ + Default MediaGoblin format checker... extended to include a few extra things + """ + checkers = copy.deepcopy(draft4_format_checker.checkers) + + +DefaultChecker.checkers[u"uri"] = (is_uri, ()) +DefaultChecker.checkers[u"date-time"] = (is_datetime, (ValueError, TypeError)) +DEFAULT_CHECKER = DefaultChecker() + +# Crappy default schema, checks for things we deem important + +DEFAULT_SCHEMA = { + "$schema": "http://json-schema.org/schema#", + + "type": "object", + "properties": { + "dcterms:rights": { + "format": "uri", + "type": "string", + }, + "dcterms:created": { + "format": "date-time", + "type": "string", + } + }, +} + + +def compact_and_validate(metadata, context=MEDIAGOBLIN_CONTEXT, + schema=DEFAULT_SCHEMA): + """ + compact json with supplied context, check against schema for errors + + raises an exception (jsonschema.exceptions.ValidationError) if + there's an error. + + You may wish to do this validation yourself... this is just for convenience. + """ + compacted = jsonld.compact(metadata, context) + validate(metadata, schema, format_checker=DEFAULT_CHECKER) + + return compacted From 6fab7734d6b6817b310865409f260ab87907eaa0 Mon Sep 17 00:00:00 2001 From: Christopher Allan Webber Date: Wed, 7 May 2014 18:50:48 -0500 Subject: [PATCH 53/78] Updating batchaddmedia to use new metadata tools --- mediagoblin/gmg_commands/batchaddmedia.py | 99 +++-------------------- mediagoblin/tools/metadata.py | 4 +- 2 files changed, 14 insertions(+), 89 deletions(-) diff --git a/mediagoblin/gmg_commands/batchaddmedia.py b/mediagoblin/gmg_commands/batchaddmedia.py index 07c0b3fc..e540e88c 100644 --- a/mediagoblin/gmg_commands/batchaddmedia.py +++ b/mediagoblin/gmg_commands/batchaddmedia.py @@ -15,7 +15,7 @@ # along with this program. If not, see . import os -import copy, tempfile, tarfile, zipfile, subprocess, re, requests +import tempfile, tarfile, zipfile, subprocess, requests from csv import reader as csv_reader from urlparse import urlparse from pyld import jsonld @@ -24,11 +24,9 @@ from mediagoblin.gmg_commands import util as commands_util from mediagoblin.submit.lib import ( submit_media, get_upload_file_limits, FileUploadLimit, UserUploadLimit, UserPastUploadLimit) -from mediagoblin.tools.translate import lazy_pass_to_ugettext as _ +from mediagoblin.tools.metadata import compact_and_validate -from jsonschema import validate, FormatChecker, draft4_format_checker from jsonschema.exceptions import ValidationError -from jsonschema.compat import str_types def parser_setup(subparser): @@ -126,25 +124,24 @@ zip files and directories" contents = all_metadata.read() media_metadata = parse_csv_file(contents) - metadata_context = { 'dcterms':'http://purl.org/dc/terms/', - 'xsd': 'http://www.w3.org/2001/XMLSchema#'} - for media_id in media_locations.keys(): files_attempted += 1 - file_metadata = media_metadata[media_id] - sanitized_metadata = check_metadata_format(file_metadata) - if sanitized_metadata == {}: continue + file_metadata = media_metadata[media_id] + try: + json_ld_metadata = compact_and_validate(file_metadata) + except ValidationError, exc: + print "Error with '%s' value '%s': %s" % ( + media_id, exc.path[0], exc.message) + continue - json_ld_metadata = jsonld.compact(file_metadata, metadata_context) original_location = media_locations[media_id]['media:original'] url = urlparse(original_location) - title = sanitized_metadata.get('dcterms:title') - description = sanitized_metadata.get('dcterms:description') + title = json_ld_metadata.get('dcterms:title') + description = json_ld_metadata.get('dcterms:description') - # TODO: this isn't the same thing - license = sanitized_metadata.get('dcterms:rights') + license = json_ld_metadata.get('license') filename = url.path.split()[-1] if url.scheme == 'http': @@ -214,75 +211,3 @@ def parse_csv_file(file_contents): def teardown(temp_files): for temp_file in temp_files: subprocess.call(['rm','-r',temp_file]) - - -## Set up the MediaGoblin checker -# - -URL_REGEX = re.compile( - r'^[a-z]+://([^/:]+|([0-9]{1,3}\.){3}[0-9]{1,3})(:[0-9]+)?(\/.*)?$', - re.IGNORECASE) - -def is_uri(instance): - if not isinstance(instance, str_types): - return True - - return URL_REGEX.match(instance) - - -class DefaultChecker(FormatChecker): - checkers = copy.deepcopy(draft4_format_checker.checkers) - -DefaultChecker.checkers[u"uri"] = (is_uri, ()) - -DEFAULT_CHECKER = DefaultChecker() - -def check_metadata_format(metadata_dict): - schema = { - "$schema": "http://json-schema.org/schema#", - - "type": "object", - "properties": { - "dcterms:rights": { - "format": "uri", - "type": "string", - }, - "dcterms:created": { - - } - }, - # "required": ["dcterms:title", "media:id"], - } - - try: - validate(metadata_dict, schema, - format_checker=DEFAULT_CHECKER) - output_dict = metadata_dict - # "media:id" is only for internal use, so we delete it for the output - del output_dict['media:id'] - - except ValidationError, exc: - title = (metadata_dict.get('dcterms:title') or - metadata_dict.get('media:id') or _(u'UNKNOWN FILE')) - - if exc.validator == "additionalProperties": - message = _(u'Invalid metadata provided for file "{title}". This \ -script only accepts the Dublin Core metadata terms.'.format(title=title)) - - elif exc.validator == "required": - message = _( -u'All necessary metadata was not provided for file "{title}", you must include \ -a "dcterms:title" column for each media file'.format(title=title)) - - else: - message = _(u'Could not find appropriate metadata for file \ -"{title}".'.format(title=title)) - - print _(u"""WARN: {message} \nSkipping File...\n""".format( - message=message)) - - output_dict = {} - except: - raise - - return output_dict diff --git a/mediagoblin/tools/metadata.py b/mediagoblin/tools/metadata.py index 428e425c..c49bcaaf 100644 --- a/mediagoblin/tools/metadata.py +++ b/mediagoblin/tools/metadata.py @@ -78,7 +78,7 @@ DEFAULT_SCHEMA = { "type": "object", "properties": { - "dcterms:rights": { + "license": { "format": "uri", "type": "string", }, @@ -96,7 +96,7 @@ def compact_and_validate(metadata, context=MEDIAGOBLIN_CONTEXT, compact json with supplied context, check against schema for errors raises an exception (jsonschema.exceptions.ValidationError) if - there's an error. + there's an error.9 You may wish to do this validation yourself... this is just for convenience. """ From 96afe1b268deb97518f68a582f5693646c577cf6 Mon Sep 17 00:00:00 2001 From: Christopher Allan Webber Date: Fri, 9 May 2014 14:10:27 -0500 Subject: [PATCH 54/78] Compact and validate tools, loading internal json tools, much more This commit sponsored by Caleb Nidey. Thank you! --- mediagoblin/tools/metadata.py | 93 ++++++++++++++++++++++++++++++++--- 1 file changed, 86 insertions(+), 7 deletions(-) diff --git a/mediagoblin/tools/metadata.py b/mediagoblin/tools/metadata.py index c49bcaaf..37d04fa1 100644 --- a/mediagoblin/tools/metadata.py +++ b/mediagoblin/tools/metadata.py @@ -26,11 +26,8 @@ from pyld import jsonld from jsonschema import validate, FormatChecker, draft4_format_checker from jsonschema.compat import str_types +from mediagoblin.tools.pluginapi import hook_handle -MEDIAGOBLIN_CONTEXT_PATH = resource_filename( - "mediagoblin", - os.path.sep.join(["static", "metadata", "mediagoblin-0.1.dev.jsonld"])) -MEDIAGOBLIN_CONTEXT = json.loads(file(MEDIAGOBLIN_CONTEXT_PATH).read()) ######################################################## @@ -90,17 +87,99 @@ DEFAULT_SCHEMA = { } -def compact_and_validate(metadata, context=MEDIAGOBLIN_CONTEXT, +def load_resource(package, resource_path): + """ + Load a resource, return it as a string. + + Args: + - package: package or module name. Eg "mediagoblin.media_types.audio" + - resource_path: path to get to this resource, a list of + directories and finally a filename. Will be joined with + os.path.sep. + """ + filename = resource_filename(package, os.path.sep.join(resource_path)) + return file(filename).read() + +def load_resource_json(package, resource_path): + """ + Load a resource json file, return a dictionary. + + Args: + - package: package or module name. Eg "mediagoblin.media_types.audio" + - resource_path: path to get to this resource, a list of + directories and finally a filename. Will be joined with + os.path.sep. + """ + return json.loads(load_resource(package, resource_path)) + + +################################## +## Load the MediaGoblin core files +################################## + + +BUILTIN_CONTEXTS = { + "http://www.w3.org/2013/json-ld-context/rdfa11": load_resource( + "mediagoblin", ["static", "metadata", "rdfa11.jsonld"])} + + +_CONTEXT_CACHE = {} + +def load_context(url): + """ + A self-aware document loader. For those contexts MediaGoblin + stores internally, load them from disk. + """ + if url in _CONTEXT_CACHE: + return _CONTEXT_CACHE[url] + + # See if it's one of our basic ones + document = BUILTIN_CONTEXTS.get(url, None) + + # No? See if we have an internal schema for this + if document is None: + document = hook_handle(("context_url_data", url)) + + # Okay, if we've gotten a document by now... let's package it up + if document is not None: + document = {'contextUrl': None, + 'documentUrl': url, + 'document': document} + + # Otherwise, use jsonld.load_document + else: + document = jsonld.load_document(url) + + # cache + _CONTEXT_CACHE[url] = document + return document + + +DEFAULT_CONTEXT = "http://www.w3.org/2013/json-ld-context/rdfa11" + +def compact_and_validate(metadata, context=DEFAULT_CONTEXT, schema=DEFAULT_SCHEMA): """ compact json with supplied context, check against schema for errors raises an exception (jsonschema.exceptions.ValidationError) if - there's an error.9 + there's an error. + + Note: Free floating" nodes are removed (eg a key just named + "bazzzzzz" which isn't specified in the context... something like + bazzzzzz:blerp will stay though. This is jsonld.compact behavior. You may wish to do this validation yourself... this is just for convenience. """ - compacted = jsonld.compact(metadata, context) + compacted = jsonld.compact( + metadata, context, + options={ + "documentLoader": load_context, + # This allows for things like "license" and etc to be preserved + "expandContext": context, + "keepFreeFloatingNodes": False}) validate(metadata, schema, format_checker=DEFAULT_CHECKER) return compacted + + From 3850b3fb56a2a5edc869af68ae40fc3f2873dc22 Mon Sep 17 00:00:00 2001 From: Christopher Allan Webber Date: Fri, 9 May 2014 14:32:46 -0500 Subject: [PATCH 55/78] Removing the mediagoblin.jsonld file, which really was just the RDFa 1.1 file. This commit sponsored by Matthew Cope. Thanks! --- .../metadata/{mediagoblin-0.1.dev.jsonld => rdfa11.jsonld} | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) rename mediagoblin/static/metadata/{mediagoblin-0.1.dev.jsonld => rdfa11.jsonld} (97%) diff --git a/mediagoblin/static/metadata/mediagoblin-0.1.dev.jsonld b/mediagoblin/static/metadata/rdfa11.jsonld similarity index 97% rename from mediagoblin/static/metadata/mediagoblin-0.1.dev.jsonld rename to mediagoblin/static/metadata/rdfa11.jsonld index 20a71b53..b2557233 100644 --- a/mediagoblin/static/metadata/mediagoblin-0.1.dev.jsonld +++ b/mediagoblin/static/metadata/rdfa11.jsonld @@ -1,5 +1,6 @@ { "@context": { + "cat": "http://www.w3.org/ns/dcat#", "qb": "http://purl.org/linked-data/cube#", "grddl": "http://www.w3.org/2003/g/data-view#", "ma": "http://www.w3.org/ns/ma-ont#", @@ -44,4 +45,4 @@ "license": "http://www.w3.org/1999/xhtml/vocab#license", "role": "http://www.w3.org/1999/xhtml/vocab#role" } -} +} \ No newline at end of file From a468db099c20fc666fafa4ce399fd04e7e6f2cbe Mon Sep 17 00:00:00 2001 From: Christopher Allan Webber Date: Fri, 9 May 2014 16:49:42 -0500 Subject: [PATCH 56/78] expand_json utility... yet another convenience function :) This commit sponsored by Benjamin Lebsanft. Thank you! --- mediagoblin/tools/metadata.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/mediagoblin/tools/metadata.py b/mediagoblin/tools/metadata.py index 37d04fa1..dde5753e 100644 --- a/mediagoblin/tools/metadata.py +++ b/mediagoblin/tools/metadata.py @@ -183,3 +183,18 @@ def compact_and_validate(metadata, context=DEFAULT_CONTEXT, return compacted +def expand_json(metadata, context=DEFAULT_CONTEXT): + """ + Expand json, but be sure to use our documentLoader. + + By default this expands with DEFAULT_CONTEXT, but if you do not need this, + you can safely set this to None. + + # @@: Is the above a good idea? Maybe it should be set to None by + # default. + """ + options = { + "documentLoader": load_context} + if context is not None: + options["expandContext"] = context + return jsonld.expand(metadata, options=options) From fd7069632b0aa6064e0abe1f3c0c129fb316faa6 Mon Sep 17 00:00:00 2001 From: Christopher Allan Webber Date: Fri, 9 May 2014 17:33:58 -0500 Subject: [PATCH 57/78] Splitting up compact_json from compact_and_validate This commit sponsored by Harry Rogoff. Thank you! --- mediagoblin/tools/metadata.py | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/mediagoblin/tools/metadata.py b/mediagoblin/tools/metadata.py index dde5753e..7de5a514 100644 --- a/mediagoblin/tools/metadata.py +++ b/mediagoblin/tools/metadata.py @@ -157,6 +157,25 @@ def load_context(url): DEFAULT_CONTEXT = "http://www.w3.org/2013/json-ld-context/rdfa11" +def compact_json(metadata, context=DEFAULT_CONTEXT): + """ + Compact json with supplied context. + + Note: Free floating" nodes are removed (eg a key just named + "bazzzzzz" which isn't specified in the context... something like + bazzzzzz:blerp will stay though. This is jsonld.compact behavior. + """ + compacted = jsonld.compact( + metadata, context, + options={ + "documentLoader": load_context, + # This allows for things like "license" and etc to be preserved + "expandContext": context, + "keepFreeFloatingNodes": False}) + + return compacted + + def compact_and_validate(metadata, context=DEFAULT_CONTEXT, schema=DEFAULT_SCHEMA): """ @@ -171,13 +190,7 @@ def compact_and_validate(metadata, context=DEFAULT_CONTEXT, You may wish to do this validation yourself... this is just for convenience. """ - compacted = jsonld.compact( - metadata, context, - options={ - "documentLoader": load_context, - # This allows for things like "license" and etc to be preserved - "expandContext": context, - "keepFreeFloatingNodes": False}) + compacted = compact_json(metadata, context) validate(metadata, schema, format_checker=DEFAULT_CHECKER) return compacted From fbea284aebafc9d41175aa2ee924012520c43b82 Mon Sep 17 00:00:00 2001 From: Christopher Allan Webber Date: Sun, 11 May 2014 18:47:55 -0500 Subject: [PATCH 58/78] Removing the prefix stuff --- mediagoblin/templates/mediagoblin/utils/metadata_table.html | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mediagoblin/templates/mediagoblin/utils/metadata_table.html b/mediagoblin/templates/mediagoblin/utils/metadata_table.html index 2eb57af3..3281c81c 100644 --- a/mediagoblin/templates/mediagoblin/utils/metadata_table.html +++ b/mediagoblin/templates/mediagoblin/utils/metadata_table.html @@ -20,8 +20,9 @@ {%- set metadata=media_entry.media_metadata %} {%- set metadata_context=metadata['@context'] %} {%- if metadata %} - + {#- NOTE: In some smart future where the context is more extensible, + we will need to add to the prefix here-#} +
{%- for key, value_dict in metadata.iteritems() if not key=='@context' %} {% if value_dict['@value'] -%} From 494bce47f92165f322347003baac22731e0ee7aa Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Mon, 12 May 2014 12:20:03 -0400 Subject: [PATCH 59/78] Changed the format of the wtforms table slightly --- mediagoblin/edit/forms.py | 4 +- mediagoblin/static/css/base.css | 10 +++-- .../templates/mediagoblin/edit/metadata.html | 23 +++------- .../templates/mediagoblin/utils/wtforms.html | 45 ++++++++++++------- 4 files changed, 46 insertions(+), 36 deletions(-) diff --git a/mediagoblin/edit/forms.py b/mediagoblin/edit/forms.py index ce66526f..7ddf603e 100644 --- a/mediagoblin/edit/forms.py +++ b/mediagoblin/edit/forms.py @@ -124,8 +124,8 @@ class ChangeEmailForm(wtforms.Form): "Enter your password to prove you own this account.")) class MetaDataForm(wtforms.Form): - identifier = wtforms.TextField('') - value = wtforms.TextField('') + identifier = wtforms.TextField(_(u'Identifier')) + value = wtforms.TextField(_(u'Value')) class EditMetaDataForm(wtforms.Form): media_metadata = wtforms.FieldList( diff --git a/mediagoblin/static/css/base.css b/mediagoblin/static/css/base.css index d0396ceb..bb2a2cbd 100644 --- a/mediagoblin/static/css/base.css +++ b/mediagoblin/static/css/base.css @@ -943,12 +943,16 @@ p.verifier { table.metadata_editor { margin: 10px auto; - width: 800px; + width: 1000px; +} + +table.metadata_editor tr th { + width:100px; } table.metadata_editor tr td { - width:350px; + width:300px; } table.metadata_editor tr td.form_field_input input { - width: 300px + width:300px; } diff --git a/mediagoblin/templates/mediagoblin/edit/metadata.html b/mediagoblin/templates/mediagoblin/edit/metadata.html index 364cad0d..d5d1fec5 100644 --- a/mediagoblin/templates/mediagoblin/edit/metadata.html +++ b/mediagoblin/templates/mediagoblin/edit/metadata.html @@ -78,20 +78,15 @@ visit http:/wwww.json-ld.org for more information. -->

{% trans %}Context{% endtrans %}

- - - - - {% for miniform in form.context -%} - {{ wtforms_util.render_table_row(miniform) }} - {% endfor -%} + {{ wtforms_util.render_fieldlist_as_table_rows(form.context) }}
- {% trans %}Identifier{% endtrans %}{% trans %}Value{% endtrans %}
+ +
@@ -99,26 +94,22 @@

{% trans %}Data{% endtrans %}

- - - - - {% for miniform in form.media_metadata -%} - {{ wtforms_util.render_table_row(miniform) }} - {% endfor -%} + {{ wtforms_util.render_fieldlist_as_table_rows(form.media_metadata) }}
- {% trans %}Identifier{% endtrans %}{% trans %}Value{% endtrans %}
+ + + diff --git a/mediagoblin/templates/mediagoblin/utils/wtforms.html b/mediagoblin/templates/mediagoblin/utils/wtforms.html index e861b674..c83d53f1 100644 --- a/mediagoblin/templates/mediagoblin/utils/wtforms.html +++ b/mediagoblin/templates/mediagoblin/utils/wtforms.html @@ -70,26 +70,14 @@ {# Auto-render a form as a table #} {% macro render_table(form) -%} {% for field in form %} - - - - + render_field_as_table_row(field) {% endfor %} {%- endmacro %} -{% macro render_table_row(form) %} +{% macro render_form_as_table_row(form) %} {%- for field in form %} + {%- endmacro %} +{% macro render_field_as_table_row(field) %} + + + + +{% endmacro %} + +{% macro render_fieldlist_as_table_rows(fieldlist) %} + {% for field in fieldlist -%} + {%- if field.type == 'FormField' %} + {{ render_form_as_table_row(field) }} + {%- else %} + {{ render_field_as_table_row(field) }} + {%- endif %} + {% endfor -%} +{% endmacro %} + {# Render a boolean field #} {% macro render_bool(field) %}
From 6b6b1b076bf0f2104dacdfa0f351555bcb6c98d6 Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Mon, 12 May 2014 13:07:11 -0400 Subject: [PATCH 60/78] Made some changes so that the metadata editing page works well with the updated metadata tools. --- mediagoblin/edit/forms.py | 3 --- mediagoblin/edit/views.py | 27 +++++-------------- .../templates/mediagoblin/edit/metadata.html | 23 ---------------- .../mediagoblin/utils/metadata_table.html | 11 +++----- 4 files changed, 10 insertions(+), 54 deletions(-) diff --git a/mediagoblin/edit/forms.py b/mediagoblin/edit/forms.py index 7ddf603e..c2355980 100644 --- a/mediagoblin/edit/forms.py +++ b/mediagoblin/edit/forms.py @@ -131,6 +131,3 @@ class EditMetaDataForm(wtforms.Form): media_metadata = wtforms.FieldList( wtforms.FormField(MetaDataForm, label="") ) - context = wtforms.FieldList( - wtforms.FormField(MetaDataForm, label="") - ) diff --git a/mediagoblin/edit/views.py b/mediagoblin/edit/views.py index 4ab3fe01..34021257 100644 --- a/mediagoblin/edit/views.py +++ b/mediagoblin/edit/views.py @@ -33,6 +33,7 @@ from mediagoblin.decorators import (require_active_login, active_user_from_url, get_user_collection, user_has_privilege, user_not_banned) from mediagoblin.tools.crypto import get_timed_signer_url +from mediagoblin.tools.metadata import compact_and_validate from mediagoblin.tools.mail import email_debug_message from mediagoblin.tools.response import (render_to_response, redirect, redirect_obj, render_404) @@ -441,32 +442,19 @@ def change_email(request): def edit_metadata(request, media): form = forms.EditMetaDataForm(request.form) if request.method == "POST" and form.validate(): - context = dict([(row['identifier'],row['value']) - for row in form.context.data]) metadata_dict = dict([(row['identifier'],row['value']) for row in form.media_metadata.data]) - # TODO VALIDATE THIS BEFORE WE ENTER IT - # validate(metadata_dict) - # validate(context) - json_ld_metadata = jsonld.compact(metadata_dict, context) - # media.media_metadata = json_ld_metadata - # media.save() + json_ld_metadata = compact_and_validate(metadata_dict) + media.media_metadata = json_ld_metadata + media.save() return redirect_obj(request, media) if media.media_metadata: - for row in media.media_metadata.iteritems(): - if row[0] == "@context": continue - identifier = row[0] - # TODO Will change when we revert the metadata branch - value = row[1]['@value'] + for identifier, value in media.media_metadata.iteritems(): + if identifier == "@context": continue form.media_metadata.append_entry({ 'identifier':identifier, 'value':value}) - for row in media.media_metadata['@context'].iteritems(): - identifier, value = row[0:2] - form.context.append_entry({ - 'identifier':identifier, - 'value':value}) else: form.media_metadata.append_entry({ 'identifier':"", @@ -474,9 +462,6 @@ def edit_metadata(request, media): form.media_metadata.append_entry({ 'identifier':"", 'value':""}) - form.context.append_entry({ - 'identifier':"", - 'value':""}) return render_to_response( request, 'mediagoblin/edit/metadata.html', diff --git a/mediagoblin/templates/mediagoblin/edit/metadata.html b/mediagoblin/templates/mediagoblin/edit/metadata.html index d5d1fec5..b5a52e5f 100644 --- a/mediagoblin/templates/mediagoblin/edit/metadata.html +++ b/mediagoblin/templates/mediagoblin/edit/metadata.html @@ -49,7 +49,6 @@ } $(document).ready(function(){ - var context_lines = {{ form.context | length }}; var metadata_lines = {{ form.media_metadata | length }}; $("#add_new_metadata_row").click(function(){ add_new_row("#metadata_list", @@ -57,14 +56,8 @@ 'media_metadata-'); metadata_lines += 1; }) - $("#add_new_context_row").click(function(){ - add_new_row("#context_list", - context_lines, - 'context-'); - context_lines += 1; }) $("#clear_empty_rows").click(function(){ - clear_empty_rows("#context_list"); clear_empty_rows("#metadata_list"); }) }) @@ -74,22 +67,6 @@

{% trans media_name=media.title -%} Metadata for "{{ media_name }}"{% endtrans %}

- -

{% trans %}Context{% endtrans %}

-
{{ field.label.text }} - {{field}} - {% if field.errors %} -
-
    - {% for error in field.errors %} -
  • {{error}}
    • - {% endfor %} -
- {% endif %} -
{{ render_label_p(field) }} {{field}} {%- if field.errors -%} @@ -105,6 +93,33 @@
{{ field.label.text }} + {{field}} + {% if field.errors %} +
+
    + {% for error in field.errors %} +
  • {{error}}
    • + {% endfor %} +
+ {% endif %} +
- {{ wtforms_util.render_fieldlist_as_table_rows(form.context) }} -
- - - - - - - - -

{% trans %}Data{% endtrans %}

diff --git a/mediagoblin/templates/mediagoblin/utils/metadata_table.html b/mediagoblin/templates/mediagoblin/utils/metadata_table.html index 0c67264a..b4c37dbd 100644 --- a/mediagoblin/templates/mediagoblin/utils/metadata_table.html +++ b/mediagoblin/templates/mediagoblin/utils/metadata_table.html @@ -20,16 +20,13 @@ {%- set metadata=media_entry.media_metadata %} {%- set metadata_context=metadata['@context'] %} {%- if metadata %} - - {%- for key, value_dict in metadata.iteritems() if not key=='@context' %} - {% if value_dict['@value'] -%} +
+ {%- for key, value in metadata.iteritems() if not key=='@context' %} - + - {%- endif -%} {%- endfor %}
{{ format_predicate(key) }} - {{ value_dict['@value'] }} + {{ value }}
{% endif %} From e8d64d453b5b712a82677759c8c9acab706a98d9 Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Mon, 12 May 2014 13:19:03 -0400 Subject: [PATCH 61/78] Cleaned up the 'batchaddmedia' command a bit --- mediagoblin/gmg_commands/batchaddmedia.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/mediagoblin/gmg_commands/batchaddmedia.py b/mediagoblin/gmg_commands/batchaddmedia.py index e540e88c..a9364daa 100644 --- a/mediagoblin/gmg_commands/batchaddmedia.py +++ b/mediagoblin/gmg_commands/batchaddmedia.py @@ -38,7 +38,7 @@ This command allows the administrator to upload many media files at once.""" subparser.add_argument( 'target_path', help=("""\ -Path to a local archive or directory containing a "location.csv" and a +Path to a local archive or directory containing a "location.csv" and a "metadata.csv" file. These are csv (comma seperated value) files with the locations and metadata of the files to be uploaded. The location must be listed with either the URL of the remote media file or the filesystem path of a local @@ -128,6 +128,12 @@ zip files and directories" files_attempted += 1 file_metadata = media_metadata[media_id] + + ### Remove all metadata entries starting with 'media' because we are ### + ### only using those for internal use. ### + file_metadata = dict([(key, value) + for key, value in file_metadata.iteritems() if + key.split(":")[0] != 'media']) try: json_ld_metadata = compact_and_validate(file_metadata) except ValidationError, exc: @@ -138,8 +144,10 @@ zip files and directories" original_location = media_locations[media_id]['media:original'] url = urlparse(original_location) - title = json_ld_metadata.get('dcterms:title') - description = json_ld_metadata.get('dcterms:description') + ### Pull the important media information for mediagoblin from the ### + ### metadata, if it is provided. ### + title = json_ld_metadata.get('dc:title') + description = json_ld_metadata.get('dc:description') license = json_ld_metadata.get('license') filename = url.path.split()[-1] From 7d52eb770501240e6872d3007d8e9ef203632eb0 Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Mon, 12 May 2014 13:59:28 -0400 Subject: [PATCH 62/78] Modified the batchaddmedia script so that it only looks for one csv file instead of the previous method which looked for two files. --- mediagoblin/gmg_commands/batchaddmedia.py | 123 ++++++++-------------- 1 file changed, 46 insertions(+), 77 deletions(-) diff --git a/mediagoblin/gmg_commands/batchaddmedia.py b/mediagoblin/gmg_commands/batchaddmedia.py index a9364daa..7ba8db1e 100644 --- a/mediagoblin/gmg_commands/batchaddmedia.py +++ b/mediagoblin/gmg_commands/batchaddmedia.py @@ -25,7 +25,7 @@ from mediagoblin.submit.lib import ( submit_media, get_upload_file_limits, FileUploadLimit, UserUploadLimit, UserPastUploadLimit) from mediagoblin.tools.metadata import compact_and_validate - +from mediagoblin.tools.translate import pass_to_ugettext as _ from jsonschema.exceptions import ValidationError @@ -36,17 +36,7 @@ This command allows the administrator to upload many media files at once.""" 'username', help="Name of user these media entries belong to") subparser.add_argument( - 'target_path', - help=("""\ -Path to a local archive or directory containing a "location.csv" and a -"metadata.csv" file. These are csv (comma seperated value) files with the -locations and metadata of the files to be uploaded. The location must be listed -with either the URL of the remote media file or the filesystem path of a local -file. The metadata should be provided with one column for each of the 15 Dublin -Core properties (http://dublincore.org/documents/dces/). Both "location.csv" and -"metadata.csv" must begin with a row demonstrating the order of the columns. We -have provided an example of these files at -""")) + 'metadata_path') subparser.add_argument( '--celery', action='store_true', @@ -65,48 +55,24 @@ def batchaddmedia(args): # get the user user = app.db.User.query.filter_by(username=args.username.lower()).first() if user is None: - print "Sorry, no user by username '%s' exists" % args.username + print _(u"Sorry, no user by username '{username}' exists".format( + username=args.username)) return upload_limit, max_file_size = get_upload_file_limits(user) temp_files = [] - if os.path.isdir(args.target_path): - dir_path = args.target_path - - elif tarfile.is_tarfile(args.target_path): - dir_path = tempfile.mkdtemp() - temp_files.append(dir_path) - tar = tarfile.open(args.target_path) - tar.extractall(path=dir_path) - - elif zipfile.is_zipfile(args.target_path): - dir_path = tempfile.mkdtemp() - temp_files.append(dir_path) - zipped_file = zipfile.ZipFile(args.target_path) - zipped_file.extractall(path=dir_path) + if os.path.isfile(args.metadata_path): + metadata_path = args.metadata_path else: - print "Couldn't recognize the file. This script only accepts tar files,\ -zip files and directories" - if dir_path.endswith('/'): - dir_path = dir_path[:-1] - - location_file_path = os.path.join(dir_path,"location.csv") - metadata_file_path = os.path.join(dir_path, "metadata.csv") - - # check for the location file, if it exists... - abs_location_filename = os.path.abspath(location_file_path) - if not os.path.exists(abs_location_filename): - print "Can't find a file with filename '%s'" % location_file_path - return - - # check for the metadata file, if it exists... - abs_metadata_filename = os.path.abspath(metadata_file_path) - if not os.path.exists(abs_metadata_filename): - print "Can't find a file with filename '%s'" % metadata_file_path + error = _(u'File at {path} not found, use -h flag for help'.format( + path=args.metadata_path)) + print error return + abs_metadata_filename = os.path.abspath(metadata_path) + abs_metadata_dir = os.path.dirname(abs_metadata_filename) upload_limit, max_file_size = get_upload_file_limits(user) def maybe_unicodeify(some_string): @@ -116,36 +82,36 @@ zip files and directories" else: return unicode(some_string) - with file(abs_location_filename, 'r') as all_locations: - contents = all_locations.read() - media_locations = parse_csv_file(contents) - with file(abs_metadata_filename, 'r') as all_metadata: contents = all_metadata.read() media_metadata = parse_csv_file(contents) - for media_id in media_locations.keys(): + for media_id, file_metadata in media_metadata.iteritems(): files_attempted += 1 + # In case the metadata was not uploaded initialize an empty dictionary. + json_ld_metadata = compact_and_validate({}) - file_metadata = media_metadata[media_id] - - ### Remove all metadata entries starting with 'media' because we are ### - ### only using those for internal use. ### + # Get all metadata entries starting with 'media' as variables and then + # delete them because those are for internal use only. + original_location = file_metadata['media:location'] file_metadata = dict([(key, value) for key, value in file_metadata.iteritems() if key.split(":")[0] != 'media']) try: json_ld_metadata = compact_and_validate(file_metadata) except ValidationError, exc: - print "Error with '%s' value '%s': %s" % ( - media_id, exc.path[0], exc.message) + error = _(u"""Error with media '{media_id}' value '{error_path}': {error_msg} +Metadata was not uploaded.""".format( + media_id=media_id, + error_path=exc.path[0], + error_msg=exc.message)) + print error continue - original_location = media_locations[media_id]['media:original'] url = urlparse(original_location) - ### Pull the important media information for mediagoblin from the ### - ### metadata, if it is provided. ### + ### Pull the important media information for mediagoblin from the + ### metadata, if it is provided. title = json_ld_metadata.get('dc:title') description = json_ld_metadata.get('dc:description') @@ -161,14 +127,14 @@ zip files and directories" if os.path.isabs(path): file_abs_path = os.path.abspath(path) else: - file_path = os.path.join(dir_path, path) + file_path = os.path.join(abs_metadata_dir, path) file_abs_path = os.path.abspath(file_path) try: media_file = file(file_abs_path, 'r') except IOError: - print "\ -FAIL: Local file {filename} could not be accessed.".format(filename=filename) - print "Skipping it." + print _(u"""\ +FAIL: Local file {filename} could not be accessed. +{filename} will not be uploaded.""".format(filename=filename)) continue try: submit_media( @@ -182,29 +148,36 @@ FAIL: Local file {filename} could not be accessed.".format(filename=filename) metadata=json_ld_metadata, tags_string=u"", upload_limit=upload_limit, max_file_size=max_file_size) - print "Successfully uploading {filename}!".format(filename=filename) - print "" + print _(u"""Successfully submitted {filename}! +Be sure to look at the Media Processing Panel on your website to be sure it +uploaded successfully.""".format(filename=filename)) files_uploaded += 1 except FileUploadLimit: - print "FAIL: This file is larger than the upload limits for this site." + print _( +u"FAIL: This file is larger than the upload limits for this site.") except UserUploadLimit: - print "FAIL: This file will put this user past their upload limits." + print _( +"FAIL: This file will put this user past their upload limits.") except UserPastUploadLimit: - print "FAIL: This user is already past their upload limits." - print "\ -{files_uploaded} out of {files_attempted} files successfully uploaded".format( + print _("FAIL: This user is already past their upload limits.") + print _( +"{files_uploaded} out of {files_attempted} files successfully submitted".format( files_uploaded=files_uploaded, - files_attempted=files_attempted) - teardown(temp_files) + files_attempted=files_attempted)) def parse_csv_file(file_contents): + """ + The helper function which converts the csv file into a dictionary where each + item's key is the provided value 'media:id' and each item's value is another + dictionary. + """ list_of_contents = file_contents.split('\n') key, lines = (list_of_contents[0].split(','), list_of_contents[1:]) objects_dict = {} - # Build a dictionaryfrom mediagoblin.tools.translate import lazy_pass_to_ugettext as _ + # Build a dictionary for line in lines: if line.isspace() or line == '': continue values = csv_reader([line]).next() @@ -215,7 +188,3 @@ def parse_csv_file(file_contents): return objects_dict - -def teardown(temp_files): - for temp_file in temp_files: - subprocess.call(['rm','-r',temp_file]) From 9ceea31c5b99201e8a222234980491b634b814c5 Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Mon, 12 May 2014 14:27:26 -0400 Subject: [PATCH 63/78] Made the metadata table look pretty --- mediagoblin/static/css/base.css | 18 ++++++++++++++++++ .../mediagoblin/utils/metadata_table.html | 10 ++++++---- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/mediagoblin/static/css/base.css b/mediagoblin/static/css/base.css index bb2a2cbd..df0e850b 100644 --- a/mediagoblin/static/css/base.css +++ b/mediagoblin/static/css/base.css @@ -610,6 +610,24 @@ a img.media_image { cursor: zoom-in; } +table.metadata_info { + font-size:85%; + margin-left:10px; +} + +table.metadata_info tr.highlight { + color:#f7f7f7; +} + +table.metadata_info th { + font-weight: bold; + border-spacing: 10px; + text-align: left; +} +table.metadata_info td { + padding: 4px 8px; +} + /* icons */ img.media_icon { diff --git a/mediagoblin/templates/mediagoblin/utils/metadata_table.html b/mediagoblin/templates/mediagoblin/utils/metadata_table.html index b4c37dbd..edbee2c2 100644 --- a/mediagoblin/templates/mediagoblin/utils/metadata_table.html +++ b/mediagoblin/templates/mediagoblin/utils/metadata_table.html @@ -20,10 +20,12 @@ {%- set metadata=media_entry.media_metadata %} {%- set metadata_context=metadata['@context'] %} {%- if metadata %} - - {%- for key, value in metadata.iteritems() if not key=='@context' %} - - +

{% trans %}Metadata Information{% endtrans %}

+
{{ format_predicate(key) }}
+ {%- for key, value in metadata.iteritems() if ( + not key=='@context' and value) %} + + From 8f65028ddff231bfb374add3dc1328af4f26ca9d Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Mon, 12 May 2014 14:48:46 -0400 Subject: [PATCH 64/78] Corrected the rdfa in the metadata table. --- mediagoblin/templates/mediagoblin/utils/metadata_table.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mediagoblin/templates/mediagoblin/utils/metadata_table.html b/mediagoblin/templates/mediagoblin/utils/metadata_table.html index edbee2c2..14e18bf3 100644 --- a/mediagoblin/templates/mediagoblin/utils/metadata_table.html +++ b/mediagoblin/templates/mediagoblin/utils/metadata_table.html @@ -21,12 +21,12 @@ {%- set metadata_context=metadata['@context'] %} {%- if metadata %}

{% trans %}Metadata Information{% endtrans %}

-
{{ format_predicate(key) }} {{ value }}
+
{%- for key, value in metadata.iteritems() if ( not key=='@context' and value) %} - {%- endfor %} From acfcaf6366bd4695c1c37c7aa8ff5a176b412e2a Mon Sep 17 00:00:00 2001 From: Christopher Allan Webber Date: Mon, 12 May 2014 14:50:58 -0500 Subject: [PATCH 65/78] Move the metadata display table over to being contained in a plugin --- .../metadata_display}/metadata_table.html | 35 +++++++++---------- .../mediagoblin/user_pages/media.html | 5 --- mediagoblin/tools/metadata.py | 5 +++ mediagoblin/user_pages/lib.py | 3 -- mediagoblin/user_pages/views.py | 5 ++- 5 files changed, 23 insertions(+), 30 deletions(-) rename mediagoblin/{templates/mediagoblin/utils => plugins/metadata_display/templates/mediagoblin/plugins/metadata_display}/metadata_table.html (51%) diff --git a/mediagoblin/templates/mediagoblin/utils/metadata_table.html b/mediagoblin/plugins/metadata_display/templates/mediagoblin/plugins/metadata_display/metadata_table.html similarity index 51% rename from mediagoblin/templates/mediagoblin/utils/metadata_table.html rename to mediagoblin/plugins/metadata_display/templates/mediagoblin/plugins/metadata_display/metadata_table.html index 3281c81c..db12f149 100644 --- a/mediagoblin/templates/mediagoblin/utils/metadata_table.html +++ b/mediagoblin/plugins/metadata_display/templates/mediagoblin/plugins/metadata_display/metadata_table.html @@ -16,22 +16,19 @@ # along with this program. If not, see . #} -{%- macro render_table(request, media_entry, format_predicate) %} - {%- set metadata=media_entry.media_metadata %} - {%- set metadata_context=metadata['@context'] %} - {%- if metadata %} - {#- NOTE: In some smart future where the context is more extensible, - we will need to add to the prefix here-#} -
{{ format_predicate(key) }} + {{ value }}
- {%- for key, value_dict in metadata.iteritems() if not key=='@context' %} - {% if value_dict['@value'] -%} - - - - - {%- endif -%} - {%- endfor %} -
{{ format_predicate(key) }} - {{ value_dict['@value'] }}
- {% endif %} -{%- endmacro %} +{%- set metadata=media.media_metadata %} +{%- set metadata_context=metadata['@context'] %} +{%- if metadata %} + {#- NOTE: In some smart future where the context is more extensible, + we will need to add to the prefix here-#} + + {%- for key, value in metadata.iteritems() if not key=='@context' %} + {% if value -%} + + + + + {%- endif -%} + {%- endfor %} +
{{ rdfa_to_readable(key) }}{{ value }}
+{% endif %} diff --git a/mediagoblin/templates/mediagoblin/user_pages/media.html b/mediagoblin/templates/mediagoblin/user_pages/media.html index 22971fec..949cbcde 100644 --- a/mediagoblin/templates/mediagoblin/user_pages/media.html +++ b/mediagoblin/templates/mediagoblin/user_pages/media.html @@ -18,7 +18,6 @@ {%- extends "mediagoblin/base.html" %} {% import "/mediagoblin/utils/wtforms.html" as wtforms_util %} -{% import "/mediagoblin/utils/metadata_table.html" as metadata_util %} {% from "mediagoblin/utils/pagination.html" import render_pagination %} {% block title %}{{ media.title }} — {{ super() }}{% endblock %} @@ -231,10 +230,6 @@ {% template_hook("media_sideinfo") %} - {% block mediagoblin_sidebar %} - {{ metadata_util.render_table(request, media, rdfa_to_readable) }} - {% endblock %} -
diff --git a/mediagoblin/tools/metadata.py b/mediagoblin/tools/metadata.py index 7de5a514..3f10e9d1 100644 --- a/mediagoblin/tools/metadata.py +++ b/mediagoblin/tools/metadata.py @@ -211,3 +211,8 @@ def expand_json(metadata, context=DEFAULT_CONTEXT): if context is not None: options["expandContext"] = context return jsonld.expand(metadata, options=options) + + +def rdfa_to_readable(rdfa_predicate): + readable = rdfa_predicate.split(u":")[1].capitalize() + return readable diff --git a/mediagoblin/user_pages/lib.py b/mediagoblin/user_pages/lib.py index 83a99cee..e5c8defc 100644 --- a/mediagoblin/user_pages/lib.py +++ b/mediagoblin/user_pages/lib.py @@ -116,6 +116,3 @@ def build_report_object(report_form, media_entry=None, comment=None): report_object.reporter_id = report_form.reporter_id.data return report_object -def rdfa_to_readable(rdfa_predicate): - readable = rdfa_predicate.split(u":")[1].capitalize() - return readable diff --git a/mediagoblin/user_pages/views.py b/mediagoblin/user_pages/views.py index f42eae1f..78751a28 100644 --- a/mediagoblin/user_pages/views.py +++ b/mediagoblin/user_pages/views.py @@ -28,7 +28,7 @@ from mediagoblin.tools.translate import pass_to_ugettext as _ from mediagoblin.tools.pagination import Pagination from mediagoblin.user_pages import forms as user_forms from mediagoblin.user_pages.lib import (send_comment_email, - add_media_to_collection, build_report_object, rdfa_to_readable) + add_media_to_collection, build_report_object) from mediagoblin.notifications import trigger_notification, \ add_comment_subscription, mark_comment_notification_seen from mediagoblin.tools.pluginapi import hook_transform @@ -152,8 +152,7 @@ def media_home(request, media, page, **kwargs): 'comments': comments, 'pagination': pagination, 'comment_form': comment_form, - 'app_config': mg_globals.app_config, - 'rdfa_to_readable':rdfa_to_readable} + 'app_config': mg_globals.app_config} # Since the media template name gets swapped out for each media # type, normal context hooks don't work if you want to affect all From 8524a6bdc575312b35df6e6aa3118ed139303d72 Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Mon, 12 May 2014 17:02:12 -0400 Subject: [PATCH 66/78] Added documentation for the batchaddmedia gmg tool to the mediagoblin docs. --- docs/source/siteadmin/commandline-upload.rst | 51 ++++++++++++++++++++ mediagoblin/gmg_commands/batchaddmedia.py | 12 +++-- 2 files changed, 60 insertions(+), 3 deletions(-) diff --git a/docs/source/siteadmin/commandline-upload.rst b/docs/source/siteadmin/commandline-upload.rst index be19df58..d67c19dd 100644 --- a/docs/source/siteadmin/commandline-upload.rst +++ b/docs/source/siteadmin/commandline-upload.rst @@ -39,3 +39,54 @@ You can also pass in the `--celery` option if you would prefer that your media be passed over to celery to be processed rather than be processed immediately. +============================ +Command-line batch uploading +============================ + +There's another way to submit media, and it can be much more powerful, although +it is a bit more complex. + + ./bin/gmg batchaddmedia admin /path/to/your/metadata.csv + +This is an example of what a script may look like. The important part here is +that you have to create the 'metadata.csv' file.:: + + media:location,dcterms:title,dcterms:creator,dcterms:type + "http://www.example.net/path/to/nap.png","Goblin taking a nap",,"Image" + "http://www.example.net/path/to/snore.ogg","Goblin Snoring","Me","Audio" + +The above is an example of a very simple metadata.csv file. The batchaddmedia +script would read this and attempt to upload only two pieces of media, and would +be able to automatically name them appropriately. + +The csv file +============ +The media:location column +------------------------- +The media:location column is the one column that is absolutely necessary for +uploading your media. This gives a path to each piece of media you upload. This +can either a path to a local file or a direct link to remote media (with the +link in http format). As you can see in the example above the (fake) media was +stored remotely on "www.example.net". + +Other columns +------------- +Other columns can be used to provide detailed metadata about each media entry. +Our metadata system accepts any information provided for in the +`RDFa Core Initial Context`_, and the batchupload script recognizes all of the +resources provided within it. + +.. _RDFa Core Initial Context: http://www.w3.org/2011/rdfa-context/rdfa-1.1 + +The uploader may include the metadata for each piece of media, or +leave them blank if they want to. A few columns from `Dublin Core`_ are +notable because the batchaddmedia script uses them to set the default +information of uploaded media entries. + +.. _Dublin Core: http://wiki.dublincore.org/index.php/User_Guide + +- **dc:title** sets a title for your media entry. If this is left blank, the media entry will be named according to the filename of the file being uploaded. +- **dc:description** sets a description of your media entry. If this is left blank the media entry's description will not be filled in. +- **dc:rights** will set a license for your media entry `if` the data provided is a valid URI. If this is left blank 'All Rights Reserved' will be selected. + +You can of course, change these values later. diff --git a/mediagoblin/gmg_commands/batchaddmedia.py b/mediagoblin/gmg_commands/batchaddmedia.py index 7ba8db1e..75e7b7c5 100644 --- a/mediagoblin/gmg_commands/batchaddmedia.py +++ b/mediagoblin/gmg_commands/batchaddmedia.py @@ -32,15 +32,21 @@ from jsonschema.exceptions import ValidationError def parser_setup(subparser): subparser.description = """\ This command allows the administrator to upload many media files at once.""" + subparser.epilog = _(u"""For more information about how to properly run this +script (and how to format the metadata csv file), read the MediaGoblin +documentation page on command line uploading +""") subparser.add_argument( 'username', - help="Name of user these media entries belong to") + help=_(u"Name of user these media entries belong to")) subparser.add_argument( - 'metadata_path') + 'metadata_path', + help=_( +u"""Path to the csv file containing metadata information.""")) subparser.add_argument( '--celery', action='store_true', - help="Don't process eagerly, pass off to celery") + help=_(u"Don't process eagerly, pass off to celery")) def batchaddmedia(args): From b5dd2459893fd1e7ae1376a7573a36bf15c983f7 Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Mon, 12 May 2014 18:07:31 -0400 Subject: [PATCH 67/78] Made the metadata table functional within the new metadata_display plugin and removed all traces of it from core. --- .../plugins/metadata_display/__init__.py | 41 +++++++++++++ mediagoblin/plugins/metadata_display/lib.py | 31 ++++++++++ .../static/css/metadata_display.css | 14 +++++ .../bits/metadata_extra_head.html | 3 + .../metadata_display/metadata_table.html | 12 ++-- .../metadata_display/metadata_table.html.orig | 60 +++++++++++++++++++ mediagoblin/static/css/base.css | 19 ------ 7 files changed, 156 insertions(+), 24 deletions(-) create mode 100644 mediagoblin/plugins/metadata_display/__init__.py create mode 100644 mediagoblin/plugins/metadata_display/lib.py create mode 100644 mediagoblin/plugins/metadata_display/static/css/metadata_display.css create mode 100644 mediagoblin/plugins/metadata_display/templates/mediagoblin/plugins/metadata_display/bits/metadata_extra_head.html create mode 100644 mediagoblin/plugins/metadata_display/templates/mediagoblin/plugins/metadata_display/metadata_table.html.orig diff --git a/mediagoblin/plugins/metadata_display/__init__.py b/mediagoblin/plugins/metadata_display/__init__.py new file mode 100644 index 00000000..4a4c898f --- /dev/null +++ b/mediagoblin/plugins/metadata_display/__init__.py @@ -0,0 +1,41 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +import os +from pkg_resources import resource_filename + +from mediagoblin.plugins.metadata_display.lib import add_rdfa_to_readable_to_media_home +from mediagoblin.tools import pluginapi +from mediagoblin.tools.staticdirect import PluginStatic + +PLUGIN_DIR = os.path.dirname(__file__) + +def setup_plugin(): + # Register the template path. + pluginapi.register_template_path(os.path.join(PLUGIN_DIR, 'templates')) + + pluginapi.register_template_hooks( + {"media_sideinfo": "mediagoblin/plugins/metadata_display/metadata_table.html", + "head": "mediagoblin/plugins/metadata_display/bits/metadata_extra_head.html"}) + + +hooks = { + 'setup': setup_plugin, + 'static_setup': lambda: PluginStatic( + 'metadata_display', + resource_filename('mediagoblin.plugins.metadata_display', 'static') + ), + 'media_home_context':add_rdfa_to_readable_to_media_home + } diff --git a/mediagoblin/plugins/metadata_display/lib.py b/mediagoblin/plugins/metadata_display/lib.py new file mode 100644 index 00000000..0985208c --- /dev/null +++ b/mediagoblin/plugins/metadata_display/lib.py @@ -0,0 +1,31 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +def rdfa_to_readable(rdfa_predicate): + """ + A simple script to convert rdfa resource descriptors into a form more + accessible for humans. + """ + readable = rdfa_predicate.split(u":")[1].capitalize() + return readable + +def add_rdfa_to_readable_to_media_home(context): + """ + A context hook which adds the 'rdfa_to_readable' filter to + the media home page. + """ + context['rdfa_to_readable'] = rdfa_to_readable + return context diff --git a/mediagoblin/plugins/metadata_display/static/css/metadata_display.css b/mediagoblin/plugins/metadata_display/static/css/metadata_display.css new file mode 100644 index 00000000..e4612b02 --- /dev/null +++ b/mediagoblin/plugins/metadata_display/static/css/metadata_display.css @@ -0,0 +1,14 @@ +table.metadata_info { + font-size:85%; + margin-left:10px; +} + +table.metadata_info th { + font-weight: bold; + border-spacing: 10px; + text-align: left; +} +table.metadata_info td { + padding: 4px 8px; +} + diff --git a/mediagoblin/plugins/metadata_display/templates/mediagoblin/plugins/metadata_display/bits/metadata_extra_head.html b/mediagoblin/plugins/metadata_display/templates/mediagoblin/plugins/metadata_display/bits/metadata_extra_head.html new file mode 100644 index 00000000..84eedf18 --- /dev/null +++ b/mediagoblin/plugins/metadata_display/templates/mediagoblin/plugins/metadata_display/bits/metadata_extra_head.html @@ -0,0 +1,3 @@ + diff --git a/mediagoblin/plugins/metadata_display/templates/mediagoblin/plugins/metadata_display/metadata_table.html b/mediagoblin/plugins/metadata_display/templates/mediagoblin/plugins/metadata_display/metadata_table.html index 73b5ec52..3a9d872c 100644 --- a/mediagoblin/plugins/metadata_display/templates/mediagoblin/plugins/metadata_display/metadata_table.html +++ b/mediagoblin/plugins/metadata_display/templates/mediagoblin/plugins/metadata_display/metadata_table.html @@ -15,17 +15,18 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . #} - +{% block metadata_information_table %} {%- set metadata=media.media_metadata %} {%- set metadata_context=metadata['@context'] %} {%- if metadata %} +

{% trans %}Metadata{% endtrans %}

{#- NOTE: In some smart future where the context is more extensible, we will need to add to the prefix here-#} - +
{%- for key, value in metadata.iteritems() if not key=='@context' %} {% if value -%} - + {%- endif -%} @@ -34,7 +35,8 @@ {% endif %} {% if request.user and request.user.has_privilege('admin') %} + user=media.get_uploader.username, + media_id=media.id) }}"> {% trans %}Edit Metadata{% endtrans %} {% endif %} +{% endblock %} diff --git a/mediagoblin/plugins/metadata_display/templates/mediagoblin/plugins/metadata_display/metadata_table.html.orig b/mediagoblin/plugins/metadata_display/templates/mediagoblin/plugins/metadata_display/metadata_table.html.orig new file mode 100644 index 00000000..2bd1a14c --- /dev/null +++ b/mediagoblin/plugins/metadata_display/templates/mediagoblin/plugins/metadata_display/metadata_table.html.orig @@ -0,0 +1,60 @@ +{# +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +#} + +<<<<<<< HEAD:mediagoblin/templates/mediagoblin/utils/metadata_table.html +{%- macro render_table(request, media_entry, format_predicate) %} + {%- set metadata=media_entry.media_metadata %} + {%- set metadata_context=metadata['@context'] %} + {%- if metadata %} +

{% trans %}Metadata Information{% endtrans %}

+
{{ rdfa_to_readable(key) }}{{ rdfa_to_readable(key) }} {{ value }}
+ {%- for key, value in metadata.iteritems() if ( + not key=='@context' and value) %} + + + + + {%- endfor %} +
{{ format_predicate(key) }} + {{ value }}
+ {% endif %} + {% if request.user and request.user.has_privilege('admin') %} + + {% trans %}Edit Metadata{% endtrans %} + {% endif %} +{%- endmacro %} +======= +{%- set metadata=media.media_metadata %} +{%- set metadata_context=metadata['@context'] %} +{%- if metadata %} + {#- NOTE: In some smart future where the context is more extensible, + we will need to add to the prefix here-#} + + {%- for key, value in metadata.iteritems() if not key=='@context' %} + {% if value -%} + + + + + {%- endif -%} + {%- endfor %} +
{{ rdfa_to_readable(key) }}{{ value }}
+{% endif %} +>>>>>>> acfcaf6366bd4695c1c37c7aa8ff5a176b412e2a:mediagoblin/plugins/metadata_display/templates/mediagoblin/plugins/metadata_display/metadata_table.html diff --git a/mediagoblin/static/css/base.css b/mediagoblin/static/css/base.css index df0e850b..a3b564ea 100644 --- a/mediagoblin/static/css/base.css +++ b/mediagoblin/static/css/base.css @@ -609,25 +609,6 @@ a img.media_image { cursor: -moz-zoom-in; cursor: zoom-in; } - -table.metadata_info { - font-size:85%; - margin-left:10px; -} - -table.metadata_info tr.highlight { - color:#f7f7f7; -} - -table.metadata_info th { - font-weight: bold; - border-spacing: 10px; - text-align: left; -} -table.metadata_info td { - padding: 4px 8px; -} - /* icons */ img.media_icon { From c0ea2bad04c7c0ce28659a73bd63ca409c847519 Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Tue, 13 May 2014 16:53:28 -0400 Subject: [PATCH 68/78] Prepared for input without an 'id' column and made all of the internal nodes into free floating nodes so that compact_and_validate will remove them. --- mediagoblin/gmg_commands/batchaddmedia.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/mediagoblin/gmg_commands/batchaddmedia.py b/mediagoblin/gmg_commands/batchaddmedia.py index 75e7b7c5..58ca7e74 100644 --- a/mediagoblin/gmg_commands/batchaddmedia.py +++ b/mediagoblin/gmg_commands/batchaddmedia.py @@ -99,10 +99,7 @@ def batchaddmedia(args): # Get all metadata entries starting with 'media' as variables and then # delete them because those are for internal use only. - original_location = file_metadata['media:location'] - file_metadata = dict([(key, value) - for key, value in file_metadata.iteritems() if - key.split(":")[0] != 'media']) + original_location = file_metadata['location'] try: json_ld_metadata = compact_and_validate(file_metadata) except ValidationError, exc: @@ -175,7 +172,7 @@ u"FAIL: This file is larger than the upload limits for this site.") def parse_csv_file(file_contents): """ The helper function which converts the csv file into a dictionary where each - item's key is the provided value 'media:id' and each item's value is another + item's key is the provided value 'id' and each item's value is another dictionary. """ list_of_contents = file_contents.split('\n') @@ -184,12 +181,12 @@ def parse_csv_file(file_contents): objects_dict = {} # Build a dictionary - for line in lines: + for index, line in enumerate(lines): if line.isspace() or line == '': continue values = csv_reader([line]).next() line_dict = dict([(key[i], val) for i, val in enumerate(values)]) - media_id = line_dict['media:id'] + media_id = line_dict.get('id') or index objects_dict[media_id] = (line_dict) return objects_dict From 80fefb851485a73d1ff9d526bdcc6ebc6052af55 Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Tue, 13 May 2014 16:59:02 -0400 Subject: [PATCH 69/78] Removed unused imports. --- mediagoblin/gmg_commands/batchaddmedia.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mediagoblin/gmg_commands/batchaddmedia.py b/mediagoblin/gmg_commands/batchaddmedia.py index 58ca7e74..d83774ee 100644 --- a/mediagoblin/gmg_commands/batchaddmedia.py +++ b/mediagoblin/gmg_commands/batchaddmedia.py @@ -15,10 +15,9 @@ # along with this program. If not, see . import os -import tempfile, tarfile, zipfile, subprocess, requests +import requests from csv import reader as csv_reader from urlparse import urlparse -from pyld import jsonld from mediagoblin.gmg_commands import util as commands_util from mediagoblin.submit.lib import ( From e78472abacaaed660b1fd534394243372f409202 Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Tue, 13 May 2014 17:44:10 -0400 Subject: [PATCH 70/78] Added 'dc:created' to the list of metadata columns being validated by jsonschema. --- mediagoblin/tools/metadata.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mediagoblin/tools/metadata.py b/mediagoblin/tools/metadata.py index b0cad9da..bfefcac9 100644 --- a/mediagoblin/tools/metadata.py +++ b/mediagoblin/tools/metadata.py @@ -82,6 +82,10 @@ DEFAULT_SCHEMA = { "dcterms:created": { "format": "date-time", "type": "string", + }, + "dc:created": { + "format": "date-time", + "type": "string", } }, } From 375db9c9bc85be7cba00413ee3a72ffd7c5bbff3 Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Tue, 13 May 2014 17:45:29 -0400 Subject: [PATCH 71/78] Wrote a test for the compact_and_validate metadata function. --- mediagoblin/tests/test_metadata.py | 103 +++++++++++++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100644 mediagoblin/tests/test_metadata.py diff --git a/mediagoblin/tests/test_metadata.py b/mediagoblin/tests/test_metadata.py new file mode 100644 index 00000000..71346bb4 --- /dev/null +++ b/mediagoblin/tests/test_metadata.py @@ -0,0 +1,103 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import pytest +from mediagoblin.tools import template +from mediagoblin.tools.metadata import compact_and_validate +from webtest import AppError +from jsonschema import ValidationError + +from .resources import GOOD_JPG + +class TestMetadataFunctionality: + + @pytest.fixture(autouse=True) + def _setup(self, test_app): + self.test_app = test_app + + def login(self, username): + self.test_app.post( + '/auth/login/', { + 'username': username, + 'password': 'toast'}) + + def logout(self): + self.test_app.get('/auth/logout/') + + def do_post(self, data, *context_keys, **kwargs): + url = kwargs.pop('url', '/submit/') + do_follow = kwargs.pop('do_follow', False) + template.clear_test_template_context() + response = self.test_app.post(url, data, **kwargs) + if do_follow: + response.follow() + context_data = template.TEMPLATE_TEST_CONTEXT + for key in context_keys: + context_data = context_data[key] + return response, context_data + + def testCompactAndValidate(self): + # First, test out a well formatted piece of metadata + ###################################################### + test_metadata = { + 'dc:title':'My Pet Bunny', + 'dc:description':'A picture displaying how cute my pet bunny is.', + 'location':'/home/goblin/Pictures/bunny.png', + 'license':'http://www.gnu.org/licenses/gpl.txt' + } + jsonld_metadata =compact_and_validate(test_metadata) + assert jsonld_metadata + assert jsonld_metadata.get('dc:title') == 'My Pet Bunny' + # Free floating nodes should be removed + assert jsonld_metadata.get('location') is None + assert jsonld_metadata.get('@context') == \ + u"http://www.w3.org/2013/json-ld-context/rdfa11" + + # Next, make sure that various badly formatted metadata + # will be rejected. + ####################################################### + #,.,.,.,.,.,.,.,.,.,.,.,.,.,.,.,.,.,.,.,.,.,.,. + # Metadata with a non-URI license should fail : + #`'`'`'`'`'`'`'`'`'`'`'`'`'`'`'`'`'`'`'`'`'`'`' + metadata_fail_1 = { + 'dc:title':'My Pet Bunny', + 'dc:description':'A picture displaying how cute my pet bunny is.', + 'location':'/home/goblin/Pictures/bunny.png', + 'license':'All Rights Reserved.' + } + jsonld_fail_1 = None + try: + jsonld_fail_1 = compact_and_validate(metadata_fail_1) + except ValidationError, e: + assert e.message == "'All Rights Reserved.' is not a 'uri'" + assert jsonld_fail_1 == None + #,.,.,.,.,.,.,.,.,.,.,.,.,.,.,.,.,.,.,.,.,.,.,.,.,.,.,.,.,., + # Metadata with an ivalid date-time dc:created should fail : + #`'`'`'`'`'`'`'`'`'`'`'`'`'`'`'`'`'`'`'`'`'`'`'`'`'`'`'`'`'' + metadata_fail_2 = { + 'dc:title':'My Pet Bunny', + 'dc:description':'A picture displaying how cute my pet bunny is.', + 'location':'/home/goblin/Pictures/bunny.png', + 'license':'http://www.gnu.org/licenses/gpl.txt', + 'dc:created':'The other day' + } + jsonld_fail_2 = None + try: + jsonld_fail_2 = compact_and_validate(metadata_fail_2) + except ValidationError, e: + assert e.message == "'The other day' is not a 'date-time'" + assert jsonld_fail_2 == None + From 65f5714118f5b59bc7f51c67ffc6ef23f2c603cc Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Tue, 13 May 2014 18:15:28 -0400 Subject: [PATCH 72/78] Adjusted batchaddmedia to make use of more internal nodes. Added to the docs. --- docs/source/siteadmin/commandline-upload.rst | 36 ++++++++++++++------ mediagoblin/gmg_commands/batchaddmedia.py | 17 ++++----- 2 files changed, 35 insertions(+), 18 deletions(-) diff --git a/docs/source/siteadmin/commandline-upload.rst b/docs/source/siteadmin/commandline-upload.rst index 742c0cb2..5ec0bb12 100644 --- a/docs/source/siteadmin/commandline-upload.rst +++ b/docs/source/siteadmin/commandline-upload.rst @@ -61,16 +61,28 @@ be able to automatically name them appropriately. The csv file ============ -The media:location column -------------------------- -The media:location column is the one column that is absolutely necessary for +The location column +------------------- +The location column is the one column that is absolutely necessary for uploading your media. This gives a path to each piece of media you upload. This can either a path to a local file or a direct link to remote media (with the link in http format). As you can see in the example above the (fake) media was stored remotely on "www.example.net". -Other columns -------------- +Other internal nodes +-------------------- +There are other columns which can be used by the script to provide information. +These are not stored as part of the media's metadata. You can use these columns to +provide default information for your media entry, but as you'll see below, it's +just as easy to provide this information through the correct metadata columns. + +- **id** is used to identify the media entry to the user in case of an error in the batchaddmedia script. +- **license** is used to set a license for your piece a media for mediagoblin's use. This must be a URI. +- **title** will set the title displayed to mediagoblin users. +- **description** will set a description of your media. + +Metadata columns +---------------- Other columns can be used to provide detailed metadata about each media entry. Our metadata system accepts any information provided for in the `RDFa Core Initial Context`_, and the batchupload script recognizes all of the @@ -80,13 +92,17 @@ resources provided within it. The uploader may include the metadata for each piece of media, or leave them blank if they want to. A few columns from `Dublin Core`_ are -notable because the batchaddmedia script uses them to set the default +notable because the batchaddmedia script also uses them to set the default information of uploaded media entries. .. _Dublin Core: http://wiki.dublincore.org/index.php/User_Guide -- **dc:title** sets a title for your media entry. If this is left blank, the media entry will be named according to the filename of the file being uploaded. -- **dc:description** sets a description of your media entry. If this is left blank the media entry's description will not be filled in. -- **dc:rights** will set a license for your media entry `if` the data provided is a valid URI. If this is left blank 'All Rights Reserved' will be selected. +- **dc:title** sets a title for your media entry. +- **dc:description** sets a description of your media entry. -You can of course, change these values later. +If both a metadata column and an internal node for the title are provided, mediagoblin +will use the internal node as the media entry's display name. This makes it so +that if you want to display a piece of media with a different title +than the one provided in its metadata, you can just provide different data for +the 'dc:title' and 'title' columns. The same is true of the 'description' and +'dc:description'. diff --git a/mediagoblin/gmg_commands/batchaddmedia.py b/mediagoblin/gmg_commands/batchaddmedia.py index d83774ee..b7f2569c 100644 --- a/mediagoblin/gmg_commands/batchaddmedia.py +++ b/mediagoblin/gmg_commands/batchaddmedia.py @@ -33,7 +33,7 @@ def parser_setup(subparser): This command allows the administrator to upload many media files at once.""" subparser.epilog = _(u"""For more information about how to properly run this script (and how to format the metadata csv file), read the MediaGoblin -documentation page on command line uploading +documentation page on command line uploading """) subparser.add_argument( 'username', @@ -99,6 +99,14 @@ def batchaddmedia(args): # Get all metadata entries starting with 'media' as variables and then # delete them because those are for internal use only. original_location = file_metadata['location'] + + ### Pull the important media information for mediagoblin from the + ### metadata, if it is provided. + title = file_metadata.get('title') or file_metadata.get('dc:title') + description = (file_metadata.get('description') or + file_metadata.get('dc:description')) + + license = file_metadata.get('license') try: json_ld_metadata = compact_and_validate(file_metadata) except ValidationError, exc: @@ -111,13 +119,6 @@ Metadata was not uploaded.""".format( continue url = urlparse(original_location) - - ### Pull the important media information for mediagoblin from the - ### metadata, if it is provided. - title = json_ld_metadata.get('dc:title') - description = json_ld_metadata.get('dc:description') - - license = json_ld_metadata.get('license') filename = url.path.split()[-1] if url.scheme == 'http': From 0d6550fb05c25e230706c719e3a476d1b1e670b9 Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Wed, 14 May 2014 11:51:13 -0400 Subject: [PATCH 73/78] Tweaked the metadata edit screen to run jsonschema validators against the data. --- mediagoblin/edit/forms.py | 32 +++++++++++++++-- mediagoblin/edit/views.py | 15 ++++---- mediagoblin/static/css/base.css | 11 ++---- .../templates/mediagoblin/edit/metadata.html | 5 +-- .../templates/mediagoblin/utils/wtforms.html | 34 +++++++++---------- 5 files changed, 57 insertions(+), 40 deletions(-) diff --git a/mediagoblin/edit/forms.py b/mediagoblin/edit/forms.py index c2355980..7c390a3f 100644 --- a/mediagoblin/edit/forms.py +++ b/mediagoblin/edit/forms.py @@ -15,10 +15,12 @@ # along with this program. If not, see . import wtforms +from jsonschema import Draft4Validator from mediagoblin.tools.text import tag_length_validator from mediagoblin.tools.translate import lazy_pass_to_ugettext as _ from mediagoblin.tools.licenses import licenses_as_choices +from mediagoblin.tools.metadata import DEFAULT_SCHEMA, DEFAULT_CHECKER from mediagoblin.auth.tools import normalize_user_or_email_field @@ -123,11 +125,37 @@ class ChangeEmailForm(wtforms.Form): description=_( "Enter your password to prove you own this account.")) +class MetaDataValidator(object): + """ + Custom validator which runs form data in a MetaDataForm through a jsonschema + validator and passes errors recieved in jsonschema to wtforms. + + :param schema The json schema to validate the data against. By + default this uses the DEFAULT_SCHEMA from + mediagoblin.tools.metadata. + :param format_checker The FormatChecker object that limits which types + jsonschema can recognize. By default this uses + DEFAULT_CHECKER from mediagoblin.tools.metadata. + """ + def __init__(self, schema=DEFAULT_SCHEMA, format_checker=DEFAULT_CHECKER): + self.schema = schema + self.format_checker = format_checker + + def __call__(self, form, field): + metadata_dict = {field.data:form.value.data} + validator = Draft4Validator(self.schema, + format_checker=self.format_checker) + errors = [e.message + for e in validator.iter_errors(metadata_dict)] + if len(errors) >= 1: + raise wtforms.validators.ValidationError( + errors.pop()) + class MetaDataForm(wtforms.Form): - identifier = wtforms.TextField(_(u'Identifier')) + identifier = wtforms.TextField(_(u'Identifier'),[MetaDataValidator()]) value = wtforms.TextField(_(u'Value')) class EditMetaDataForm(wtforms.Form): media_metadata = wtforms.FieldList( - wtforms.FormField(MetaDataForm, label="") + wtforms.FormField(MetaDataForm, ""), ) diff --git a/mediagoblin/edit/views.py b/mediagoblin/edit/views.py index 34021257..cfbaf2fa 100644 --- a/mediagoblin/edit/views.py +++ b/mediagoblin/edit/views.py @@ -20,6 +20,7 @@ from itsdangerous import BadSignature from pyld import jsonld from werkzeug.exceptions import Forbidden from werkzeug.utils import secure_filename +from jsonschema import ValidationError, Draft4Validator from mediagoblin import messages from mediagoblin import mg_globals @@ -33,7 +34,8 @@ from mediagoblin.decorators import (require_active_login, active_user_from_url, get_user_collection, user_has_privilege, user_not_banned) from mediagoblin.tools.crypto import get_timed_signer_url -from mediagoblin.tools.metadata import compact_and_validate +from mediagoblin.tools.metadata import (compact_and_validate, DEFAULT_CHECKER, + DEFAULT_SCHEMA) from mediagoblin.tools.mail import email_debug_message from mediagoblin.tools.response import (render_to_response, redirect, redirect_obj, render_404) @@ -444,24 +446,19 @@ def edit_metadata(request, media): if request.method == "POST" and form.validate(): metadata_dict = dict([(row['identifier'],row['value']) for row in form.media_metadata.data]) + json_ld_metadata = None json_ld_metadata = compact_and_validate(metadata_dict) media.media_metadata = json_ld_metadata media.save() return redirect_obj(request, media) - if media.media_metadata: + if media.media_metadata and len(form.media_metadata) == 0: for identifier, value in media.media_metadata.iteritems(): if identifier == "@context": continue form.media_metadata.append_entry({ 'identifier':identifier, 'value':value}) - else: - form.media_metadata.append_entry({ - 'identifier':"", - 'value':""}) - form.media_metadata.append_entry({ - 'identifier':"", - 'value':""}) + return render_to_response( request, 'mediagoblin/edit/metadata.html', diff --git a/mediagoblin/static/css/base.css b/mediagoblin/static/css/base.css index a3b564ea..9087034b 100644 --- a/mediagoblin/static/css/base.css +++ b/mediagoblin/static/css/base.css @@ -940,18 +940,13 @@ p.verifier { /* for the media metadata editing table */ table.metadata_editor { - margin: 10px auto; - width: 1000px; -} - -table.metadata_editor tr th { - width:100px; + width: 800px; } table.metadata_editor tr td { - width:300px; + width:350px; } table.metadata_editor tr td.form_field_input input { - width:300px; + width:350px; } diff --git a/mediagoblin/templates/mediagoblin/edit/metadata.html b/mediagoblin/templates/mediagoblin/edit/metadata.html index b5a52e5f..21eb27b1 100644 --- a/mediagoblin/templates/mediagoblin/edit/metadata.html +++ b/mediagoblin/templates/mediagoblin/edit/metadata.html @@ -69,7 +69,7 @@ -

{% trans %}Data{% endtrans %}

+

{% trans %}MetaData{% endtrans %}

{{ wtforms_util.render_fieldlist_as_table_rows(form.media_metadata) }}
@@ -77,16 +77,13 @@ - - - diff --git a/mediagoblin/templates/mediagoblin/utils/wtforms.html b/mediagoblin/templates/mediagoblin/utils/wtforms.html index c83d53f1..7e16708c 100644 --- a/mediagoblin/templates/mediagoblin/utils/wtforms.html +++ b/mediagoblin/templates/mediagoblin/utils/wtforms.html @@ -77,20 +77,21 @@ {% macro render_form_as_table_row(form) %} {%- for field in form %} - {%- endfor %} + + {%- for field in form %} + {% for error in field.errors %} + + + + {%- endfor %} + {%- endfor %} {%- endmacro %} {% macro render_field_as_table_row(field) %} @@ -98,16 +99,15 @@ + {% for error in field.errors %} + + + + {%- endfor %} {% endmacro %} {% macro render_fieldlist_as_table_rows(fieldlist) %} From c8abeb58afd36c56af705a69a2d3ebec90002e74 Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Wed, 14 May 2014 11:56:59 -0400 Subject: [PATCH 74/78] Set a default value to MediaEntry.media_metadata --- mediagoblin/db/migrations.py | 3 ++- mediagoblin/db/models.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/mediagoblin/db/migrations.py b/mediagoblin/db/migrations.py index 8dac3214..dd69ad6e 100644 --- a/mediagoblin/db/migrations.py +++ b/mediagoblin/db/migrations.py @@ -728,7 +728,8 @@ def add_metadata_column(db): media_entry = inspect_table(metadata, 'core__media_entries') - col = Column('media_metadata', MutationDict.as_mutable(JSONEncoded)) + col = Column('media_metadata', MutationDict.as_mutable(JSONEncoded), + default=MutationDict()) col.create(media_entry) db.commit() diff --git a/mediagoblin/db/models.py b/mediagoblin/db/models.py index defa0849..8499ea3f 100644 --- a/mediagoblin/db/models.py +++ b/mediagoblin/db/models.py @@ -264,7 +264,8 @@ class MediaEntry(Base, MediaEntryMixin): cascade="all, delete-orphan" ) collections = association_proxy("collections_helper", "in_collection") - media_metadata = Column(MutationDict.as_mutable(JSONEncoded)) + media_metadata = Column(MutationDict.as_mutable(JSONEncoded), + default=MutationDict()) ## TODO # fail_error From 0e69d93295abead71bf0cf63d526238427b6e648 Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Wed, 14 May 2014 12:02:54 -0400 Subject: [PATCH 75/78] Fixed small error in the edit.metadata javascript --- mediagoblin/templates/mediagoblin/edit/metadata.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediagoblin/templates/mediagoblin/edit/metadata.html b/mediagoblin/templates/mediagoblin/edit/metadata.html index 21eb27b1..d0efbbfb 100644 --- a/mediagoblin/templates/mediagoblin/edit/metadata.html +++ b/mediagoblin/templates/mediagoblin/edit/metadata.html @@ -56,7 +56,7 @@ 'media_metadata-'); metadata_lines += 1; }) - }) + $("#clear_empty_rows").click(function(){ clear_empty_rows("#metadata_list"); }) From 1688abbfc4c429af075ff95f2bb5d15266077c77 Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Wed, 14 May 2014 12:03:58 -0400 Subject: [PATCH 76/78] Cleaned up the code a little bit --- mediagoblin/edit/views.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mediagoblin/edit/views.py b/mediagoblin/edit/views.py index cfbaf2fa..e998d6be 100644 --- a/mediagoblin/edit/views.py +++ b/mediagoblin/edit/views.py @@ -450,9 +450,9 @@ def edit_metadata(request, media): json_ld_metadata = compact_and_validate(metadata_dict) media.media_metadata = json_ld_metadata media.save() - return redirect_obj(request, media) + return redirect_obj(request, media) - if media.media_metadata and len(form.media_metadata) == 0: + if len(form.media_metadata) == 0: for identifier, value in media.media_metadata.iteritems(): if identifier == "@context": continue form.media_metadata.append_entry({ From 414c682fb4a5b684cf259743a77e6e569395c16d Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Wed, 14 May 2014 12:16:03 -0400 Subject: [PATCH 77/78] Added some tests for metadata. --- mediagoblin/tests/test_edit.py | 84 ++++++++++++++++++++++++++++-- mediagoblin/tests/test_metadata.py | 25 --------- 2 files changed, 81 insertions(+), 28 deletions(-) diff --git a/mediagoblin/tests/test_edit.py b/mediagoblin/tests/test_edit.py index 4f44e0b9..dc9c422f 100644 --- a/mediagoblin/tests/test_edit.py +++ b/mediagoblin/tests/test_edit.py @@ -14,11 +14,11 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -import urlparse +import urlparse, os, pytest from mediagoblin import mg_globals -from mediagoblin.db.models import User -from mediagoblin.tests.tools import fixture_add_user +from mediagoblin.db.models import User, MediaEntry +from mediagoblin.tests.tools import fixture_add_user, fixture_media_entry from mediagoblin import auth from mediagoblin.tools import template, mail @@ -174,3 +174,81 @@ class TestUserEdit(object): email = User.query.filter_by(username='chris').first().email assert email == 'new@example.com' # test changing the url inproperly + +class TestMetaDataEdit: + @pytest.fixture(autouse=True) + def setup(self, test_app): + # set up new user + self.user_password = u'toast' + self.user = fixture_add_user(password = self.user_password, + privileges=[u'active',u'admin']) + self.test_app = test_app + + def login(self, test_app): + test_app.post( + '/auth/login/', { + 'username': self.user.username, + 'password': self.user_password}) + + def do_post(self, data, *context_keys, **kwargs): + url = kwargs.pop('url', '/submit/') + do_follow = kwargs.pop('do_follow', False) + template.clear_test_template_context() + response = self.test_app.post(url, data, **kwargs) + if do_follow: + response.follow() + context_data = template.TEMPLATE_TEST_CONTEXT + for key in context_keys: + context_data = context_data[key] + return response, context_data + + def test_edit_metadata(self, test_app): + media_entry = fixture_media_entry(uploader=self.user.id, + state=u'processed') + media_slug = "/u/{username}/m/{media_id}/metadata/".format( + username = str(self.user.username), + media_id = str(media_entry.id)) + + self.login(test_app) + response = test_app.get(media_slug) + assert response.status == '200 OK' + assert media_entry.media_metadata == {} + # First test adding in metadata + ################################ + response, context = self.do_post({ + "media_metadata-0-identifier":"dc:title", + "media_metadata-0-value":"Some title", + "media_metadata-1-identifier":"dc:creator", + "media_metadata-1-value":"Me"},url=media_slug) + + media_entry = MediaEntry.query.first() + new_metadata = media_entry.media_metadata + assert new_metadata != {} + assert new_metadata.get("dc:title") == "Some title" + assert new_metadata.get("dc:creator") == "Me" + # Now test removing the metadata + ################################ + response, context = self.do_post({ + "media_metadata-0-identifier":"dc:title", + "media_metadata-0-value":"Some title"},url=media_slug) + + media_entry = MediaEntry.query.first() + new_metadata = media_entry.media_metadata + assert new_metadata.get("dc:title") == "Some title" + assert new_metadata.get("dc:creator") is None + # Now test adding bad metadata + ############################### + response, context = self.do_post({ + "media_metadata-0-identifier":"dc:title", + "media_metadata-0-value":"Some title", + "media_metadata-1-identifier":"dc:creator", + "media_metadata-1-value":"Me", + "media_metadata-2-identifier":"dc:created", + "media_metadata-2-value":"On the worst day"},url=media_slug) + + media_entry = MediaEntry.query.first() + old_metadata = new_metadata + new_metadata = media_entry.media_metadata + assert new_metadata == old_metadata + assert ("u'On the worst day' is not a 'date-time'" in + response.body) diff --git a/mediagoblin/tests/test_metadata.py b/mediagoblin/tests/test_metadata.py index 71346bb4..b4ea646e 100644 --- a/mediagoblin/tests/test_metadata.py +++ b/mediagoblin/tests/test_metadata.py @@ -15,40 +15,15 @@ # along with this program. If not, see . import pytest -from mediagoblin.tools import template from mediagoblin.tools.metadata import compact_and_validate -from webtest import AppError from jsonschema import ValidationError -from .resources import GOOD_JPG - class TestMetadataFunctionality: @pytest.fixture(autouse=True) def _setup(self, test_app): self.test_app = test_app - def login(self, username): - self.test_app.post( - '/auth/login/', { - 'username': username, - 'password': 'toast'}) - - def logout(self): - self.test_app.get('/auth/logout/') - - def do_post(self, data, *context_keys, **kwargs): - url = kwargs.pop('url', '/submit/') - do_follow = kwargs.pop('do_follow', False) - template.clear_test_template_context() - response = self.test_app.post(url, data, **kwargs) - if do_follow: - response.follow() - context_data = template.TEMPLATE_TEST_CONTEXT - for key in context_keys: - context_data = context_data[key] - return response, context_data - def testCompactAndValidate(self): # First, test out a well formatted piece of metadata ###################################################### From 2daf8ec00043a4cc5cd120f875a5382aca6ec7f9 Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Wed, 14 May 2014 12:34:13 -0400 Subject: [PATCH 78/78] Fixed a small error relating to the default value of media_metadata --- mediagoblin/submit/lib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediagoblin/submit/lib.py b/mediagoblin/submit/lib.py index df3f7b62..93ae7a1f 100644 --- a/mediagoblin/submit/lib.py +++ b/mediagoblin/submit/lib.py @@ -142,7 +142,7 @@ def submit_media(mg_app, user, submitted_file, filename, entry.license = license or None - entry.media_metadata = metadata or u"" + entry.media_metadata = metadata or {} # Process the user's folksonomy "tags" entry.tags = convert_to_tag_list_of_dicts(tags_string)
{{ render_label_p(field) }} {{field}} - {%- if field.errors -%} -
-
    - {% for error in field.errors %} -
  • {{error}}
    • - {%- endfor %} -
- {%- endif -%}
+

{{error}}

+
{{ field.label.text }} {{field}} - {% if field.errors %} -
-
    - {% for error in field.errors %} -
  • {{error}}
    • - {% endfor %} -
- {% endif %}
+

{{error}}

+