Merge branch 'master' into merge-python3-port

Has some issues, will iteratively fix!

Conflicts:
	mediagoblin/gmg_commands/__init__.py
	mediagoblin/gmg_commands/deletemedia.py
	mediagoblin/gmg_commands/users.py
	mediagoblin/oauth/views.py
	mediagoblin/plugins/api/views.py
	mediagoblin/tests/test_api.py
	mediagoblin/tests/test_edit.py
	mediagoblin/tests/test_oauth1.py
	mediagoblin/tests/test_util.py
	mediagoblin/tools/mail.py
	mediagoblin/webfinger/views.py
	setup.py

mediagoblin/_version.py

@@ -23,4 +23,4 @@
 
 # see http://www.python.org/dev/peps/pep-0386/
 
-__version__ = "0.6.2.dev"
+__version__ = "0.7.1.dev"

mediagoblin/app.py

@@ -234,6 +234,8 @@ class MediaGoblinApp(object):
                 request, e,
                 e.get_description(environ))(environ, start_response)
 
+        request = hook_transform("modify_request", request)
+
         request.start_response = start_response
 
         # get the Http response from the controller

mediagoblin/auth/__init__.py

@@ -25,7 +25,6 @@ def create_user(register_form):
     results = hook_runall("auth_create_user", register_form)
     return results[0]
 
-
 def extra_validation(register_form):
     from mediagoblin.auth.tools import basic_extra_validation
 

mediagoblin/auth/tools.py

@@ -134,11 +134,7 @@ def register_user(request, register_form):
         user = auth.create_user(register_form)
 
         # give the user the default privileges
-        default_privileges = [
-            Privilege.query.filter(Privilege.privilege_name==u'commenter').first(),
-            Privilege.query.filter(Privilege.privilege_name==u'uploader').first(),
-            Privilege.query.filter(Privilege.privilege_name==u'reporter').first()]
-        user.all_privileges += default_privileges
+        user.all_privileges += get_default_privileges(user)
         user.save()
 
         # log the user in
@@ -153,6 +149,14 @@ def register_user(request, register_form):
 
     return None
 
+def get_default_privileges(user):
+    instance_privilege_scheme = mg_globals.app_config['user_privilege_scheme']
+    default_privileges = [Privilege.query.filter(
+        Privilege.privilege_name==privilege_name).first()
+        for privilege_name in instance_privilege_scheme.split(',')]
+    default_privileges = [privilege for privilege in default_privileges if not privilege == None]
+
+    return default_privileges
 
 def check_login_simple(username, password):
     user = auth.get_user(username=username)

mediagoblin/config_spec.ini

@@ -23,13 +23,29 @@ direct_remote_path = string(default="/mgoblin_static/")
 
 # set to false to enable sending notices
 email_debug_mode = boolean(default=True)
+
+# Uses SSL/TLS when connecting to SMTP server
 email_smtp_use_ssl = boolean(default=False)
+
+# Uses STARTTLS when connecting to SMTP server
+email_smtp_force_starttls = boolean(default=False)
+
+# Email address which notices are sent from
 email_sender_address = string(default="notice@mediagoblin.example.org")
+
+# Hostname of SMTP server
 email_smtp_host = string(default='')
+
+# Port for SMTP server
 email_smtp_port = integer(default=0)
+
+# Username used for SMTP server
 email_smtp_user = string(default=None)
+
+# Password used for SMTP server
 email_smtp_pass = string(default=None)
 
+
 # Set to false to disable registrations
 allow_registration = boolean(default=True)
 
@@ -89,6 +105,13 @@ upload_limit = integer(default=None)
 # Max file size (in Mb)
 max_file_size = integer(default=None)
 
+# Privilege scheme
+user_privilege_scheme = string(default="uploader,commenter,reporter")
+
+# Frequency garbage collection will run (setting to 0 or false to disable)
+# Setting units are minutes.
+garbage_collection = integer(default=60)
+
 [jinja2]
 # Jinja2 supports more directives than the minimum required by mediagoblin. 
 # This setting allows users creating custom templates to specify a list of

mediagoblin/db/__init__.py

@@ -14,36 +14,3 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-"""
-Database Abstraction/Wrapper Layer
-==================================
-
-This submodule is for most of the db specific stuff.
-
-There are two main ideas here:
-
-1. Open up a small possibility to replace mongo by another
-   db.  This means, that all direct mongo accesses should
-   happen in the db submodule.  While all the rest uses an
-   API defined by this submodule.
-
-   Currently this API happens to be basicly mongo.
-   Which means, that the abstraction/wrapper layer is
-   extremely thin.
-
-2. Give the rest of the app a simple and easy way to get most of
-   their db needs. Which often means some simple import
-   from db.util.
-
-What does that mean?
-
-* Never import mongo directly outside of this submodule.
-
-* Inside this submodule you can do whatever is needed. The
-  API border is exactly at the submodule layer. Nowhere
-  else.
-
-* helper functions can be moved in here. They become part
-  of the db.* API
-
-"""

mediagoblin/db/migrations.py

@@ -21,18 +21,19 @@ import six
 
 from sqlalchemy import (MetaData, Table, Column, Boolean, SmallInteger,
                         Integer, Unicode, UnicodeText, DateTime,
-                        ForeignKey, Date)
+                        ForeignKey, Date, Index)
 from sqlalchemy.exc import ProgrammingError
 from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy.sql import and_
 from sqlalchemy.schema import UniqueConstraint
 
-
 from mediagoblin.db.extratypes import JSONEncoded, MutationDict
 from mediagoblin.db.migration_tools import (
     RegisterMigration, inspect_table, replace_table_hack)
-from mediagoblin.db.models import (MediaEntry, Collection, MediaComment, User, 
-        Privilege)
+from mediagoblin.db.models import (MediaEntry, Collection, MediaComment, User,
+    Privilege)
+from mediagoblin.db.extratypes import JSONEncoded, MutationDict
+
 
 MIGRATIONS = {}
 
@@ -467,7 +468,6 @@ def create_oauth1_tables(db):
 
     db.commit()
 
-
 @RegisterMigration(15, MIGRATIONS)
 def wants_notifications(db):
     """Add a wants_notifications field to User model"""
@@ -661,8 +661,8 @@ def create_moderation_tables(db):
     # admin, an active user or an inactive user ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
     for admin_user in admin_users_ids:
         admin_user_id = admin_user['id']
-        for privilege_id in [admin_privilege_id, uploader_privilege_id, 
-                            reporter_privilege_id, commenter_privilege_id, 
+        for privilege_id in [admin_privilege_id, uploader_privilege_id,
+                            reporter_privilege_id, commenter_privilege_id,
                             active_privilege_id]:
             db.execute(user_privilege_assoc.insert().values(
                 core__privilege_id=admin_user_id,
@@ -670,7 +670,7 @@ def create_moderation_tables(db):
 
     for active_user in active_users_ids:
         active_user_id = active_user['id']
-        for privilege_id in [uploader_privilege_id, reporter_privilege_id, 
+        for privilege_id in [uploader_privilege_id, reporter_privilege_id,
                             commenter_privilege_id, active_privilege_id]:
             db.execute(user_privilege_assoc.insert().values(
                 core__privilege_id=active_user_id,
@@ -678,7 +678,7 @@ def create_moderation_tables(db):
 
     for inactive_user in inactive_users_ids:
         inactive_user_id = inactive_user['id']
-        for privilege_id in [uploader_privilege_id, reporter_privilege_id, 
+        for privilege_id in [uploader_privilege_id, reporter_privilege_id,
                              commenter_privilege_id]:
             db.execute(user_privilege_assoc.insert().values(
                 core__privilege_id=inactive_user_id,
@@ -709,6 +709,8 @@ def create_moderation_tables(db):
         is_admin.drop()
 
     db.commit()
+
+
 @RegisterMigration(19, MIGRATIONS)
 def drop_MediaEntry_collected(db):
     """
@@ -722,3 +724,171 @@ def drop_MediaEntry_collected(db):
     media_collected.drop()
 
     db.commit()
+
+
+@RegisterMigration(20, MIGRATIONS)
+def add_metadata_column(db):
+    metadata = MetaData(bind=db.bind)
+
+    media_entry = inspect_table(metadata, 'core__media_entries')
+
+    col = Column('media_metadata', MutationDict.as_mutable(JSONEncoded),
+        default=MutationDict())
+    col.create(media_entry)
+
+    db.commit()
+
+
+class PrivilegeUserAssociation_R1(declarative_base()):
+    __tablename__ = 'rename__privileges_users'
+    user = Column(
+        "user",
+        Integer,
+        ForeignKey(User.id),
+        primary_key=True)
+    privilege = Column(
+        "privilege",
+        Integer,
+        ForeignKey(Privilege.id),
+        primary_key=True)
+
+@RegisterMigration(21, MIGRATIONS)
+def fix_privilege_user_association_table(db):
+    """
+    There was an error in the PrivilegeUserAssociation table that allowed for a
+    dangerous sql error. We need to the change the name of the columns to be
+    unique, and properly referenced.
+    """
+    metadata = MetaData(bind=db.bind)
+
+    privilege_user_assoc = inspect_table(
+        metadata, 'core__privileges_users')
+
+    # This whole process is more complex if we're dealing with sqlite
+    if db.bind.url.drivername == 'sqlite':
+        PrivilegeUserAssociation_R1.__table__.create(db.bind)
+        db.commit()
+
+        new_privilege_user_assoc = inspect_table(
+            metadata, 'rename__privileges_users')
+        result = db.execute(privilege_user_assoc.select())
+        for row in result:
+            # The columns were improperly named before, so we switch the columns
+            user_id, priv_id = row['core__privilege_id'], row['core__user_id']
+            db.execute(new_privilege_user_assoc.insert().values(
+                user=user_id,
+                privilege=priv_id))
+
+        db.commit()
+
+        privilege_user_assoc.drop()
+        new_privilege_user_assoc.rename('core__privileges_users')
+
+    # much simpler if postgres though!
+    else:
+        privilege_user_assoc.c.core__user_id.alter(name="privilege")
+        privilege_user_assoc.c.core__privilege_id.alter(name="user")
+
+    db.commit()
+
+
+@RegisterMigration(22, MIGRATIONS)
+def add_index_username_field(db):
+    """
+    This migration has been found to be doing the wrong thing.  See
+    the documentation in migration 23 (revert_username_index) below
+    which undoes this for those databases that did run this migration.
+
+    Old description:
+      This indexes the User.username field which is frequently queried
+      for example a user logging in. This solves the issue #894
+    """
+    ## This code is left commented out *on purpose!*
+    ##
+    ## We do not normally allow commented out code like this in
+    ## MediaGoblin but this is a special case: since this migration has
+    ## been nullified but with great work to set things back below,
+    ## this is commented out for historical clarity.
+    #
+    # metadata = MetaData(bind=db.bind)
+    # user_table = inspect_table(metadata, "core__users")
+    #
+    # new_index = Index("ix_core__users_uploader", user_table.c.username)
+    # new_index.create()
+    #
+    # db.commit()
+    pass
+
+
+@RegisterMigration(23, MIGRATIONS)
+def revert_username_index(db):
+    """
+    Revert the stuff we did in migration 22 above.
+
+    There were a couple of problems with what we did:
+     - There was never a need for this migration!  The unique
+       constraint had an implicit b-tree index, so it wasn't really
+       needed.  (This is my (Chris Webber's) fault for suggesting it
+       needed to happen without knowing what's going on... my bad!)
+     - On top of that, databases created after the models.py was
+       changed weren't the same as those that had been run through
+       migration 22 above.
+
+    As such, we're setting things back to the way they were before,
+    but as it turns out, that's tricky to do!
+    """
+    metadata = MetaData(bind=db.bind)
+    user_table = inspect_table(metadata, "core__users")
+    indexes = dict(
+        [(index.name, index) for index in user_table.indexes])
+
+    # index from unnecessary migration
+    users_uploader_index = indexes.get(u'ix_core__users_uploader')
+    # index created from models.py after (unique=True, index=True)
+    # was set in models.py
+    users_username_index = indexes.get(u'ix_core__users_username')
+
+    if users_uploader_index is None and users_username_index is None:
+        # We don't need to do anything.
+        # The database isn't in a state where it needs fixing
+        #
+        # (ie, either went through the previous borked migration or
+        #  was initialized with a models.py where core__users was both
+        #  unique=True and index=True)
+        return
+
+    if db.bind.url.drivername == 'sqlite':
+        # Again, sqlite has problems.  So this is tricky.
+
+        # Yes, this is correct to use User_vR1!  Nothing has changed
+        # between the *correct* version of this table and migration 18.
+        User_vR1.__table__.create(db.bind)
+        db.commit()
+        new_user_table = inspect_table(metadata, 'rename__users')
+        replace_table_hack(db, user_table, new_user_table)
+
+    else:
+        # If the db is not run using SQLite, we don't need to do crazy
+        # table copying.
+
+        # Remove whichever of the not-used indexes are in place
+        if users_uploader_index is not None:
+            users_uploader_index.drop()
+        if users_username_index is not None:
+            users_username_index.drop()
+
+        # Given we're removing indexes then adding a unique constraint
+        # which *we know might fail*, thus probably rolling back the
+        # session, let's commit here.
+        db.commit()
+
+        try:
+            # Add the unique constraint
+            constraint = UniqueConstraint(
+                'username', table=user_table)
+            constraint.create()
+        except ProgrammingError:
+            # constraint already exists, no need to add
+            db.rollback()
+
+    db.commit()

mediagoblin/db/mixin.py

@@ -202,6 +202,17 @@ class MediaEntryMixin(GenerateSlugMixin):
             thumb_url = mg_globals.app.staticdirector(manager[u'default_thumb'])
         return thumb_url
 
+    @property
+    def original_url(self):
+        """ Returns the URL for the original image
+        will return self.thumb_url if original url doesn't exist"""
+        if u"original" not in self.media_files:
+            return self.thumb_url
+        
+        return mg_globals.app.public_store.file_url(
+            self.media_files[u"original"]
+            )
+
     @cached_property
     def media_manager(self):
         """Returns the MEDIA_MANAGER of the media's media_type
@@ -248,7 +259,7 @@ class MediaEntryMixin(GenerateSlugMixin):
 
         if 'Image DateTimeOriginal' in exif_all:
             # format date taken
-            takendate = datetime.datetime.strptime(
+            takendate = datetime.strptime(
                 exif_all['Image DateTimeOriginal']['printable'],
                 '%Y:%m:%d %H:%M:%S').date()
             taken = takendate.strftime('%B %d %Y')
@@ -294,6 +305,13 @@ class MediaCommentMixin(object):
         """
         return cleaned_markdown_conversion(self.content)
 
+    def __unicode__(self):
+        return u'<{klass} #{id} {author} "{comment}">'.format(
+            klass=self.__class__.__name__,
+            id=self.id,
+            author=self.get_author,
+            comment=self.content)
+
     def __repr__(self):
         return '<{klass} #{id} {author} "{comment}">'.format(
             klass=self.__class__.__name__,

mediagoblin/db/models.py

@@ -101,25 +101,26 @@ class User(Base, UserMixin):
         super(User, self).delete(**kwargs)
         _log.info('Deleted user "{0}" account'.format(self.username))
 
-    def has_privilege(self,*priv_names):
+    def has_privilege(self, privilege, allow_admin=True):
         """
         This method checks to make sure a user has all the correct privileges
         to access a piece of content.
 
-        :param  priv_names      A variable number of unicode objects which rep-
-                                -resent the different privileges which may give
-                                the user access to this content. If you pass
-                                multiple arguments, the user will be granted
-                                access if they have ANY of the privileges
-                                passed.
+        :param  privilege       A unicode object which represent the different
+                                privileges which may give the user access to
+                                content.
+
+        :param  allow_admin     If this is set to True the then if the user is
+                                an admin, then this will always return True
+                                even if the user hasn't been given the
+                                privilege. (defaults to True)
         """
-        if len(priv_names) == 1:
-            priv = Privilege.query.filter(
-                Privilege.privilege_name==priv_names[0]).one()
-            return (priv in self.all_privileges)
-        elif len(priv_names) > 1:
-            return self.has_privilege(priv_names[0]) or \
-                self.has_privilege(*priv_names[1:])
+        priv = Privilege.query.filter_by(privilege_name=privilege).one()
+        if priv in self.all_privileges:
+            return True
+        elif allow_admin and self.has_privilege(u'admin', allow_admin=False):
+            return True
+
         return False
 
     def is_banned(self):
@@ -132,6 +133,48 @@ class User(Base, UserMixin):
         return UserBan.query.get(self.id) is not None
 
 
+    def serialize(self, request):
+        user = {
+            "id": "acct:{0}@{1}".format(self.username, request.host),
+            "preferredUsername": self.username,
+            "displayName": "{0}@{1}".format(self.username, request.host),
+            "objectType": "person",
+            "pump_io": {
+                "shared": False,
+                "followed": False,
+            },
+            "links": {
+                "self": {
+                    "href": request.urlgen(
+                            "mediagoblin.federation.user.profile",
+                             username=self.username,
+                             qualified=True
+                             ),
+                },
+                "activity-inbox": {
+                    "href": request.urlgen(
+                            "mediagoblin.federation.inbox",
+                            username=self.username,
+                            qualified=True
+                            )
+                },
+                "activity-outbox": {
+                    "href": request.urlgen(
+                            "mediagoblin.federation.feed",
+                            username=self.username,
+                            qualified=True
+                            )
+                },
+            },
+        }
+
+        if self.bio:
+            user.update({"summary": self.bio})
+        if self.url:
+            user.update({"url": self.url})
+
+        return user
+
 class Client(Base):
     """
         Model representing a client - Used for API Auth
@@ -197,7 +240,6 @@ class NonceTimestamp(Base):
     nonce = Column(Unicode, nullable=False, primary_key=True)
     timestamp = Column(DateTime, nullable=False, primary_key=True)
 
-
 class MediaEntry(Base, MediaEntryMixin):
     """
     TODO: Consider fetching the media_files using join
@@ -260,6 +302,8 @@ class MediaEntry(Base, MediaEntryMixin):
         cascade="all, delete-orphan"
         )
     collections = association_proxy("collections_helper", "in_collection")
+    media_metadata = Column(MutationDict.as_mutable(JSONEncoded),
+        default=MutationDict())
 
     ## TODO
     # fail_error
@@ -382,6 +426,80 @@ class MediaEntry(Base, MediaEntryMixin):
         # pass through commit=False/True in kwargs
         super(MediaEntry, self).delete(**kwargs)
 
+    @property
+    def objectType(self):
+        """ Converts media_type to pump-like type - don't use internally """
+        return self.media_type.split(".")[-1]
+
+    def serialize(self, request, show_comments=True):
+        """ Unserialize MediaEntry to object """
+        author = self.get_uploader
+        context = {
+            "id": self.id,
+            "author": author.serialize(request),
+            "objectType": self.objectType,
+            "url": self.url_for_self(request.urlgen),
+            "image": {
+                "url": request.host_url + self.thumb_url[1:],
+            },
+            "fullImage":{
+                "url": request.host_url + self.original_url[1:],
+            },
+            "published": self.created.isoformat(),
+            "updated": self.created.isoformat(),
+            "pump_io": {
+                "shared": False,
+            },
+            "links": {
+                "self": {
+                    "href": request.urlgen(
+                        "mediagoblin.federation.object",
+                        objectType=self.objectType,
+                        id=self.id,
+                        qualified=True
+                    ),
+                },
+
+            }
+        }
+
+        if self.title:
+            context["displayName"] = self.title
+
+        if self.description:
+            context["content"] = self.description
+
+        if self.license:
+            context["license"] = self.license
+
+        if show_comments:
+            comments = [comment.serialize(request) for comment in self.get_comments()]
+            total = len(comments)
+            context["replies"] = {
+                "totalItems": total,
+                "items": comments,
+                "url": request.urlgen(
+                        "mediagoblin.federation.object.comments",
+                        objectType=self.objectType,
+                        id=self.id,
+                        qualified=True
+                        ),
+            }
+
+        return context
+
+    def unserialize(self, data):
+        """ Takes API objects and unserializes on existing MediaEntry """
+        if "displayName" in data:
+            self.title = data["displayName"]
+
+        if "content" in data:
+            self.description = data["content"]
+
+        if "license" in data:
+            self.license = data["license"]
+
+        return True
 
 class FileKeynames(Base):
     """
@@ -528,6 +646,47 @@ class MediaComment(Base, MediaCommentMixin):
                                                    lazy="dynamic",
                                                    cascade="all, delete-orphan"))
 
+    def serialize(self, request):
+        """ Unserialize to python dictionary for API """
+        media = MediaEntry.query.filter_by(id=self.media_entry).first()
+        author = self.get_author
+        context = {
+            "id": self.id,
+            "objectType": "comment",
+            "content": self.content,
+            "inReplyTo": media.serialize(request, show_comments=False),
+            "author": author.serialize(request)
+        }
+
+        return context
+
+    def unserialize(self, data):
+        """ Takes API objects and unserializes on existing comment """
+        # Do initial checks to verify the object is correct
+        required_attributes = ["content", "inReplyTo"]
+        for attr in required_attributes:
+            if attr not in data:
+                return False
+
+        # Validate inReplyTo has ID
+        if "id" not in data["inReplyTo"]:
+            return False
+
+        # Validate that the ID is correct
+        try:
+            media_id = int(data["inReplyTo"]["id"])
+        except ValueError:
+            return False
+
+        media = MediaEntry.query.filter_by(id=media_id).first()
+        if media is None:
+            return False
+
+        self.media_entry = media.id
+        self.content = data["content"]
+        return True
+
+
 
 class Collection(Base, CollectionMixin):
     """An 'album' or 'set' of media by a user.
@@ -563,6 +722,14 @@ class Collection(Base, CollectionMixin):
         return CollectionItem.query.filter_by(
             collection=self.id).order_by(order_col)
 
+    def __repr__(self):
+        safe_title = self.title.encode('ascii', 'replace')
+        return '<{classname} #{id}: {title} by {creator}>'.format(
+            id=self.id,
+            classname=self.__class__.__name__,
+            creator=self.creator,
+            title=safe_title)
+
 
 class CollectionItem(Base, CollectionItemMixin):
     __tablename__ = "core__collection_items"
@@ -592,6 +759,13 @@ class CollectionItem(Base, CollectionItemMixin):
         """A dict like view on this object"""
         return DictReadAttrProxy(self)
 
+    def __repr__(self):
+        return '<{classname} #{id}: Entry {entry} in {collection}>'.format(
+            id=self.id,
+            classname=self.__class__.__name__,
+            collection=self.collection,
+            entry=self.media_entry)
+
 
 class ProcessingMetaData(Base):
     __tablename__ = 'core__processing_metadata'
@@ -667,6 +841,14 @@ class Notification(Base):
             subject=getattr(self, 'subject', None),
             seen='unseen' if not self.seen else 'seen')
 
+    def __unicode__(self):
+        return u'<{klass} #{id}: {user}: {subject} ({seen})>'.format(
+            id=self.id,
+            klass=self.__class__.__name__,
+            user=self.user,
+            subject=getattr(self, 'subject', None),
+            seen='unseen' if not self.seen else 'seen')
+
 
 class CommentNotification(Notification):
     __tablename__ = 'core__comment_notifications'
@@ -871,13 +1053,13 @@ class PrivilegeUserAssociation(Base):
 
     __tablename__ = 'core__privileges_users'
 
-    privilege_id = Column(
-        'core__privilege_id',
+    user = Column(
+        "user",
         Integer,
         ForeignKey(User.id),
         primary_key=True)
-    user_id = Column(
-        'core__user_id',
+    privilege = Column(
+        "privilege",
         Integer,
         ForeignKey(Privilege.id),
         primary_key=True)

mediagoblin/db/util.py

@@ -76,11 +76,16 @@ def check_db_up_to_date():
     dbdatas = gather_database_data(mgg.global_config.get('plugins', {}).keys())
 
     for dbdata in dbdatas:
-        migration_manager = dbdata.make_migration_manager(Session())
-        if migration_manager.database_current_migration is None or \
-                migration_manager.migrations_to_run():
-            sys.exit("Your database is not up to date. Please run "
-                     "'gmg dbupdate' before starting MediaGoblin.")
+        session = Session()
+        try:
+            migration_manager = dbdata.make_migration_manager(session)
+            if migration_manager.database_current_migration is None or \
+                    migration_manager.migrations_to_run():
+                sys.exit("Your database is not up to date. Please run "
+                         "'gmg dbupdate' before starting MediaGoblin.")
+        finally:
+            Session.rollback()
+            Session.remove()
 
 
 if __name__ == '__main__':

mediagoblin/decorators.py

@@ -23,7 +23,7 @@ from six.moves.urllib.parse import urljoin
 
 from mediagoblin import mg_globals as mgg
 from mediagoblin import messages
-from mediagoblin.db.models import MediaEntry, User, MediaComment
+from mediagoblin.db.models import MediaEntry, User, MediaComment, AccessToken
 from mediagoblin.tools.response import (
     redirect, render_404,
     render_user_banned, json_response)
@@ -75,7 +75,7 @@ def require_active_login(controller):
     return new_controller_func
 
 
-def user_has_privilege(privilege_name):
+def user_has_privilege(privilege_name, allow_admin=True):
     """
     Requires that a user have a particular privilege in order to access a page.
     In order to require that a user have multiple privileges, use this
@@ -86,14 +86,17 @@ def user_has_privilege(privilege_name):
                                         the privilege object. This object is
                                         the name of the privilege, as assigned
                                         in the Privilege.privilege_name column
+
+        :param allow_admin          If this is true then if the user is an admin
+                                    it will allow the user even if the user doesn't
+                                    have the privilage given in privilage_name.
     """
 
     def user_has_privilege_decorator(controller):
         @wraps(controller)
         @require_active_login
         def wrapper(request, *args, **kwargs):
-            user_id = request.user.id
-            if not request.user.has_privilege(privilege_name):
+            if not request.user.has_privilege(privilege_name, allow_admin):
                 raise Forbidden()
 
             return controller(request, *args, **kwargs)
@@ -370,7 +373,8 @@ def require_admin_or_moderator_login(controller):
     @wraps(controller)
     def new_controller_func(request, *args, **kwargs):
         if request.user and \
-            not request.user.has_privilege(u'admin',u'moderator'):
+            not (request.user.has_privilege(u'admin')
+                or request.user.has_privilege(u'moderator')):
 
             raise Forbidden()
         elif not request.user:
@@ -402,10 +406,10 @@ def oauth_required(controller):
 
         request_validator = GMGRequestValidator()
         resource_endpoint = ResourceEndpoint(request_validator)
-        valid, request = resource_endpoint.validate_protected_resource_request(
+        valid, r = resource_endpoint.validate_protected_resource_request(
                 uri=request.url,
                 http_method=request.method,
-                body=request.get_data(),
+                body=request.data,
                 headers=dict(request.headers),
                 )
 
@@ -413,6 +417,13 @@ def oauth_required(controller):
             error = "Invalid oauth prarameter."
             return json_response({"error": error}, status=400)
 
+        # Fill user if not already
+        token = authorization[u"oauth_token"]
+        access_token = AccessToken.query.filter_by(token=token).first()
+        if access_token is not None and request.user is None:
+            user_id = access_token.user
+            request.user = User.query.filter_by(id=user_id).first()
+
         return controller(request, *args, **kwargs)
 
     return wrapper

mediagoblin/edit/forms.py

@@ -15,10 +15,12 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import wtforms
+from jsonschema import Draft4Validator
 
 from mediagoblin.tools.text import tag_length_validator
 from mediagoblin.tools.translate import lazy_pass_to_ugettext as _
 from mediagoblin.tools.licenses import licenses_as_choices
+from mediagoblin.tools.metadata import DEFAULT_SCHEMA, DEFAULT_CHECKER
 from mediagoblin.auth.tools import normalize_user_or_email_field
 
 
@@ -38,7 +40,7 @@ class EditForm(wtforms.Form):
             "Separate tags by commas."))
     slug = wtforms.TextField(
         _('Slug'),
-        [wtforms.validators.Required(message=_("The slug can't be empty"))],
+        [wtforms.validators.InputRequired(message=_("The slug can't be empty"))],
         description=_(
             "The title part of this media's address. "
             "You usually don't need to change this."))
@@ -85,7 +87,7 @@ class EditAttachmentsForm(wtforms.Form):
 class EditCollectionForm(wtforms.Form):
     title = wtforms.TextField(
         _('Title'),
-        [wtforms.validators.Length(min=0, max=500), wtforms.validators.Required(message=_("The title can't be empty"))])
+        [wtforms.validators.Length(min=0, max=500), wtforms.validators.InputRequired(message=_("The title can't be empty"))])
     description = wtforms.TextAreaField(
         _('Description of this collection'),
         description=_("""You can use
@@ -93,7 +95,7 @@ class EditCollectionForm(wtforms.Form):
                       Markdown</a> for formatting."""))
     slug = wtforms.TextField(
         _('Slug'),
-        [wtforms.validators.Required(message=_("The slug can't be empty"))],
+        [wtforms.validators.InputRequired(message=_("The slug can't be empty"))],
         description=_(
             "The title part of this collection's address. "
             "You usually don't need to change this."))
@@ -102,12 +104,12 @@ class EditCollectionForm(wtforms.Form):
 class ChangePassForm(wtforms.Form):
     old_password = wtforms.PasswordField(
         _('Old password'),
-        [wtforms.validators.Required()],
+        [wtforms.validators.InputRequired()],
         description=_(
             "Enter your old password to prove you own this account."))
     new_password = wtforms.PasswordField(
         _('New password'),
-        [wtforms.validators.Required(),
+        [wtforms.validators.InputRequired(),
          wtforms.validators.Length(min=6, max=30)],
         id="password")
 
@@ -115,10 +117,45 @@ class ChangePassForm(wtforms.Form):
 class ChangeEmailForm(wtforms.Form):
     new_email = wtforms.TextField(
         _('New email address'),
-        [wtforms.validators.Required(),
+        [wtforms.validators.InputRequired(),
          normalize_user_or_email_field(allow_user=False)])
     password = wtforms.PasswordField(
         _('Password'),
-        [wtforms.validators.Required()],
+        [wtforms.validators.InputRequired()],
         description=_(
             "Enter your password to prove you own this account."))
+
+class MetaDataValidator(object):
+    """
+    Custom validator which runs form data in a MetaDataForm through a jsonschema
+    validator and passes errors recieved in jsonschema to wtforms.
+
+    :param  schema              The json schema to validate the data against. By
+                                default this uses the DEFAULT_SCHEMA from
+                                mediagoblin.tools.metadata.
+    :param  format_checker      The FormatChecker object that limits which types
+                                jsonschema can recognize. By default this uses
+                                DEFAULT_CHECKER from mediagoblin.tools.metadata.
+    """
+    def __init__(self, schema=DEFAULT_SCHEMA, format_checker=DEFAULT_CHECKER):
+        self.schema = schema
+        self.format_checker = format_checker
+
+    def __call__(self, form, field):
+        metadata_dict = {field.data:form.value.data}
+        validator = Draft4Validator(self.schema,
+                        format_checker=self.format_checker)
+        errors = [e.message
+            for e in validator.iter_errors(metadata_dict)]
+        if len(errors) >= 1:
+            raise wtforms.validators.ValidationError(
+                errors.pop())
+
+class MetaDataForm(wtforms.Form):
+    identifier = wtforms.TextField(_(u'Identifier'),[MetaDataValidator()])
+    value = wtforms.TextField(_(u'Value'))
+
+class EditMetaDataForm(wtforms.Form):
+    media_metadata = wtforms.FieldList(
+        wtforms.FormField(MetaDataForm, ""),
+    )

mediagoblin/edit/views.py

@@ -19,8 +19,10 @@ import six
 from datetime import datetime
 
 from itsdangerous import BadSignature
+from pyld import jsonld
 from werkzeug.exceptions import Forbidden
 from werkzeug.utils import secure_filename
+from jsonschema import ValidationError, Draft4Validator
 
 from mediagoblin import messages
 from mediagoblin import mg_globals
@@ -31,8 +33,11 @@ from mediagoblin.edit import forms
 from mediagoblin.edit.lib import may_edit_media
 from mediagoblin.decorators import (require_active_login, active_user_from_url,
                             get_media_entry_by_id, user_may_alter_collection,
-                            get_user_collection)
+                            get_user_collection, user_has_privilege,
+                            user_not_banned)
 from mediagoblin.tools.crypto import get_timed_signer_url
+from mediagoblin.tools.metadata import (compact_and_validate, DEFAULT_CHECKER,
+                                        DEFAULT_SCHEMA)
 from mediagoblin.tools.mail import email_debug_message
 from mediagoblin.tools.response import (render_to_response,
                                         redirect, redirect_obj, render_404)
@@ -434,3 +439,30 @@ def change_email(request):
         'mediagoblin/edit/change_email.html',
         {'form': form,
          'user': user})
+
+@user_has_privilege(u'admin')
+@require_active_login
+@get_media_entry_by_id
+def edit_metadata(request, media):
+    form = forms.EditMetaDataForm(request.form)
+    if request.method == "POST" and form.validate():
+        metadata_dict = dict([(row['identifier'],row['value'])
+                            for row in form.media_metadata.data])
+        json_ld_metadata = None
+        json_ld_metadata = compact_and_validate(metadata_dict)
+        media.media_metadata = json_ld_metadata
+        media.save()
+        return redirect_obj(request, media)      
+
+    if len(form.media_metadata) == 0:
+        for identifier, value in media.media_metadata.iteritems():
+            if identifier == "@context": continue
+            form.media_metadata.append_entry({
+                'identifier':identifier,
+                'value':value})
+
+    return render_to_response(
+        request,
+        'mediagoblin/edit/metadata.html',
+        {'form':form,
+         'media':media})

mediagoblin/federation/__init__.py

@@ -13,13 +13,3 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
-'''
-mediagoblin.webfinger_ provides an LRDD discovery service and
-a web host meta information file
-
-Links:
-- `LRDD Discovery Draft
-  <http://tools.ietf.org/html/draft-hammer-discovery-06>`_.
-- `RFC 6415 - Web Host Metadata
-  <http://tools.ietf.org/html/rfc6415>`_.
-'''

mediagoblin/federation/decorators.py

@@ -0,0 +1,49 @@
+# GNU MediaGoblin -- federated, autonomous media hosting
+# Copyright (C) 2011, 2012 MediaGoblin contributors.  See AUTHORS.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+from functools import wraps
+
+from mediagoblin.db.models import User
+from mediagoblin.decorators import require_active_login
+from mediagoblin.tools.response import json_response
+
+def user_has_privilege(privilege_name):
+    """
+    Requires that a user have a particular privilege in order to access a page.
+    In order to require that a user have multiple privileges, use this
+    decorator twice on the same view. This decorator also makes sure that the
+    user is not banned, or else it redirects them to the "You are Banned" page.
+
+        :param privilege_name       A unicode object that is that represents
+                                        the privilege object. This object is
+                                        the name of the privilege, as assigned
+                                        in the Privilege.privilege_name column
+    """
+
+    def user_has_privilege_decorator(controller):
+        @wraps(controller)
+        @require_active_login
+        def wrapper(request, *args, **kwargs):
+            if not request.user.has_privilege(privilege_name):
+                error = "User '{0}' needs '{1}' privilege".format(
+                    request.user.username,
+                    privilege_name
+                )
+                return json_response({"error": error}, status=403)
+
+            return controller(request, *args, **kwargs)
+
+        return wrapper
+    return user_has_privilege_decorator

mediagoblin/federation/routing.py

@@ -0,0 +1,79 @@
+# GNU MediaGoblin -- federated, autonomous media hosting
+# Copyright (C) 2011, 2012 MediaGoblin contributors.  See AUTHORS.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from mediagoblin.tools.routing import add_route
+
+# Add user profile
+add_route(
+    "mediagoblin.federation.user",
+    "/api/user/<string:username>/",
+    "mediagoblin.federation.views:user_endpoint"
+)
+
+add_route(
+    "mediagoblin.federation.user.profile",
+    "/api/user/<string:username>/profile",
+    "mediagoblin.federation.views:profile_endpoint"
+)
+
+# Inbox and Outbox (feed)
+add_route(
+    "mediagoblin.federation.feed",
+    "/api/user/<string:username>/feed",
+    "mediagoblin.federation.views:feed_endpoint"
+)
+
+add_route(
+    "mediagoblin.federation.user.uploads",
+    "/api/user/<string:username>/uploads",
+    "mediagoblin.federation.views:uploads_endpoint"
+)
+
+add_route(
+    "mediagoblin.federation.inbox",
+    "/api/user/<string:username>/inbox",
+    "mediagoblin.federation.views:feed_endpoint"
+)
+
+# object endpoints
+add_route(
+    "mediagoblin.federation.object",
+    "/api/<string:objectType>/<string:id>",
+    "mediagoblin.federation.views:object_endpoint"
+    )
+add_route(
+    "mediagoblin.federation.object.comments",
+    "/api/<string:objectType>/<string:id>/comments",
+    "mediagoblin.federation.views:object_comments"
+)
+
+add_route(
+    "mediagoblin.webfinger.well-known.host-meta",
+    "/.well-known/host-meta",
+    "mediagoblin.federation.views:host_meta"
+)
+
+add_route(
+    "mediagoblin.webfinger.well-known.host-meta.json",
+    "/.well-known/host-meta.json",
+    "mediagoblin.federation.views:host_meta"
+)
+
+add_route(
+    "mediagoblin.webfinger.whoami",
+    "/api/whoami",
+    "mediagoblin.federation.views:whoami"
+)

mediagoblin/federation/views.py

@@ -0,0 +1,469 @@
+# GNU MediaGoblin -- federated, autonomous media hosting
+# Copyright (C) 2011, 2012 MediaGoblin contributors.  See AUTHORS.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import json
+import io
+import mimetypes
+
+from werkzeug.datastructures import FileStorage
+
+from mediagoblin.decorators import oauth_required
+from mediagoblin.federation.decorators import user_has_privilege
+from mediagoblin.db.models import User, MediaEntry, MediaComment
+from mediagoblin.tools.response import redirect, json_response, json_error
+from mediagoblin.meddleware.csrf import csrf_exempt
+from mediagoblin.submit.lib import new_upload_entry, api_upload_request, \
+                                    api_add_to_feed
+
+# MediaTypes
+from mediagoblin.media_types.image import MEDIA_TYPE as IMAGE_MEDIA_TYPE
+
+# Getters
+def get_profile(request):
+    """
+    Gets the user's profile for the endpoint requested.
+
+    For example an endpoint which is /api/{username}/feed
+    as /api/cwebber/feed would get cwebber's profile. This
+    will return a tuple (username, user_profile). If no user
+    can be found then this function returns a (None, None).
+    """
+    username = request.matchdict["username"]
+    user = User.query.filter_by(username=username).first()
+
+    if user is None:
+        return None, None
+
+    return user, user.serialize(request)
+
+
+# Endpoints
+@oauth_required
+def profile_endpoint(request):
+    """ This is /api/user/<username>/profile - This will give profile info """
+    user, user_profile = get_profile(request)
+
+    if user is None:
+        username = request.matchdict["username"]
+        return json_error(
+            "No such 'user' with username '{0}'".format(username),
+            status=404
+        )
+
+    # user profiles are public so return information
+    return json_response(user_profile)
+
+@oauth_required
+def user_endpoint(request):
+    """ This is /api/user/<username> - This will get the user """
+    user, user_profile = get_profile(request)
+    
+    if user is None:
+        username = request.matchdict["username"]
+        return json_error(
+            "No such 'user' with username '{0}'".format(username),
+            status=404
+        )
+    
+    return json_response({
+        "nickname": user.username,
+        "updated": user.created.isoformat(),
+        "published": user.created.isoformat(),
+        "profile": user_profile,
+    })
+
+@oauth_required
+@csrf_exempt
+@user_has_privilege(u'uploader')
+def uploads_endpoint(request):
+    """ Endpoint for file uploads """
+    username = request.matchdict["username"]
+    requested_user = User.query.filter_by(username=username).first()
+
+    if requested_user is None:
+        return json_error("No such 'user' with id '{0}'".format(username), 404)
+
+    if request.method == "POST":
+        # Ensure that the user is only able to upload to their own
+        # upload endpoint.
+        if requested_user.id != request.user.id:
+            return json_error(
+                "Not able to post to another users feed.",
+                status=403
+            )
+
+        # Wrap the data in the werkzeug file wrapper
+        if "Content-Type" not in request.headers:
+            return json_error(
+                "Must supply 'Content-Type' header to upload media."
+            )
+
+        mimetype = request.headers["Content-Type"]
+        filename = mimetypes.guess_all_extensions(mimetype)
+        filename = 'unknown' + filename[0] if filename else filename
+        file_data = FileStorage(
+            stream=io.BytesIO(request.data),
+            filename=filename,
+            content_type=mimetype
+        )
+
+        # Find media manager
+        entry = new_upload_entry(request.user)
+        entry.media_type = IMAGE_MEDIA_TYPE
+        return api_upload_request(request, file_data, entry)
+
+    return json_error("Not yet implemented", 501)
+
+@oauth_required
+@csrf_exempt
+def feed_endpoint(request):
+    """ Handles the user's outbox - /api/user/<username>/feed """
+    username = request.matchdict["username"]
+    requested_user = User.query.filter_by(username=username).first()
+
+    # check if the user exists
+    if requested_user is None:
+        return json_error("No such 'user' with id '{0}'".format(username), 404)
+
+    if request.data:
+        data = json.loads(request.data)
+    else:
+        data = {"verb": None, "object": {}}
+
+
+    if request.method in ["POST", "PUT"]:
+        # Validate that the activity is valid
+        if "verb" not in data or "object" not in data:
+            return json_error("Invalid activity provided.")
+
+        # Check that the verb is valid
+        if data["verb"] not in ["post", "update"]:
+            return json_error("Verb not yet implemented", 501)
+
+        # We need to check that the user they're posting to is
+        # the person that they are.
+        if requested_user.id != request.user.id:
+            return json_error(
+                "Not able to post to another users feed.",
+                status=403
+            )
+
+        # Handle new posts
+        if data["verb"] == "post":
+            obj = data.get("object", None)
+            if obj is None:
+                return json_error("Could not find 'object' element.")
+
+            if obj.get("objectType", None) == "comment":
+                # post a comment
+                if not request.user.has_privilege(u'commenter'):
+                    return json_error(
+                        "Privilege 'commenter' required to comment.",
+                        status=403
+                    )
+
+                comment = MediaComment(author=request.user.id)
+                comment.unserialize(data["object"])
+                comment.save()
+                data = {
+                    "verb": "post",
+                    "object": comment.serialize(request)
+                }
+                return json_response(data)
+
+            elif obj.get("objectType", None) == "image":
+                # Posting an image to the feed
+                media_id = int(data["object"]["id"])
+                media = MediaEntry.query.filter_by(id=media_id).first()
+
+                if media is None:
+                    return json_response(
+                        "No such 'image' with id '{0}'".format(media_id),
+                        status=404
+                    )
+
+                if media.uploader != request.user.id:
+                    return json_error(
+                        "Privilege 'commenter' required to comment.",
+                        status=403
+                    )
+
+
+                if not media.unserialize(data["object"]):
+                    return json_error(
+                        "Invalid 'image' with id '{0}'".format(media_id)
+                    )
+
+                media.save()
+                api_add_to_feed(request, media)
+
+                return json_response({
+                    "verb": "post",
+                    "object": media.serialize(request)
+                })
+
+            elif obj.get("objectType", None) is None:
+                # They need to tell us what type of object they're giving us.
+                return json_error("No objectType specified.")
+            else:
+                # Oh no! We don't know about this type of object (yet)
+                object_type = obj.get("objectType", None)
+                return json_error(
+                    "Unknown object type '{0}'.".format(object_type)
+                )
+
+        # Updating existing objects
+        if data["verb"] == "update":
+            # Check we've got a valid object
+            obj = data.get("object", None)
+
+            if obj is None:
+                return json_error("Could not find 'object' element.")
+
+            if "objectType" not in obj:
+                return json_error("No objectType specified.")
+
+            if "id" not in obj:
+                return json_error("Object ID has not been specified.")
+
+            obj_id = obj["id"]
+
+            # Now try and find object
+            if obj["objectType"] == "comment":
+                if not request.user.has_privilege(u'commenter'):
+                    return json_error(
+                        "Privilege 'commenter' required to comment.",
+                        status=403
+                    )
+
+                comment = MediaComment.query.filter_by(id=obj_id).first()
+                if comment is None:
+                    return json_error(
+                        "No such 'comment' with id '{0}'.".format(obj_id)
+                    )
+
+                # Check that the person trying to update the comment is
+                # the author of the comment.
+                if comment.author != request.user.id:
+                    return json_error(
+                        "Only author of comment is able to update comment.",
+                        status=403
+                    )
+
+                if not comment.unserialize(data["object"]):
+                    return json_error(
+                        "Invalid 'comment' with id '{0}'".format(obj_id)
+                    )
+
+                comment.save()
+
+                activity = {
+                    "verb": "update",
+                    "object": comment.serialize(request),
+                }
+                return json_response(activity)
+
+            elif obj["objectType"] == "image":
+                image = MediaEntry.query.filter_by(id=obj_id).first()
+                if image is None:
+                    return json_error(
+                        "No such 'image' with the id '{0}'.".format(obj_id)
+                    )
+
+                # Check that the person trying to update the comment is
+                # the author of the comment.
+                if image.uploader != request.user.id:
+                    return json_error(
+                        "Only uploader of image is able to update image.",
+                        status=403
+                    )
+
+                if not image.unserialize(obj):
+                    return json_error(
+                        "Invalid 'image' with id '{0}'".format(obj_id)
+                    )
+                image.save()
+
+                activity = {
+                    "verb": "update",
+                    "object": image.serialize(request),
+                }
+                return json_response(activity)
+
+    elif request.method != "GET":
+        return json_error(
+            "Unsupported HTTP method {0}".format(request.method),
+            status=501
+        )
+
+    feed_url = request.urlgen(
+        "mediagoblin.federation.feed",
+        username=request.user.username,
+        qualified=True
+    )
+
+    feed = {
+        "displayName": "Activities by {user}@{host}".format(
+            user=request.user.username,
+            host=request.host
+        ),
+        "objectTypes": ["activity"],
+        "url": feed_url,
+        "links": {
+            "first": {
+                "href": feed_url,
+            },
+            "self": {
+                "href": request.url,
+            },
+            "prev": {
+                "href": feed_url,
+            },
+            "next": {
+                "href": feed_url,
+            }
+        },
+        "author": request.user.serialize(request),
+        "items": [],
+    }
+
+
+    # Look up all the media to put in the feed (this will be changed
+    # when we get real feeds/inboxes/outboxes/activites)
+    for media in MediaEntry.query.all():
+        item = {
+            "verb": "post",
+            "object": media.serialize(request),
+            "actor": media.get_uploader.serialize(request),
+            "content": "{0} posted a picture".format(request.user.username),
+            "id": media.id,
+        }
+        item["updated"] = item["object"]["updated"]
+        item["published"] = item["object"]["published"]
+        item["url"] = item["object"]["url"]
+        feed["items"].append(item)
+    feed["totalItems"] = len(feed["items"])
+
+    return json_response(feed)
+
+@oauth_required
+def object_endpoint(request):
+    """ Lookup for a object type """
+    object_type = request.matchdict["objectType"]
+    try:
+        object_id = int(request.matchdict["id"])
+    except ValueError:
+        error = "Invalid object ID '{0}' for '{1}'".format(
+            request.matchdict["id"],
+            object_type
+        )
+        return json_error(error)
+
+    if object_type not in ["image"]:
+        # not sure why this is 404, maybe ask evan. Maybe 400?
+        return json_error(
+            "Unknown type: {0}".format(object_type),
+            status=404
+        )
+
+    media = MediaEntry.query.filter_by(id=object_id).first()
+    if media is None:
+        return json_error(
+            "Can't find '{0}' with ID '{1}'".format(object_type, object_id),
+            status=404
+        )
+
+    return json_response(media.serialize(request))
+
+@oauth_required
+def object_comments(request):
+    """ Looks up for the comments on a object """
+    media = MediaEntry.query.filter_by(id=request.matchdict["id"]).first()
+    if media is None:
+        return json_error("Can't find '{0}' with ID '{1}'".format(
+            request.matchdict["objectType"],
+            request.matchdict["id"]
+        ), 404)
+
+    comments = response.serialize(request)
+    comments = comments.get("replies", {
+        "totalItems": 0,
+        "items": [],
+        "url": request.urlgen(
+            "mediagoblin.federation.object.comments",
+            objectType=media.objectType,
+            id=media.id,
+            qualified=True
+        )
+    })
+
+    comments["displayName"] = "Replies to {0}".format(comments["url"])
+    comments["links"] = {
+        "first": comments["url"],
+        "self": comments["url"],
+    }
+    return json_response(comments)
+
+##
+# Well known
+##
+def host_meta(request):
+    """ /.well-known/host-meta - provide URLs to resources """
+    links = []
+
+    links.append({
+        "ref": "registration_endpoint",
+        "href": request.urlgen(
+            "mediagoblin.oauth.client_register",
+            qualified=True
+        ),
+    })
+    links.append({
+        "ref": "http://apinamespace.org/oauth/request_token",
+        "href": request.urlgen(
+            "mediagoblin.oauth.request_token",
+            qualified=True
+        ),
+    })
+    links.append({
+        "ref": "http://apinamespace.org/oauth/authorize",
+        "href": request.urlgen(
+            "mediagoblin.oauth.authorize",
+            qualified=True
+        ),
+    })
+    links.append({
+        "ref": "http://apinamespace.org/oauth/access_token",
+        "href": request.urlgen(
+            "mediagoblin.oauth.access_token",
+            qualified=True
+        ),
+    })
+
+    return json_response({"links": links})
+
+def whoami(request):
+    """ /api/whoami - HTTP redirect to API profile """
+    if request.user is None:
+        return json_error("Not logged in.", status=401)
+
+    profile = request.urlgen(
+        "mediagoblin.federation.user.profile",
+        username=request.user.username,
+        qualified=True
+    )
+
+    return redirect(request, location=profile)

mediagoblin/gmg_commands/__init__.py

@@ -39,6 +39,10 @@ SUBCOMMAND_MAP = {
         'setup': 'mediagoblin.gmg_commands.users:changepw_parser_setup',
         'func': 'mediagoblin.gmg_commands.users:changepw',
         'help': 'Changes a user\'s password'},
+    'deleteuser': {
+        'setup': 'mediagoblin.gmg_commands.users:deleteuser_parser_setup',
+        'func': 'mediagoblin.gmg_commands.users:deleteuser',
+        'help': 'Deletes a user'},
     'dbupdate': {
         'setup': 'mediagoblin.gmg_commands.dbupdate:dbupdate_parse_setup',
         'func': 'mediagoblin.gmg_commands.dbupdate:dbupdate',
@@ -63,6 +67,10 @@ SUBCOMMAND_MAP = {
             'setup': 'mediagoblin.gmg_commands.serve:parser_setup',
             'func': 'mediagoblin.gmg_commands.serve:serve',
             'help': 'PasteScript replacement'},
+    'batchaddmedia': {
+        'setup': 'mediagoblin.gmg_commands.batchaddmedia:parser_setup',
+        'func': 'mediagoblin.gmg_commands.batchaddmedia:batchaddmedia',
+        'help': 'Add many media entries at once'},
     # 'theme': {
     #     'setup': 'mediagoblin.gmg_commands.theme:theme_parser_setup',
     #     'func': 'mediagoblin.gmg_commands.theme:theme',

mediagoblin/gmg_commands/batchaddmedia.py

@@ -0,0 +1,206 @@
+# GNU MediaGoblin -- federated, autonomous media hosting
+# Copyright (C) 2011, 2012 MediaGoblin contributors.  See AUTHORS.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import requests, codecs
+import csv
+from urlparse import urlparse
+
+from mediagoblin.gmg_commands import util as commands_util
+from mediagoblin.submit.lib import (
+    submit_media, get_upload_file_limits,
+    FileUploadLimit, UserUploadLimit, UserPastUploadLimit)
+from mediagoblin.tools.metadata import compact_and_validate
+from mediagoblin.tools.translate import pass_to_ugettext as _
+from jsonschema.exceptions import ValidationError
+
+
+def parser_setup(subparser):
+    subparser.description = """\
+This command allows the administrator to upload many media files at once."""
+    subparser.epilog = _(u"""For more information about how to properly run this
+script (and how to format the metadata csv file), read the MediaGoblin
+documentation page on command line uploading
+<http://docs.mediagoblin.org/siteadmin/commandline-upload.html>""")
+    subparser.add_argument(
+        'username',
+        help=_(u"Name of user these media entries belong to"))
+    subparser.add_argument(
+        'metadata_path',
+        help=_(
+u"""Path to the csv file containing metadata information."""))
+    subparser.add_argument(
+        '--celery',
+        action='store_true',
+        help=_(u"Don't process eagerly, pass off to celery"))
+
+
+def batchaddmedia(args):
+    # Run eagerly unless explicetly set not to
+    if not args.celery:
+        os.environ['CELERY_ALWAYS_EAGER'] = 'true'
+
+    app = commands_util.setup_app(args)
+
+    files_uploaded, files_attempted = 0, 0
+
+    # get the user
+    user = app.db.User.query.filter_by(username=args.username.lower()).first()
+    if user is None:
+        print _(u"Sorry, no user by username '{username}' exists".format(
+                    username=args.username))
+        return
+
+    upload_limit, max_file_size = get_upload_file_limits(user)
+    temp_files = []
+
+    if os.path.isfile(args.metadata_path):
+        metadata_path = args.metadata_path
+
+    else:
+        error = _(u'File at {path} not found, use -h flag for help'.format(
+                    path=args.metadata_path))
+        print error
+        return
+
+    abs_metadata_filename = os.path.abspath(metadata_path)
+    abs_metadata_dir = os.path.dirname(abs_metadata_filename)
+    upload_limit, max_file_size = get_upload_file_limits(user)
+
+    def maybe_unicodeify(some_string):
+        # this is kinda terrible
+        if some_string is None:
+            return None
+        else:
+            return unicode(some_string)
+
+    with codecs.open(
+            abs_metadata_filename, 'r', encoding='utf-8') as all_metadata:
+        contents = all_metadata.read()
+        media_metadata = parse_csv_file(contents)
+
+    for media_id, file_metadata in media_metadata.iteritems():
+        files_attempted += 1
+        # In case the metadata was not uploaded initialize an empty dictionary.
+        json_ld_metadata = compact_and_validate({})
+
+        # Get all metadata entries starting with 'media' as variables and then
+        # delete them because those are for internal use only.
+        original_location = file_metadata['location']
+
+        ### Pull the important media information for mediagoblin from the
+        ### metadata, if it is provided.
+        title = file_metadata.get('title') or file_metadata.get('dc:title')
+        description = (file_metadata.get('description') or
+            file_metadata.get('dc:description'))
+
+        license = file_metadata.get('license')
+        try:
+            json_ld_metadata = compact_and_validate(file_metadata)
+        except ValidationError, exc:
+            error = _(u"""Error with media '{media_id}' value '{error_path}': {error_msg}
+Metadata was not uploaded.""".format(
+                media_id=media_id,
+                error_path=exc.path[0],
+                error_msg=exc.message))
+            print error
+            continue
+
+        url = urlparse(original_location)
+        filename = url.path.split()[-1]
+
+        if url.scheme == 'http':
+            res = requests.get(url.geturl(), stream=True)
+            media_file = res.raw
+
+        elif url.scheme == '':
+            path = url.path
+            if os.path.isabs(path):
+                file_abs_path = os.path.abspath(path)
+            else:
+                file_path = os.path.join(abs_metadata_dir, path)
+                file_abs_path = os.path.abspath(file_path)
+            try:
+                media_file = file(file_abs_path, 'r')
+            except IOError:
+                print _(u"""\
+FAIL: Local file {filename} could not be accessed.
+{filename} will not be uploaded.""".format(filename=filename))
+                continue
+        try:
+            submit_media(
+                mg_app=app,
+                user=user,
+                submitted_file=media_file,
+                filename=filename,
+                title=maybe_unicodeify(title),
+                description=maybe_unicodeify(description),
+                license=maybe_unicodeify(license),
+                metadata=json_ld_metadata,
+                tags_string=u"",
+                upload_limit=upload_limit, max_file_size=max_file_size)
+            print _(u"""Successfully submitted {filename}!
+Be sure to look at the Media Processing Panel on your website to be sure it
+uploaded successfully.""".format(filename=filename))
+            files_uploaded += 1
+        except FileUploadLimit:
+            print _(
+u"FAIL: This file is larger than the upload limits for this site.")
+        except UserUploadLimit:
+            print _(
+"FAIL: This file will put this user past their upload limits.")
+        except UserPastUploadLimit:
+            print _("FAIL: This user is already past their upload limits.")
+    print _(
+"{files_uploaded} out of {files_attempted} files successfully submitted".format(
+        files_uploaded=files_uploaded,
+        files_attempted=files_attempted))
+
+
+def unicode_csv_reader(unicode_csv_data, dialect=csv.excel, **kwargs):
+    # csv.py doesn't do Unicode; encode temporarily as UTF-8:
+    csv_reader = csv.reader(utf_8_encoder(unicode_csv_data),
+                            dialect=dialect, **kwargs)
+    for row in csv_reader:
+        # decode UTF-8 back to Unicode, cell by cell:
+        yield [unicode(cell, 'utf-8') for cell in row]
+
+def utf_8_encoder(unicode_csv_data):
+    for line in unicode_csv_data:
+        yield line.encode('utf-8')
+
+def parse_csv_file(file_contents):
+    """
+    The helper function which converts the csv file into a dictionary where each
+    item's key is the provided value 'id' and each item's value is another
+    dictionary.
+    """
+    list_of_contents = file_contents.split('\n')
+    key, lines = (list_of_contents[0].split(','),
+                  list_of_contents[1:])
+    objects_dict = {}
+
+    # Build a dictionary
+    for index, line in enumerate(lines):
+        if line.isspace() or line == u'': continue
+        values = unicode_csv_reader([line]).next()
+        line_dict = dict([(key[i], val)
+            for i, val in enumerate(values)])
+        media_id = line_dict.get('id') or index
+        objects_dict[media_id] = (line_dict)
+
+    return objects_dict
+

mediagoblin/gmg_commands/deletemedia.py

@@ -15,19 +15,23 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 from __future__ import print_function
+import sys
 
 from mediagoblin.gmg_commands import util as commands_util
 
 
 def parser_setup(subparser):
     subparser.add_argument('media_ids',
-                           help='Comma separated list of media IDs to will be deleted.')
+                           help='Comma separated list of media IDs will be deleted.')
 
 
 def deletemedia(args):
     app = commands_util.setup_app(args)
 
-    media_ids = set(map(int, args.media_ids.split(',')))
+    media_ids = set([int(mid) for mid in args.media_ids.split(',') if mid.isdigit()])
+    if not media_ids:
+        print 'Can\'t find any valid media ID(s).'
+        sys.exit(1)
     found_medias = set()
     filter_ids = app.db.MediaEntry.id.in_(media_ids)
     medias = app.db.MediaEntry.query.filter(filter_ids).all()
@@ -38,3 +42,4 @@ def deletemedia(args):
     for media in media_ids - found_medias:
         print('Can\'t find a media with ID %d.' % media)
     print('Done.')
+    sys.exit(0)

mediagoblin/gmg_commands/users.py

@@ -38,7 +38,7 @@ def adduser(args):
     #TODO: Lets trust admins this do not validate Emails :)
     commands_util.setup_app(args)
 
-    args.username = commands_util.prompt_if_not_set(args.username, "Username:")
+    args.username = unicode(commands_util.prompt_if_not_set(args.username, "Username:"))
     args.password = commands_util.prompt_if_not_set(args.password, "Password:",True)
     args.email = commands_util.prompt_if_not_set(args.email, "Email:")
 
@@ -119,3 +119,23 @@ def changepw(args):
         print(u'Password successfully changed')
     else:
         print(u'The user doesn\'t exist')
+
+
+def deleteuser_parser_setup(subparser):
+    subparser.add_argument(
+        'username',
+        help="Username to delete")
+
+
+def deleteuser(args):
+    commands_util.setup_app(args)
+
+    db = mg_globals.database
+
+    user = db.User.query.filter_by(
+        username=unicode(args.username.lower())).one()
+    if user:
+        user.delete()
+        print('The user %s has been deleted' % args.username)
+    else:
+        print('The user %s doesn\'t exist' % args.username)

mediagoblin/init/celery/__init__.py

@@ -16,6 +16,7 @@
 
 import os
 import sys
+import datetime
 import logging
 
 import six
@@ -29,7 +30,9 @@ _log = logging.getLogger(__name__)
 
 MANDATORY_CELERY_IMPORTS = [
     'mediagoblin.processing.task',
-    'mediagoblin.notifications.task']
+    'mediagoblin.notifications.task',
+    'mediagoblin.submit.task',
+]
 
 DEFAULT_SETTINGS_MODULE = 'mediagoblin.init.celery.dummy_settings_module'
 
@@ -60,6 +63,18 @@ def get_celery_settings_dict(app_config, global_config,
         celery_settings['CELERY_ALWAYS_EAGER'] = True
         celery_settings['CELERY_EAGER_PROPAGATES_EXCEPTIONS'] = True
 
+    # Garbage collection periodic task
+    frequency = app_config.get('garbage_collection', 60)
+    if frequency:
+        frequency = int(frequency)
+        celery_settings['CELERYBEAT_SCHEDULE'] = {
+            'garbage-collection': {
+                'task': 'mediagoblin.submit.task.garbage_collection',
+                'schedule': datetime.timedelta(minutes=frequency),
+            }
+        }
+        celery_settings['BROKER_HEARTBEAT'] = 1
+
     return celery_settings