From 637b966ac20e448d17b310ccbf29389410d7cdf2 Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Mon, 29 Jul 2013 17:31:42 +0100 Subject: [PATCH 01/44] Adds seralize on user --- mediagoblin/db/models.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/mediagoblin/db/models.py b/mediagoblin/db/models.py index 4c9345fc..b96129ae 100644 --- a/mediagoblin/db/models.py +++ b/mediagoblin/db/models.py @@ -136,6 +136,16 @@ class User(Base, UserMixin): return UserBan.query.get(self.id) is not None + def serialize(self, request): + user = { + "preferredUsername": self.username, + "displayName": "{username}@{server}".format(username=self.username, server=request.url) + "objectType": "person", + "url": self.url, + "links": { + }, + } + class Client(Base): """ Model representing a client - Used for API Auth From d7b3805f2dde435e211560ba6500cc30780739eb Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Mon, 29 Jul 2013 21:53:08 +0100 Subject: [PATCH 02/44] Starts the user (profile) endpoint and lays groundwork for inbox and feed endpoint --- mediagoblin/db/models.py | 26 +++++++++++++++++- mediagoblin/decorators.py | 2 +- mediagoblin/federation/__init__.py | 0 mediagoblin/federation/routing.py | 43 ++++++++++++++++++++++++++++++ mediagoblin/federation/views.py | 42 +++++++++++++++++++++++++++++ 5 files changed, 111 insertions(+), 2 deletions(-) create mode 100644 mediagoblin/federation/__init__.py create mode 100644 mediagoblin/federation/routing.py create mode 100644 mediagoblin/federation/views.py diff --git a/mediagoblin/db/models.py b/mediagoblin/db/models.py index b96129ae..61a7f251 100644 --- a/mediagoblin/db/models.py +++ b/mediagoblin/db/models.py @@ -138,13 +138,37 @@ class User(Base, UserMixin): def serialize(self, request): user = { + "id": "acct:{0}@{1}".format(self.username, request.url), "preferredUsername": self.username, - "displayName": "{username}@{server}".format(username=self.username, server=request.url) + "displayName": "{0}@{1}".format(self.username, request.url), "objectType": "person", "url": self.url, + "summary": self.bio, "links": { + "self": { + "href": request.urlgen( + "mediagoblin.federation.profile", + username=self.username, + qualified=True + ), + }, + "activity-inbox": { + "href": request.urlgen( + "mediagoblin.federation.inbox", + username=self.username, + qualified=True + ) + }, + "activity-outbox": { + "href": request.urlgen( + "mediagoblin.federation.feed", + username=self.username, + qualified=True + ) + }, }, } + return user class Client(Base): """ diff --git a/mediagoblin/decorators.py b/mediagoblin/decorators.py index 8515d091..040a11fa 100644 --- a/mediagoblin/decorators.py +++ b/mediagoblin/decorators.py @@ -401,7 +401,7 @@ def oauth_required(controller): request_validator = GMGRequestValidator() resource_endpoint = ResourceEndpoint(request_validator) - valid, request = resource_endpoint.validate_protected_resource_request( + valid, r = resource_endpoint.validate_protected_resource_request( uri=request.url, http_method=request.method, body=request.get_data(), diff --git a/mediagoblin/federation/__init__.py b/mediagoblin/federation/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/mediagoblin/federation/routing.py b/mediagoblin/federation/routing.py new file mode 100644 index 00000000..9c3e0dff --- /dev/null +++ b/mediagoblin/federation/routing.py @@ -0,0 +1,43 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from mediagoblin.tools.routing import add_route + +# Add user profile +add_route( + "mediagoblin.federation.user", + "/api/user//", + "mediagoblin.federation.views:user" + ) + +add_route( + "mediagoblin.federation.profile", + "/api/user//profile", + "mediagoblin.federation.views:user" + ) + +# Inbox and Outbox (feed) +add_route( + "mediagoblin.federation.feed", + "/api/user//feed", + "mediagoblin.federation.views:feed" + ) + +add_route( + "mediagoblin.federation.inbox", + "/api/user//inbox", + "mediagoblin.federation.views:inbox" + ) diff --git a/mediagoblin/federation/views.py b/mediagoblin/federation/views.py new file mode 100644 index 00000000..337f28ed --- /dev/null +++ b/mediagoblin/federation/views.py @@ -0,0 +1,42 @@ +from mediagoblin.decorators import oauth_required +from mediagoblin.db.models import User +from mediagoblin.tools.response import json_response + +@oauth_required +def user(request): + """ Handles user response at /api/user// """ + user = request.matchdict["username"] + requested_user = User.query.filter_by(username=user) + + # check if the user exists + if requested_user is None: + error = "No such 'user' with id '{0}'".format(user) + return json_response({"error": error}, status=404) + + user = requested_user[0] + + # user profiles are public so return information + return json_response(user.serialize(request)) + +@oauth_required +def feed(request): + """ Handles the user's outbox - /api/user//feed """ + user = request.matchdict["username"] + requested_user = User.query.filter_by(username=user) + + # check if the user exists + if requested_user is None: + error = "No such 'user' with id '{0}'".format(user) + return json_response({"error": error}, status=404) + + user = request_user[0] + + # Now lookup the user's feed. + raise NotImplemented("Yet to implement looking up user's feed") + +@oauth_required +def inbox(request): + """ Handles the user's inbox - /api/user//inbox """ + pass + + raise NotImplemented("Yet to implement looking up user's inbox") From e590179ab61c873acfa291f93cba7cc412432a58 Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Wed, 14 Aug 2013 15:41:02 +0100 Subject: [PATCH 03/44] Adds migration on MediaEntry to add uuid --- mediagoblin/db/migrations.py | 1 - mediagoblin/db/models.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/mediagoblin/db/migrations.py b/mediagoblin/db/migrations.py index 59aec4d2..85b1eded 100644 --- a/mediagoblin/db/migrations.py +++ b/mediagoblin/db/migrations.py @@ -466,7 +466,6 @@ def create_oauth1_tables(db): db.commit() - @RegisterMigration(15, MIGRATIONS) def wants_notifications(db): """Add a wants_notifications field to User model""" diff --git a/mediagoblin/db/models.py b/mediagoblin/db/models.py index 61a7f251..02392792 100644 --- a/mediagoblin/db/models.py +++ b/mediagoblin/db/models.py @@ -20,6 +20,7 @@ TODO: indexes on foreignkeys, where useful. import logging import datetime +import base64 from sqlalchemy import Column, Integer, Unicode, UnicodeText, DateTime, \ Boolean, ForeignKey, UniqueConstraint, PrimaryKeyConstraint, \ @@ -422,7 +423,6 @@ class MediaEntry(Base, MediaEntryMixin): # pass through commit=False/True in kwargs super(MediaEntry, self).delete(**kwargs) - class FileKeynames(Base): """ keywords for various places. From 3015d31a79dcb12b641d70f36eda6536f36a04c8 Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Wed, 14 Aug 2013 15:49:12 +0100 Subject: [PATCH 04/44] Adds the federation routing --- mediagoblin/routing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediagoblin/routing.py b/mediagoblin/routing.py index 9f2584d3..7d9d01ee 100644 --- a/mediagoblin/routing.py +++ b/mediagoblin/routing.py @@ -39,8 +39,8 @@ def get_url_map(): import mediagoblin.listings.routing import mediagoblin.notifications.routing import mediagoblin.oauth.routing + import mediagoblin.federation.routing - for route in PluginManager().get_routes(): add_route(*route) From 2b7b9de32e53d34635059afc571ac1a318e41071 Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Wed, 14 Aug 2013 16:16:49 +0100 Subject: [PATCH 05/44] Make sure new media has a new uuid added on --- mediagoblin/db/migrations.py | 13 ++++++------- mediagoblin/db/models.py | 4 ++++ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/mediagoblin/db/migrations.py b/mediagoblin/db/migrations.py index 85b1eded..242d72d9 100644 --- a/mediagoblin/db/migrations.py +++ b/mediagoblin/db/migrations.py @@ -25,12 +25,11 @@ from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.sql import and_ from migrate.changeset.constraint import UniqueConstraint - from mediagoblin.db.extratypes import JSONEncoded, MutationDict from mediagoblin.db.migration_tools import ( RegisterMigration, inspect_table, replace_table_hack) -from mediagoblin.db.models import (MediaEntry, Collection, MediaComment, User, - Privilege) +from mediagoblin.db.models import (MediaEntry, Collection, MediaComment, User, + create_uuid, Privilege) from mediagoblin.db.extratypes import JSONEncoded, MutationDict MIGRATIONS = {} @@ -659,8 +658,8 @@ def create_moderation_tables(db): # admin, an active user or an inactive user ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ for admin_user in admin_users_ids: admin_user_id = admin_user['id'] - for privilege_id in [admin_privilege_id, uploader_privilege_id, - reporter_privilege_id, commenter_privilege_id, + for privilege_id in [admin_privilege_id, uploader_privilege_id, + reporter_privilege_id, commenter_privilege_id, active_privilege_id]: db.execute(user_privilege_assoc.insert().values( core__privilege_id=admin_user_id, @@ -668,7 +667,7 @@ def create_moderation_tables(db): for active_user in active_users_ids: active_user_id = active_user['id'] - for privilege_id in [uploader_privilege_id, reporter_privilege_id, + for privilege_id in [uploader_privilege_id, reporter_privilege_id, commenter_privilege_id, active_privilege_id]: db.execute(user_privilege_assoc.insert().values( core__privilege_id=active_user_id, @@ -676,7 +675,7 @@ def create_moderation_tables(db): for inactive_user in inactive_users_ids: inactive_user_id = inactive_user['id'] - for privilege_id in [uploader_privilege_id, reporter_privilege_id, + for privilege_id in [uploader_privilege_id, reporter_privilege_id, commenter_privilege_id]: db.execute(user_privilege_assoc.insert().values( core__privilege_id=inactive_user_id, diff --git a/mediagoblin/db/models.py b/mediagoblin/db/models.py index 02392792..e1b37aa0 100644 --- a/mediagoblin/db/models.py +++ b/mediagoblin/db/models.py @@ -237,6 +237,10 @@ class NonceTimestamp(Base): timestamp = Column(DateTime, nullable=False, primary_key=True) +def create_uuid(): + """ Creates a new uuid which is suitable for use in a URL """ + return base64.urlsafe_b64encode(uuid.uuid4().bytes).strip("=") + class MediaEntry(Base, MediaEntryMixin): """ TODO: Consider fetching the media_files using join From 5a2056f7386371bd84b9481380235bb0f06ca149 Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Wed, 14 Aug 2013 17:00:26 +0100 Subject: [PATCH 06/44] Adds endpoint /api/image/ so that you can now view an image endpoint --- mediagoblin/federation/routing.py | 7 ++++++ mediagoblin/federation/views.py | 42 ++++++++++++++++++++++++++++--- 2 files changed, 46 insertions(+), 3 deletions(-) diff --git a/mediagoblin/federation/routing.py b/mediagoblin/federation/routing.py index 9c3e0dff..9c870588 100644 --- a/mediagoblin/federation/routing.py +++ b/mediagoblin/federation/routing.py @@ -41,3 +41,10 @@ add_route( "/api/user//inbox", "mediagoblin.federation.views:inbox" ) + +# object endpoints +add_route( + "mediagoblin.federation.object", + "/api//", + "mediagoblin.federation.views:object" + ) diff --git a/mediagoblin/federation/views.py b/mediagoblin/federation/views.py index 337f28ed..de9084d0 100644 --- a/mediagoblin/federation/views.py +++ b/mediagoblin/federation/views.py @@ -1,5 +1,5 @@ from mediagoblin.decorators import oauth_required -from mediagoblin.db.models import User +from mediagoblin.db.models import User, MediaEntry from mediagoblin.tools.response import json_response @oauth_required @@ -37,6 +37,42 @@ def feed(request): @oauth_required def inbox(request): """ Handles the user's inbox - /api/user//inbox """ - pass - raise NotImplemented("Yet to implement looking up user's inbox") + +def image_object(request, media): + """ Return image object - /api/image/ """ + author = media.get_uploader + url = request.urlgen( + "mediagoblin.user_pages.media_home", + user=author.username, + media=media.slug, + qualified=True + ) + + context = { + "author": author.serialize(request), + "displayName": media.title, + "objectType": "image", + "url": url, + } + + return json_response(context) + +@oauth_required +def object(request): + """ Lookup for a object type """ + objectType = request.matchdict["objectType"] + uuid = request.matchdict["uuid"] + if objectType not in ["image"]: + error = "Unknown type: {0}".format(objectType) + # not sure why this is 404, maybe ask evan. Maybe 400? + return json_response({"error": error}, status=404) + + media = MediaEntry.query.filter_by(uuid=uuid).first() + if media is None: + # no media found with that uuid + error = "Can't find a {0} with ID = {1}".format(objectType, uuid) + return json_response({"error": error}, status=404) + + if objectType == "image": + return image_object(request, media) From bdde87a4b3a584a2dde5803b1a069496aee73daf Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Wed, 14 Aug 2013 17:51:36 +0100 Subject: [PATCH 07/44] Changes serialization to .serialize method on object - MediaEntry --- mediagoblin/db/models.py | 43 +++++++++++++++++++++++++++++++++ mediagoblin/federation/views.py | 22 +---------------- 2 files changed, 44 insertions(+), 21 deletions(-) diff --git a/mediagoblin/db/models.py b/mediagoblin/db/models.py index e1b37aa0..404aaa94 100644 --- a/mediagoblin/db/models.py +++ b/mediagoblin/db/models.py @@ -427,6 +427,37 @@ class MediaEntry(Base, MediaEntryMixin): # pass through commit=False/True in kwargs super(MediaEntry, self).delete(**kwargs) + @property + def objectType(self): + """ Converts media_type to pump-like type - don't use internally """ + return self.media_type.split(".")[-1] + + def serialize(self, request): + """ Unserialize MediaEntry to object """ + author = self.get_uploader + url = request.urlgen( + "mediagoblin.user_pages.media_home", + user=author.username, + media=self.slug, + qualified=True + ) + + id = request.urlgen( + "mediagoblin.federation.object", + objectType=self.objectType, + uuid=self.uuid, + qualified=True + ) + + context = { + "id": id, + "author": author.serialize(request), + "displayName": self.title, + "objectType": self.objectType, + "url": url, + } + return context + class FileKeynames(Base): """ keywords for various places. @@ -573,6 +604,18 @@ class MediaComment(Base, MediaCommentMixin): cascade="all, delete-orphan")) + def serialize(self, request): + """ Unserialize to python dictionary for API """ + media = MediaEntry.query.filter_by(self.media_entry).first() + context = { + "objectType": "comment", + "content": self.content, + "inReplyTo": media.unserialize(request), + "author": self.get_author.unserialize(request) + } + + return context + class Collection(Base, CollectionMixin): """An 'album' or 'set' of media by a user. diff --git a/mediagoblin/federation/views.py b/mediagoblin/federation/views.py index de9084d0..3fe5b3b5 100644 --- a/mediagoblin/federation/views.py +++ b/mediagoblin/federation/views.py @@ -39,25 +39,6 @@ def inbox(request): """ Handles the user's inbox - /api/user//inbox """ raise NotImplemented("Yet to implement looking up user's inbox") -def image_object(request, media): - """ Return image object - /api/image/ """ - author = media.get_uploader - url = request.urlgen( - "mediagoblin.user_pages.media_home", - user=author.username, - media=media.slug, - qualified=True - ) - - context = { - "author": author.serialize(request), - "displayName": media.title, - "objectType": "image", - "url": url, - } - - return json_response(context) - @oauth_required def object(request): """ Lookup for a object type """ @@ -74,5 +55,4 @@ def object(request): error = "Can't find a {0} with ID = {1}".format(objectType, uuid) return json_response({"error": error}, status=404) - if objectType == "image": - return image_object(request, media) + return json_response(media.serialize(request)) From a840d2a848743fe36fc800557de7bb7a6e693b57 Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Wed, 14 Aug 2013 18:23:52 +0100 Subject: [PATCH 08/44] Adds comments for the MediaEntry api --- mediagoblin/db/models.py | 21 ++++++++++++++++----- mediagoblin/federation/views.py | 4 ++-- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/mediagoblin/db/models.py b/mediagoblin/db/models.py index 404aaa94..6d6b2032 100644 --- a/mediagoblin/db/models.py +++ b/mediagoblin/db/models.py @@ -432,7 +432,7 @@ class MediaEntry(Base, MediaEntryMixin): """ Converts media_type to pump-like type - don't use internally """ return self.media_type.split(".")[-1] - def serialize(self, request): + def serialize(self, request, show_comments=True): """ Unserialize MediaEntry to object """ author = self.get_uploader url = request.urlgen( @@ -456,6 +456,17 @@ class MediaEntry(Base, MediaEntryMixin): "objectType": self.objectType, "url": url, } + + if show_comments: + comments = [comment.serialize(request) for comment in self.get_comments()] + total = len(comments) + if total > 0: + # we only want to include replies if there are any. + context["replies"] = { + "totalItems": total, + "items": comments + } + return context class FileKeynames(Base): @@ -603,15 +614,15 @@ class MediaComment(Base, MediaCommentMixin): lazy="dynamic", cascade="all, delete-orphan")) - def serialize(self, request): """ Unserialize to python dictionary for API """ - media = MediaEntry.query.filter_by(self.media_entry).first() + media = MediaEntry.query.filter_by(id=self.media_entry).first() + author = self.get_author context = { "objectType": "comment", "content": self.content, - "inReplyTo": media.unserialize(request), - "author": self.get_author.unserialize(request) + "inReplyTo": media.serialize(request, show_comments=False), + "author": author.serialize(request) } return context diff --git a/mediagoblin/federation/views.py b/mediagoblin/federation/views.py index 3fe5b3b5..b3f63db5 100644 --- a/mediagoblin/federation/views.py +++ b/mediagoblin/federation/views.py @@ -2,7 +2,7 @@ from mediagoblin.decorators import oauth_required from mediagoblin.db.models import User, MediaEntry from mediagoblin.tools.response import json_response -@oauth_required +#@oauth_required def user(request): """ Handles user response at /api/user// """ user = request.matchdict["username"] @@ -39,7 +39,7 @@ def inbox(request): """ Handles the user's inbox - /api/user//inbox """ raise NotImplemented("Yet to implement looking up user's inbox") -@oauth_required +#@oauth_required def object(request): """ Lookup for a object type """ objectType = request.matchdict["objectType"] From c8bd2542d7b8face6033884fccfb898be1d12989 Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Wed, 14 Aug 2013 18:32:27 +0100 Subject: [PATCH 09/44] Fixes where User id in API would return url rather than host --- mediagoblin/db/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediagoblin/db/models.py b/mediagoblin/db/models.py index 6d6b2032..281c09d9 100644 --- a/mediagoblin/db/models.py +++ b/mediagoblin/db/models.py @@ -139,7 +139,7 @@ class User(Base, UserMixin): def serialize(self, request): user = { - "id": "acct:{0}@{1}".format(self.username, request.url), + "id": "acct:{0}@{1}".format(self.username, request.host), "preferredUsername": self.username, "displayName": "{0}@{1}".format(self.username, request.url), "objectType": "person", From 5b014a08661f718bd92971e71d173a0ea4b62c40 Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Wed, 14 Aug 2013 19:58:01 +0100 Subject: [PATCH 10/44] Add image URL's (thumb & full) --- mediagoblin/db/mixin.py | 11 +++++++++++ mediagoblin/db/models.py | 6 ++++++ mediagoblin/federation/views.py | 4 ++-- 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/mediagoblin/db/mixin.py b/mediagoblin/db/mixin.py index 3d96ba34..87f4383a 100644 --- a/mediagoblin/db/mixin.py +++ b/mediagoblin/db/mixin.py @@ -202,6 +202,17 @@ class MediaEntryMixin(GenerateSlugMixin): thumb_url = mg_globals.app.staticdirector(manager[u'default_thumb']) return thumb_url + @property + def original_url(self): + """ Returns the URL for the original image + will return self.thumb_url if original url doesn't exist""" + if u"original" not in self.media_files: + return self.thumb_url + + return mg_globals.app.public_store.file_url( + self.media_files[u"original"] + ) + @cached_property def media_manager(self): """Returns the MEDIA_MANAGER of the media's media_type diff --git a/mediagoblin/db/models.py b/mediagoblin/db/models.py index 281c09d9..925f0d24 100644 --- a/mediagoblin/db/models.py +++ b/mediagoblin/db/models.py @@ -455,6 +455,12 @@ class MediaEntry(Base, MediaEntryMixin): "displayName": self.title, "objectType": self.objectType, "url": url, + "image": { + "url": request.host_url + self.thumb_url[1:], + }, + "fullImage":{ + "url": request.host_url + self.original_url[1:], + } } if show_comments: diff --git a/mediagoblin/federation/views.py b/mediagoblin/federation/views.py index b3f63db5..3fe5b3b5 100644 --- a/mediagoblin/federation/views.py +++ b/mediagoblin/federation/views.py @@ -2,7 +2,7 @@ from mediagoblin.decorators import oauth_required from mediagoblin.db.models import User, MediaEntry from mediagoblin.tools.response import json_response -#@oauth_required +@oauth_required def user(request): """ Handles user response at /api/user// """ user = request.matchdict["username"] @@ -39,7 +39,7 @@ def inbox(request): """ Handles the user's inbox - /api/user//inbox """ raise NotImplemented("Yet to implement looking up user's inbox") -#@oauth_required +@oauth_required def object(request): """ Lookup for a object type """ objectType = request.matchdict["objectType"] From d461fbe5cb20ed56c3c1e3696464c3d323e5b4b0 Mon Sep 17 00:00:00 2001 From: xray7224 Date: Mon, 2 Sep 2013 16:22:24 +0100 Subject: [PATCH 11/44] Use the the slug as the UUID instead of a newly generated UUID --- mediagoblin/db/models.py | 7 +------ mediagoblin/federation/views.py | 2 +- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/mediagoblin/db/models.py b/mediagoblin/db/models.py index 925f0d24..ca4efdd1 100644 --- a/mediagoblin/db/models.py +++ b/mediagoblin/db/models.py @@ -236,11 +236,6 @@ class NonceTimestamp(Base): nonce = Column(Unicode, nullable=False, primary_key=True) timestamp = Column(DateTime, nullable=False, primary_key=True) - -def create_uuid(): - """ Creates a new uuid which is suitable for use in a URL """ - return base64.urlsafe_b64encode(uuid.uuid4().bytes).strip("=") - class MediaEntry(Base, MediaEntryMixin): """ TODO: Consider fetching the media_files using join @@ -445,7 +440,7 @@ class MediaEntry(Base, MediaEntryMixin): id = request.urlgen( "mediagoblin.federation.object", objectType=self.objectType, - uuid=self.uuid, + uuid=self.slug, qualified=True ) diff --git a/mediagoblin/federation/views.py b/mediagoblin/federation/views.py index 3fe5b3b5..01082942 100644 --- a/mediagoblin/federation/views.py +++ b/mediagoblin/federation/views.py @@ -49,7 +49,7 @@ def object(request): # not sure why this is 404, maybe ask evan. Maybe 400? return json_response({"error": error}, status=404) - media = MediaEntry.query.filter_by(uuid=uuid).first() + media = MediaEntry.query.filter_by(slug=uuid).first() if media is None: # no media found with that uuid error = "Can't find a {0} with ID = {1}".format(objectType, uuid) From 37f070b06786c20f320231bc467b35ccab6270dc Mon Sep 17 00:00:00 2001 From: xray7224 Date: Mon, 2 Sep 2013 16:23:40 +0100 Subject: [PATCH 12/44] Fixes problem where full URL was being used inplace of host --- mediagoblin/db/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediagoblin/db/models.py b/mediagoblin/db/models.py index ca4efdd1..91efc0b6 100644 --- a/mediagoblin/db/models.py +++ b/mediagoblin/db/models.py @@ -141,7 +141,7 @@ class User(Base, UserMixin): user = { "id": "acct:{0}@{1}".format(self.username, request.host), "preferredUsername": self.username, - "displayName": "{0}@{1}".format(self.username, request.url), + "displayName": "{0}@{1}".format(self.username, request.host), "objectType": "person", "url": self.url, "summary": self.bio, From 98596dd072597c5d9c474e882f57407817d049f5 Mon Sep 17 00:00:00 2001 From: xray7224 Date: Mon, 2 Sep 2013 19:25:24 +0100 Subject: [PATCH 13/44] Support for the comments endpoint --- mediagoblin/db/models.py | 12 ++++++++++-- mediagoblin/federation/routing.py | 5 +++++ mediagoblin/federation/views.py | 26 ++++++++++++++++++++++++-- 3 files changed, 39 insertions(+), 4 deletions(-) diff --git a/mediagoblin/db/models.py b/mediagoblin/db/models.py index 91efc0b6..4377f60f 100644 --- a/mediagoblin/db/models.py +++ b/mediagoblin/db/models.py @@ -455,7 +455,9 @@ class MediaEntry(Base, MediaEntryMixin): }, "fullImage":{ "url": request.host_url + self.original_url[1:], - } + }, + "published": self.created.isoformat(), + "updated": self.created.isoformat(), } if show_comments: @@ -465,7 +467,13 @@ class MediaEntry(Base, MediaEntryMixin): # we only want to include replies if there are any. context["replies"] = { "totalItems": total, - "items": comments + "items": comments, + "url": request.urlgen( + "mediagoblin.federation.object.comments", + objectType=self.objectType, + uuid=self.slug, + qualified=True + ), } return context diff --git a/mediagoblin/federation/routing.py b/mediagoblin/federation/routing.py index 9c870588..16184866 100644 --- a/mediagoblin/federation/routing.py +++ b/mediagoblin/federation/routing.py @@ -48,3 +48,8 @@ add_route( "/api//", "mediagoblin.federation.views:object" ) +add_route( + "mediagoblin.federation.object.comments", + "/api///comments", + "mediagoblin.federation.views:object_comments" + ) diff --git a/mediagoblin/federation/views.py b/mediagoblin/federation/views.py index 01082942..ab67e457 100644 --- a/mediagoblin/federation/views.py +++ b/mediagoblin/federation/views.py @@ -39,8 +39,8 @@ def inbox(request): """ Handles the user's inbox - /api/user//inbox """ raise NotImplemented("Yet to implement looking up user's inbox") -@oauth_required -def object(request): +#@oauth_required +def object(request, raw_obj=False): """ Lookup for a object type """ objectType = request.matchdict["objectType"] uuid = request.matchdict["uuid"] @@ -55,4 +55,26 @@ def object(request): error = "Can't find a {0} with ID = {1}".format(objectType, uuid) return json_response({"error": error}, status=404) + if raw_obj: + return media + return json_response(media.serialize(request)) + +def object_comments(request): + """ Looks up for the comments on a object """ + media = object(request, raw_obj=True) + response = media + if isinstance(response, MediaEntry): + comments = response.serialize(request) + comments = comments.get("replies", { + "totalItems": 0, + "items": [], + "url": request.urlgen( + "mediagoblin.federation.object.comments", + objectType=media.objectType, + uuid=media.slug, + qualified=True) + }) + response = json_response(comments) + + return response From a5682e89602ddc266d05c760a319d7647755f0b4 Mon Sep 17 00:00:00 2001 From: xray7224 Date: Tue, 3 Sep 2013 17:17:07 +0100 Subject: [PATCH 14/44] Support some webfinger API's and real profile and /api/user// --- mediagoblin/db/models.py | 2 +- mediagoblin/federation/routing.py | 16 +++++++- mediagoblin/federation/views.py | 61 +++++++++++++++++++++++++++++-- mediagoblin/oauth/routing.py | 8 ++-- 4 files changed, 76 insertions(+), 11 deletions(-) diff --git a/mediagoblin/db/models.py b/mediagoblin/db/models.py index 4377f60f..4f5182d6 100644 --- a/mediagoblin/db/models.py +++ b/mediagoblin/db/models.py @@ -148,7 +148,7 @@ class User(Base, UserMixin): "links": { "self": { "href": request.urlgen( - "mediagoblin.federation.profile", + "mediagoblin.federation.user.profile", username=self.username, qualified=True ), diff --git a/mediagoblin/federation/routing.py b/mediagoblin/federation/routing.py index 16184866..daf60d00 100644 --- a/mediagoblin/federation/routing.py +++ b/mediagoblin/federation/routing.py @@ -24,9 +24,9 @@ add_route( ) add_route( - "mediagoblin.federation.profile", + "mediagoblin.federation.user.profile", "/api/user//profile", - "mediagoblin.federation.views:user" + "mediagoblin.federation.views:profile" ) # Inbox and Outbox (feed) @@ -53,3 +53,15 @@ add_route( "/api///comments", "mediagoblin.federation.views:object_comments" ) + +add_route( + "mediagoblin.webfinger.well-known.host-meta", + "/.well-known/host-meta", + "mediagoblin.federation.views:host_meta" + ) + +add_route( + "mediagoblin.webfinger.whoami", + "/api/whoami", + "mediagoblin.federation.views:whoami" + ) diff --git a/mediagoblin/federation/views.py b/mediagoblin/federation/views.py index ab67e457..85bf1540 100644 --- a/mediagoblin/federation/views.py +++ b/mediagoblin/federation/views.py @@ -1,10 +1,10 @@ from mediagoblin.decorators import oauth_required from mediagoblin.db.models import User, MediaEntry -from mediagoblin.tools.response import json_response +from mediagoblin.tools.response import redirect, json_response -@oauth_required -def user(request): - """ Handles user response at /api/user// """ +#@oauth_required +def profile(request, raw=False): + """ This is /api/user//profile - This will give profile info """ user = request.matchdict["username"] requested_user = User.query.filter_by(username=user) @@ -15,9 +15,24 @@ def user(request): user = requested_user[0] + if raw: + return (user, user.serialize(request)) + # user profiles are public so return information return json_response(user.serialize(request)) +def user(request): + """ This is /api/user/ - This will get the user """ + user, user_profile = profile(request, raw=True) + data = { + "nickname": user.username, + "updated": user.created.isoformat(), + "published": user.created.isoformat(), + "profile": user_profile + } + + return json_response(data) + @oauth_required def feed(request): """ Handles the user's outbox - /api/user//feed """ @@ -78,3 +93,41 @@ def object_comments(request): response = json_response(comments) return response + + +## +# Well known +## +def host_meta(request): + """ This is /.well-known/host-meta - provides URL's to resources on server """ + links = [] + + # Client registration links + links.append({ + "ref": "registration_endpoint", + "href": request.urlgen("mediagoblin.oauth.client_register", qualified=True), + }) + links.append({ + "ref": "http://apinamespace.org/oauth/request_token", + "href": request.urlgen("mediagoblin.oauth.request_token", qualified=True), + }) + links.append({ + "ref": "http://apinamespace.org/oauth/authorize", + "href": request.urlgen("mediagoblin.oauth.authorize", qualified=True), + }) + links.append({ + "ref": "http://apinamespace.org/oauth/access_token", + "href": request.urlgen("mediagoblin.oauth.access_token", qualified=True), + }) + + return json_response(links) + +def whoami(request): + """ This is /api/whoami - This is a HTTP redirect to api profile """ + profile = request.urlgen( + "mediagoblin.federation.user.profile", + username=request.user.username, + qualified=True + ) + + return redirect(request, location=profile) diff --git a/mediagoblin/oauth/routing.py b/mediagoblin/oauth/routing.py index e45077bb..7f2aa11d 100644 --- a/mediagoblin/oauth/routing.py +++ b/mediagoblin/oauth/routing.py @@ -18,25 +18,25 @@ from mediagoblin.tools.routing import add_route # client registration & oauth add_route( - "mediagoblin.oauth", + "mediagoblin.oauth.client_register", "/api/client/register", "mediagoblin.oauth.views:client_register" ) add_route( - "mediagoblin.oauth", + "mediagoblin.oauth.request_token", "/oauth/request_token", "mediagoblin.oauth.views:request_token" ) add_route( - "mediagoblin.oauth", + "mediagoblin.oauth.authorize", "/oauth/authorize", "mediagoblin.oauth.views:authorize", ) add_route( - "mediagoblin.oauth", + "mediagoblin.oauth.access_token", "/oauth/access_token", "mediagoblin.oauth.views:access_token" ) From 1829765537be3c057b8b6f6d01c5a3964536f6e0 Mon Sep 17 00:00:00 2001 From: xray7224 Date: Tue, 3 Sep 2013 17:24:24 +0100 Subject: [PATCH 15/44] Add .json url for host-meta and fix host-meta problem of not having 'links' --- mediagoblin/federation/routing.py | 6 ++++++ mediagoblin/federation/views.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/mediagoblin/federation/routing.py b/mediagoblin/federation/routing.py index daf60d00..be6451e0 100644 --- a/mediagoblin/federation/routing.py +++ b/mediagoblin/federation/routing.py @@ -60,6 +60,12 @@ add_route( "mediagoblin.federation.views:host_meta" ) +add_route( + "mediagoblin.webfinger.well-known.host-meta.json", + "/.well-known/host-meta.json", + "mediagoblin.federation.views:host_meta" + ) + add_route( "mediagoblin.webfinger.whoami", "/api/whoami", diff --git a/mediagoblin/federation/views.py b/mediagoblin/federation/views.py index 85bf1540..c84956c3 100644 --- a/mediagoblin/federation/views.py +++ b/mediagoblin/federation/views.py @@ -120,7 +120,7 @@ def host_meta(request): "href": request.urlgen("mediagoblin.oauth.access_token", qualified=True), }) - return json_response(links) + return json_response({"links": links}) def whoami(request): """ This is /api/whoami - This is a HTTP redirect to api profile """ From d6dce0f7d8cda420c632334251203392055fb716 Mon Sep 17 00:00:00 2001 From: xray7224 Date: Tue, 3 Sep 2013 18:50:33 +0100 Subject: [PATCH 16/44] Remove old webfinger support --- mediagoblin/routing.py | 1 - mediagoblin/webfinger/__init__.py | 25 ------- mediagoblin/webfinger/routing.py | 23 ------ mediagoblin/webfinger/views.py | 117 ------------------------------ 4 files changed, 166 deletions(-) delete mode 100644 mediagoblin/webfinger/__init__.py delete mode 100644 mediagoblin/webfinger/routing.py delete mode 100644 mediagoblin/webfinger/views.py diff --git a/mediagoblin/routing.py b/mediagoblin/routing.py index 7d9d01ee..3ec1dba0 100644 --- a/mediagoblin/routing.py +++ b/mediagoblin/routing.py @@ -35,7 +35,6 @@ def get_url_map(): import mediagoblin.submit.routing import mediagoblin.user_pages.routing import mediagoblin.edit.routing - import mediagoblin.webfinger.routing import mediagoblin.listings.routing import mediagoblin.notifications.routing import mediagoblin.oauth.routing diff --git a/mediagoblin/webfinger/__init__.py b/mediagoblin/webfinger/__init__.py deleted file mode 100644 index 126e6ea2..00000000 --- a/mediagoblin/webfinger/__init__.py +++ /dev/null @@ -1,25 +0,0 @@ -# GNU MediaGoblin -- federated, autonomous media hosting -# Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . -''' -mediagoblin.webfinger_ provides an LRDD discovery service and -a web host meta information file - -Links: -- `LRDD Discovery Draft - `_. -- `RFC 6415 - Web Host Metadata - `_. -''' diff --git a/mediagoblin/webfinger/routing.py b/mediagoblin/webfinger/routing.py deleted file mode 100644 index eb10509f..00000000 --- a/mediagoblin/webfinger/routing.py +++ /dev/null @@ -1,23 +0,0 @@ -# GNU MediaGoblin -- federated, autonomous media hosting -# Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -from mediagoblin.tools.routing import add_route - -add_route('mediagoblin.webfinger.host_meta', '/.well-known/host-meta', - 'mediagoblin.webfinger.views:host_meta') - -add_route('mediagoblin.webfinger.xrd', '/webfinger/xrd', - 'mediagoblin.webfinger.views:xrd') diff --git a/mediagoblin/webfinger/views.py b/mediagoblin/webfinger/views.py deleted file mode 100644 index 97fc3ef7..00000000 --- a/mediagoblin/webfinger/views.py +++ /dev/null @@ -1,117 +0,0 @@ -# GNU MediaGoblin -- federated, autonomous media hosting -# Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . -''' -For references, see docstring in mediagoblin/webfinger/__init__.py -''' - -import re - -from urlparse import urlparse - -from mediagoblin.tools.response import render_to_response, render_404 - -def host_meta(request): - ''' - Webfinger host-meta - ''' - - placeholder = 'MG_LRDD_PLACEHOLDER' - - lrdd_title = 'GNU MediaGoblin - User lookup' - - lrdd_template = request.urlgen( - 'mediagoblin.webfinger.xrd', - uri=placeholder, - qualified=True) - - return render_to_response( - request, - 'mediagoblin/webfinger/host-meta.xml', - {'request': request, - 'lrdd_template': lrdd_template, - 'lrdd_title': lrdd_title, - 'placeholder': placeholder}) - -MATCH_SCHEME_PATTERN = re.compile(r'^acct:') - -def xrd(request): - ''' - Find user data based on a webfinger URI - ''' - param_uri = request.GET.get('uri') - - if not param_uri: - return render_404(request) - - ''' - :py:module:`urlparse` does not recognize usernames in URIs of the - form ``acct:user@example.org`` or ``user@example.org``. - ''' - if not MATCH_SCHEME_PATTERN.search(param_uri): - # Assume the URI is in the form ``user@example.org`` - uri = 'acct://' + param_uri - else: - # Assumes the URI looks like ``acct:user@example.org - uri = MATCH_SCHEME_PATTERN.sub( - 'acct://', param_uri) - - parsed = urlparse(uri) - - xrd_subject = param_uri - - # TODO: Verify that the user exists - # Q: Does webfinger support error handling in this case? - # Returning 404 seems intuitive, need to check. - if parsed.username: - # The user object - # TODO: Fetch from database instead of using the MockUser - user = MockUser() - user.username = parsed.username - - xrd_links = [ - {'attrs': { - 'rel': 'http://microformats.org/profile/hcard', - 'href': request.urlgen( - 'mediagoblin.user_pages.user_home', - user=user.username, - qualified=True)}}, - {'attrs': { - 'rel': 'http://schemas.google.com/g/2010#updates-from', - 'href': request.urlgen( - 'mediagoblin.user_pages.atom_feed', - user=user.username, - qualified=True)}}] - - xrd_alias = request.urlgen( - 'mediagoblin.user_pages.user_home', - user=user.username, - qualified=True) - - return render_to_response( - request, - 'mediagoblin/webfinger/xrd.xml', - {'request': request, - 'subject': xrd_subject, - 'alias': xrd_alias, - 'links': xrd_links }) - else: - return render_404(request) - -class MockUser(object): - ''' - TEMPORARY user object - ''' - username = None From c434fc31c9b4195dabfb9c323bf13aca3337e5f9 Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Wed, 4 Sep 2013 16:32:49 +0100 Subject: [PATCH 17/44] Add static pump_io to API and fix problem where null appeared in profile --- mediagoblin/db/models.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/mediagoblin/db/models.py b/mediagoblin/db/models.py index 4f5182d6..cc22450f 100644 --- a/mediagoblin/db/models.py +++ b/mediagoblin/db/models.py @@ -143,8 +143,10 @@ class User(Base, UserMixin): "preferredUsername": self.username, "displayName": "{0}@{1}".format(self.username, request.host), "objectType": "person", - "url": self.url, - "summary": self.bio, + "pump_io": { + "shared": False, + "followed": False, + }, "links": { "self": { "href": request.urlgen( @@ -169,6 +171,12 @@ class User(Base, UserMixin): }, }, } + + if self.bio: + user.update({"summary": self.bio}) + if self.url: + user.update({"url": self.url}) + return user class Client(Base): @@ -458,6 +466,9 @@ class MediaEntry(Base, MediaEntryMixin): }, "published": self.created.isoformat(), "updated": self.created.isoformat(), + "pump_io": { + "shared": False, + }, } if show_comments: From c894b4246a1211e7d8e63e7c51b8d7095482a10c Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Wed, 4 Sep 2013 19:34:29 +0100 Subject: [PATCH 18/44] Add basic comment support and flesh out some other endpoints --- mediagoblin/db/models.py | 9 +--- mediagoblin/decorators.py | 9 +++- mediagoblin/federation/routing.py | 2 +- mediagoblin/federation/views.py | 82 +++++++++++++++++++++++++++++-- 4 files changed, 89 insertions(+), 13 deletions(-) diff --git a/mediagoblin/db/models.py b/mediagoblin/db/models.py index cc22450f..215e7552 100644 --- a/mediagoblin/db/models.py +++ b/mediagoblin/db/models.py @@ -445,15 +445,8 @@ class MediaEntry(Base, MediaEntryMixin): qualified=True ) - id = request.urlgen( - "mediagoblin.federation.object", - objectType=self.objectType, - uuid=self.slug, - qualified=True - ) - context = { - "id": id, + "id": self.id, "author": author.serialize(request), "displayName": self.title, "objectType": self.objectType, diff --git a/mediagoblin/decorators.py b/mediagoblin/decorators.py index 040a11fa..5cba6fee 100644 --- a/mediagoblin/decorators.py +++ b/mediagoblin/decorators.py @@ -22,7 +22,7 @@ from oauthlib.oauth1 import ResourceEndpoint from mediagoblin import mg_globals as mgg from mediagoblin import messages -from mediagoblin.db.models import MediaEntry, User, MediaComment +from mediagoblin.db.models import MediaEntry, User, MediaComment, AccessToken from mediagoblin.tools.response import ( redirect, render_404, render_user_banned, json_response) @@ -412,6 +412,13 @@ def oauth_required(controller): error = "Invalid oauth prarameter." return json_response({"error": error}, status=400) + # Fill user if not already + token = authorization[u"oauth_token"] + access_token = AccessToken.query.filter_by(token=token).first() + if access_token is not None and request.user is None: + user_id = access_token.user + request.user = User.query.filter_by(id=user_id).first() + return controller(request, *args, **kwargs) return wrapper diff --git a/mediagoblin/federation/routing.py b/mediagoblin/federation/routing.py index be6451e0..12306766 100644 --- a/mediagoblin/federation/routing.py +++ b/mediagoblin/federation/routing.py @@ -39,7 +39,7 @@ add_route( add_route( "mediagoblin.federation.inbox", "/api/user//inbox", - "mediagoblin.federation.views:inbox" + "mediagoblin.federation.views:feed" ) # object endpoints diff --git a/mediagoblin/federation/views.py b/mediagoblin/federation/views.py index c84956c3..cff3d499 100644 --- a/mediagoblin/federation/views.py +++ b/mediagoblin/federation/views.py @@ -1,6 +1,9 @@ +import json + from mediagoblin.decorators import oauth_required -from mediagoblin.db.models import User, MediaEntry +from mediagoblin.db.models import User, MediaEntry, MediaComment from mediagoblin.tools.response import redirect, json_response +from mediagoblin.meddleware.csrf import csrf_exempt #@oauth_required def profile(request, raw=False): @@ -34,8 +37,10 @@ def user(request): return json_response(data) @oauth_required +@csrf_exempt def feed(request): """ Handles the user's outbox - /api/user//feed """ + print request.user user = request.matchdict["username"] requested_user = User.query.filter_by(username=user) @@ -44,10 +49,76 @@ def feed(request): error = "No such 'user' with id '{0}'".format(user) return json_response({"error": error}, status=404) - user = request_user[0] + user = requested_user[0] + + if request.method == "POST": + data = json.loads(request.data) + obj = data.get("object", None) + if obj is None: + error = {"error": "Could not find 'object' element."} + return json_response(error, status=400) + + if obj.get("objectType", None) == "comment": + # post a comment + media = int(data["object"]["inReplyTo"]["id"]) + author = request.user + comment = MediaComment( + media_entry=media, + author=request.user.id, + content=data["object"]["content"] + ) + comment.save() + elif obj.get("objectType", None) is None: + error = {"error": "No objectType specified."} + return json_response(error, status=400) + else: + error = {"error": "Unknown object type '{0}'.".format(obj.get("objectType", None))} + return json_response(error, status=400) + + feed_url = request.urlgen( + "mediagoblin.federation.feed", + username=user.username, + qualified=True + ) + + feed = { + "displayName": "Activities by {0}@{1}".format(user.username, request.host), + "objectTypes": ["activity"], + "url": feed_url, + "links": { + "first": { + "href": feed_url, + }, + "self": { + "href": request.url, + }, + "prev": { + "href": feed_url, + }, + "next": { + "href": feed_url, + } + }, + "author": user.serialize(request), + "items": [], + } + # Now lookup the user's feed. - raise NotImplemented("Yet to implement looking up user's feed") + for media in MediaEntry.query.all(): + feed["items"].append({ + "verb": "post", + "object": media.serialize(request), + "actor": user.serialize(request), + "content": "{0} posted a picture".format(user.username), + "id": 1, + }) + feed["items"][-1]["updated"] = feed["items"][-1]["object"]["updated"] + feed["items"][-1]["published"] = feed["items"][-1]["object"]["published"] + feed["items"][-1]["url"] = feed["items"][-1]["object"]["url"] + feed["totalItems"] = len(feed["items"]) + + return json_response(feed) @oauth_required def inbox(request): @@ -90,6 +161,11 @@ def object_comments(request): uuid=media.slug, qualified=True) }) + comments["displayName"] = "Replies to {0}".format(comments["url"]) + comments["links"] = { + "first": comments["url"], + "self": comments["url"], + } response = json_response(comments) return response From d4a21d7e746dc1284f44137d1c3e45b7b5ee09c0 Mon Sep 17 00:00:00 2001 From: xray7224 Date: Tue, 24 Sep 2013 20:30:51 +0100 Subject: [PATCH 19/44] Add basic upload image capabilities --- mediagoblin/decorators.py | 2 +- mediagoblin/federation/routing.py | 6 ++++ mediagoblin/federation/views.py | 56 ++++++++++++++++++++++++++++++- mediagoblin/oauth/oauth.py | 2 +- mediagoblin/tools/request.py | 2 +- 5 files changed, 64 insertions(+), 4 deletions(-) diff --git a/mediagoblin/decorators.py b/mediagoblin/decorators.py index 5cba6fee..90edf96b 100644 --- a/mediagoblin/decorators.py +++ b/mediagoblin/decorators.py @@ -404,7 +404,7 @@ def oauth_required(controller): valid, r = resource_endpoint.validate_protected_resource_request( uri=request.url, http_method=request.method, - body=request.get_data(), + body=request.data, headers=dict(request.headers), ) diff --git a/mediagoblin/federation/routing.py b/mediagoblin/federation/routing.py index 12306766..b9cc4e2e 100644 --- a/mediagoblin/federation/routing.py +++ b/mediagoblin/federation/routing.py @@ -36,6 +36,12 @@ add_route( "mediagoblin.federation.views:feed" ) +add_route( + "mediagoblin.federation.user.uploads", + "/api/user//uploads", + "mediagoblin.federation.views:uploads" + ) + add_route( "mediagoblin.federation.inbox", "/api/user//inbox", diff --git a/mediagoblin/federation/views.py b/mediagoblin/federation/views.py index cff3d499..851a3f39 100644 --- a/mediagoblin/federation/views.py +++ b/mediagoblin/federation/views.py @@ -1,9 +1,16 @@ import json +import io +from werkzeug.datastructures import FileStorage + +from mediagoblin.media_types import sniff_media from mediagoblin.decorators import oauth_required from mediagoblin.db.models import User, MediaEntry, MediaComment from mediagoblin.tools.response import redirect, json_response from mediagoblin.meddleware.csrf import csrf_exempt +from mediagoblin.notifications import add_comment_subscription +from mediagoblin.submit.lib import (new_upload_entry, prepare_queue_task, + run_process_media) #@oauth_required def profile(request, raw=False): @@ -36,11 +43,58 @@ def user(request): return json_response(data) +#@oauth_required +@csrf_exempt +def uploads(request): + """ This is the endpoint which uploads can be sent ot - /api/user//uploads """ + user = request.matchdict["username"] + requested_user = User.query.filter_by(username=user) + + if requested_user is None: + error = "No such 'user' with id '{0}'".format(user) + return json_response({"error": error}, status=404) + + request.user = requested_user[0] + if request.method == "POST": + # Wrap the data in the werkzeug file wrapper + file_data = FileStorage( + stream=io.BytesIO(request.data), + filename=request.form.get("qqfile", "unknown.jpg"), + content_type=request.headers.get("Content-Type", "application/octal-stream") + ) + + # Use the same kind of method from mediagoblin/submit/views:submit_start + media_type, media_manager = sniff_media(file_data) + entry = new_upload_entry(request.user) + entry.media_type = unicode(media_type) + entry.title = u"Hello ^_^" + entry.description = u"" + entry.license = None + + entry.generate_slug() + + queue_file = prepare_queue_task(request.app, entry, file_data.filename) + with queue_file: + queue_file.write(request.data) + + entry.save() + + # run the processing + feed_url = request.urlgen( + 'mediagoblin.user_pages.atom_feed', + qualified=True, user=request.user.username) + + run_process_media(entry, feed_url) + add_comment_subscription(request.user, entry) + + return json_response(entry.serialize(request)) + + return json_response({"error": "Not yet implemented"}, status=400) + @oauth_required @csrf_exempt def feed(request): """ Handles the user's outbox - /api/user//feed """ - print request.user user = request.matchdict["username"] requested_user = User.query.filter_by(username=user) diff --git a/mediagoblin/oauth/oauth.py b/mediagoblin/oauth/oauth.py index 8229c47d..d9defa4b 100644 --- a/mediagoblin/oauth/oauth.py +++ b/mediagoblin/oauth/oauth.py @@ -126,7 +126,7 @@ class GMGRequest(Request): """ kwargs["uri"] = kwargs.get("uri", request.url) kwargs["http_method"] = kwargs.get("http_method", request.method) - kwargs["body"] = kwargs.get("body", request.get_data()) + kwargs["body"] = kwargs.get("body", request.data) kwargs["headers"] = kwargs.get("headers", dict(request.headers)) super(GMGRequest, self).__init__(*args, **kwargs) diff --git a/mediagoblin/tools/request.py b/mediagoblin/tools/request.py index d4739039..2de0b32f 100644 --- a/mediagoblin/tools/request.py +++ b/mediagoblin/tools/request.py @@ -45,7 +45,7 @@ def setup_user_in_request(request): def decode_request(request): """ Decodes a request based on MIME-Type """ - data = request.get_data() + data = request.data if request.content_type == json_encoded: data = json.loads(data) From 62dc7d3e6c50eda84074e43ff93e2945c3b81e1b Mon Sep 17 00:00:00 2001 From: xray7224 Date: Sat, 28 Sep 2013 15:22:18 -0400 Subject: [PATCH 20/44] Add some more code to work better with image uploads --- mediagoblin/federation/views.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/mediagoblin/federation/views.py b/mediagoblin/federation/views.py index 851a3f39..f19edef7 100644 --- a/mediagoblin/federation/views.py +++ b/mediagoblin/federation/views.py @@ -59,7 +59,7 @@ def uploads(request): # Wrap the data in the werkzeug file wrapper file_data = FileStorage( stream=io.BytesIO(request.data), - filename=request.form.get("qqfile", "unknown.jpg"), + filename=request.args.get("qqfile", "unknown.jpg"), content_type=request.headers.get("Content-Type", "application/octal-stream") ) @@ -67,8 +67,8 @@ def uploads(request): media_type, media_manager = sniff_media(file_data) entry = new_upload_entry(request.user) entry.media_type = unicode(media_type) - entry.title = u"Hello ^_^" - entry.description = u"" + entry.title = unicode(request.args.get("title", "Hello ^_^")) + entry.description = unicode(request.args.get("description", "")) entry.license = None entry.generate_slug() @@ -122,10 +122,25 @@ def feed(request): content=data["object"]["content"] ) comment.save() + elif obj.get("objectType", None) == "image": + # Posting an image to the feed + # NB: This is currently just handing the image back until we have an + # to send the image to the actual feed + + media_id = int(data["object"]["id"]) + media = MediaEntry.query.filter_by(id=media_id) + if media is None: + error = "No such 'image' with id '{0}'".format(id=media_id) + return json_response(error, status=404) + media = media[0] + return json_response(media.serialize(request)) + elif obj.get("objectType", None) is None: + # They need to tell us what type of object they're giving us. error = {"error": "No objectType specified."} return json_response(error, status=400) else: + # Oh no! We don't know about this type of object (yet) error = {"error": "Unknown object type '{0}'.".format(obj.get("objectType", None))} return json_response(error, status=400) From 3c3fa5e7bfd60fc80215c2c96ccf3c68be7b424e Mon Sep 17 00:00:00 2001 From: xray7224 Date: Sat, 28 Sep 2013 16:37:37 -0400 Subject: [PATCH 21/44] Fix some problems with comments and image posting --- mediagoblin/federation/views.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/mediagoblin/federation/views.py b/mediagoblin/federation/views.py index f19edef7..4add17d9 100644 --- a/mediagoblin/federation/views.py +++ b/mediagoblin/federation/views.py @@ -103,7 +103,7 @@ def feed(request): error = "No such 'user' with id '{0}'".format(user) return json_response({"error": error}, status=404) - user = requested_user[0] + request.user = requested_user[0] if request.method == "POST": data = json.loads(request.data) @@ -122,6 +122,8 @@ def feed(request): content=data["object"]["content"] ) comment.save() + data = {"verb": "post", "object": comment.serialize(request)} + return json_response(data) elif obj.get("objectType", None) == "image": # Posting an image to the feed # NB: This is currently just handing the image back until we have an @@ -146,12 +148,12 @@ def feed(request): feed_url = request.urlgen( "mediagoblin.federation.feed", - username=user.username, + username=request.user.username, qualified=True ) feed = { - "displayName": "Activities by {0}@{1}".format(user.username, request.host), + "displayName": "Activities by {0}@{1}".format(request.user.username, request.host), "objectTypes": ["activity"], "url": feed_url, "links": { @@ -168,7 +170,7 @@ def feed(request): "href": feed_url, } }, - "author": user.serialize(request), + "author": request.user.serialize(request), "items": [], } @@ -178,8 +180,8 @@ def feed(request): feed["items"].append({ "verb": "post", "object": media.serialize(request), - "actor": user.serialize(request), - "content": "{0} posted a picture".format(user.username), + "actor": request.user.serialize(request), + "content": "{0} posted a picture".format(request.user.username), "id": 1, }) feed["items"][-1]["updated"] = feed["items"][-1]["object"]["updated"] From 7810817caf73bcc0dcdfe1cec249c86e3e77c148 Mon Sep 17 00:00:00 2001 From: xray7224 Date: Thu, 10 Oct 2013 20:19:58 +0100 Subject: [PATCH 22/44] Refactors api uploading to media managers --- mediagoblin/db/models.py | 22 ++++++------ mediagoblin/federation/views.py | 44 ++++++++--------------- mediagoblin/media_types/image/__init__.py | 28 ++++++++++++++- 3 files changed, 51 insertions(+), 43 deletions(-) diff --git a/mediagoblin/db/models.py b/mediagoblin/db/models.py index 215e7552..cc5d0afa 100644 --- a/mediagoblin/db/models.py +++ b/mediagoblin/db/models.py @@ -467,18 +467,16 @@ class MediaEntry(Base, MediaEntryMixin): if show_comments: comments = [comment.serialize(request) for comment in self.get_comments()] total = len(comments) - if total > 0: - # we only want to include replies if there are any. - context["replies"] = { - "totalItems": total, - "items": comments, - "url": request.urlgen( - "mediagoblin.federation.object.comments", - objectType=self.objectType, - uuid=self.slug, - qualified=True - ), - } + context["replies"] = { + "totalItems": total, + "items": comments, + "url": request.urlgen( + "mediagoblin.federation.object.comments", + objectType=self.objectType, + uuid=self.slug, + qualified=True + ), + } return context diff --git a/mediagoblin/federation/views.py b/mediagoblin/federation/views.py index 4add17d9..bdc93d9b 100644 --- a/mediagoblin/federation/views.py +++ b/mediagoblin/federation/views.py @@ -46,7 +46,7 @@ def user(request): #@oauth_required @csrf_exempt def uploads(request): - """ This is the endpoint which uploads can be sent ot - /api/user//uploads """ + """ This is the endpoint which uploads can be sent to - /api/user//uploads """ user = request.matchdict["username"] requested_user = User.query.filter_by(username=user) @@ -58,40 +58,22 @@ def uploads(request): if request.method == "POST": # Wrap the data in the werkzeug file wrapper file_data = FileStorage( - stream=io.BytesIO(request.data), - filename=request.args.get("qqfile", "unknown.jpg"), - content_type=request.headers.get("Content-Type", "application/octal-stream") - ) - - # Use the same kind of method from mediagoblin/submit/views:submit_start + stream=io.BytesIO(request.data), + filename=request.args.get("qqfile", "unknown.jpg"), + content_type=request.headers.get("Content-Type", "application/octal-stream") + ) + + # Find media manager media_type, media_manager = sniff_media(file_data) entry = new_upload_entry(request.user) - entry.media_type = unicode(media_type) - entry.title = unicode(request.args.get("title", "Hello ^_^")) - entry.description = unicode(request.args.get("description", "")) - entry.license = None - - entry.generate_slug() - - queue_file = prepare_queue_task(request.app, entry, file_data.filename) - with queue_file: - queue_file.write(request.data) - - entry.save() - - # run the processing - feed_url = request.urlgen( - 'mediagoblin.user_pages.atom_feed', - qualified=True, user=request.user.username) - - run_process_media(entry, feed_url) - add_comment_subscription(request.user, entry) - - return json_response(entry.serialize(request)) + if hasattr(media_manager, "api_upload_request"): + return media_manager.api_upload_request(request, file_data, entry) + else: + return json_response({"error": "Not yet implemented"}, status=400) return json_response({"error": "Not yet implemented"}, status=400) -@oauth_required +#@oauth_required @csrf_exempt def feed(request): """ Handles the user's outbox - /api/user//feed """ @@ -124,6 +106,7 @@ def feed(request): comment.save() data = {"verb": "post", "object": comment.serialize(request)} return json_response(data) + elif obj.get("objectType", None) == "image": # Posting an image to the feed # NB: This is currently just handing the image back until we have an @@ -146,6 +129,7 @@ def feed(request): error = {"error": "Unknown object type '{0}'.".format(obj.get("objectType", None))} return json_response(error, status=400) + feed_url = request.urlgen( "mediagoblin.federation.feed", username=request.user.username, diff --git a/mediagoblin/media_types/image/__init__.py b/mediagoblin/media_types/image/__init__.py index 06e0f08f..0a77b0ce 100644 --- a/mediagoblin/media_types/image/__init__.py +++ b/mediagoblin/media_types/image/__init__.py @@ -19,7 +19,9 @@ import logging from mediagoblin.media_types import MediaManagerBase from mediagoblin.media_types.image.processing import sniff_handler, \ ImageProcessingManager - +from mediagoblin.tools.response import json_response +from mediagoblin.submit.lib import prepare_queue_task, run_process_media +from mediagoblin.notifications import add_comment_subscription _log = logging.getLogger(__name__) @@ -56,6 +58,30 @@ class ImageMediaManager(MediaManagerBase): except (KeyError, ValueError): return None + @staticmethod + def api_upload_request(request, file_data, entry): + """ This handles a image upload request """ + # Use the same kind of method from mediagoblin/submit/views:submit_start + entry.media_type = unicode(MEDIA_TYPE) + entry.title = unicode(request.args.get("title", file_data.filename)) + entry.description = unicode(request.args.get("description", "")) + entry.license = request.args.get("license", "") # not part of the standard API + + entry.generate_slug() + + queue_file = prepare_queue_task(request.app, entry, file_data.filename) + with queue_file: + queue_file.write(request.data) + + entry.save() + + feed_url = request.urlgen( + 'mediagoblin.user_pages.atom_feed', + qualified=True, user=request.user.username) + + run_process_media(entry, feed_url) + add_comment_subscription(request.user, entry) + return json_response(entry.serialize(request)) def get_media_type_and_manager(ext): if ext in ACCEPTED_EXTENSIONS: From c64fc16b13c14d65544aa1185a457c5dee48b169 Mon Sep 17 00:00:00 2001 From: xray7224 Date: Thu, 14 Nov 2013 17:27:06 +0000 Subject: [PATCH 23/44] Clean up code (after linting) --- mediagoblin/federation/views.py | 37 ++++++++++++++++----------------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/mediagoblin/federation/views.py b/mediagoblin/federation/views.py index bdc93d9b..0dc32e74 100644 --- a/mediagoblin/federation/views.py +++ b/mediagoblin/federation/views.py @@ -1,23 +1,21 @@ import json import io -from werkzeug.datastructures import FileStorage +from werkzeug.datastructures import FileStorage from mediagoblin.media_types import sniff_media from mediagoblin.decorators import oauth_required from mediagoblin.db.models import User, MediaEntry, MediaComment from mediagoblin.tools.response import redirect, json_response from mediagoblin.meddleware.csrf import csrf_exempt -from mediagoblin.notifications import add_comment_subscription -from mediagoblin.submit.lib import (new_upload_entry, prepare_queue_task, - run_process_media) +from mediagoblin.submit.lib import new_upload_entry #@oauth_required def profile(request, raw=False): """ This is /api/user//profile - This will give profile info """ user = request.matchdict["username"] requested_user = User.query.filter_by(username=user) - + # check if the user exists if requested_user is None: error = "No such 'user' with id '{0}'".format(user) @@ -46,7 +44,7 @@ def user(request): #@oauth_required @csrf_exempt def uploads(request): - """ This is the endpoint which uploads can be sent to - /api/user//uploads """ + """ Endpoint for file uploads """ user = request.matchdict["username"] requested_user = User.query.filter_by(username=user) @@ -93,11 +91,10 @@ def feed(request): if obj is None: error = {"error": "Could not find 'object' element."} return json_response(error, status=400) - + if obj.get("objectType", None) == "comment": # post a comment media = int(data["object"]["inReplyTo"]["id"]) - author = request.user comment = MediaComment( media_entry=media, author=request.user.id, @@ -111,7 +108,7 @@ def feed(request): # Posting an image to the feed # NB: This is currently just handing the image back until we have an # to send the image to the actual feed - + media_id = int(data["object"]["id"]) media = MediaEntry.query.filter_by(id=media_id) if media is None: @@ -126,7 +123,11 @@ def feed(request): return json_response(error, status=400) else: # Oh no! We don't know about this type of object (yet) - error = {"error": "Unknown object type '{0}'.".format(obj.get("objectType", None))} + error_message = "Unknown object type '{0}'.".format( + obj.get("objectType", None) + ) + + error = {"error": error_message} return json_response(error, status=400) @@ -137,7 +138,10 @@ def feed(request): ) feed = { - "displayName": "Activities by {0}@{1}".format(request.user.username, request.host), + "displayName": "Activities by {user}@{host}".format( + user=request.user.username, + host=request.host + ), "objectTypes": ["activity"], "url": feed_url, "links": { @@ -157,7 +161,7 @@ def feed(request): "author": request.user.serialize(request), "items": [], } - + # Now lookup the user's feed. for media in MediaEntry.query.all(): @@ -176,18 +180,13 @@ def feed(request): return json_response(feed) @oauth_required -def inbox(request): - """ Handles the user's inbox - /api/user//inbox """ - raise NotImplemented("Yet to implement looking up user's inbox") - -#@oauth_required def object(request, raw_obj=False): """ Lookup for a object type """ objectType = request.matchdict["objectType"] uuid = request.matchdict["uuid"] if objectType not in ["image"]: error = "Unknown type: {0}".format(objectType) - # not sure why this is 404, maybe ask evan. Maybe 400? + # not sure why this is 404, maybe ask evan. Maybe 400? return json_response({"error": error}, status=404) media = MediaEntry.query.filter_by(slug=uuid).first() @@ -232,7 +231,7 @@ def object_comments(request): def host_meta(request): """ This is /.well-known/host-meta - provides URL's to resources on server """ links = [] - + # Client registration links links.append({ "ref": "registration_endpoint", From 247a3b788f3deea120c3f272eda7f7ce9ff54764 Mon Sep 17 00:00:00 2001 From: xray7224 Date: Thu, 14 Nov 2013 22:42:07 +0000 Subject: [PATCH 24/44] Adds the unit-tests for API and cleans up API --- docs/source/api/images.rst | 41 +++++++++++++ mediagoblin/federation/views.py | 10 ++-- mediagoblin/tests/test_api.py | 97 ++++++++++++------------------- mediagoblin/tests/test_joarapi.py | 92 +++++++++++++++++++++++++++++ 4 files changed, 176 insertions(+), 64 deletions(-) create mode 100644 docs/source/api/images.rst create mode 100644 mediagoblin/tests/test_joarapi.py diff --git a/docs/source/api/images.rst b/docs/source/api/images.rst new file mode 100644 index 00000000..6e4d3d48 --- /dev/null +++ b/docs/source/api/images.rst @@ -0,0 +1,41 @@ +.. MediaGoblin Documentation + + Written in 2011, 2012 by MediaGoblin contributors + + To the extent possible under law, the author(s) have dedicated all + copyright and related and neighboring rights to this software to + the public domain worldwide. This software is distributed without + any warranty. + + You should have received a copy of the CC0 Public Domain + Dedication along with this software. If not, see + . + +================== +Uploading an Image +================== + +You must have fully authenticated with oauth to upload an image. + +The endpoint is: ``/api/user//uploads/`` (POST endpoint) + +There are four GET parameters available to use, if they're not specified the defaults (listed below) will be used, the parameters are: + ++-------------+-----------+---------------------+--------------------+ +| Parameter | Required | Default | Example | ++=============+===========+=====================+====================+ +| qqfile | No | unknown | my_picture.jpg | ++-------------+-----------+---------------------+--------------------+ +| title | No | | My Picture! | ++-------------+-----------+---------------------+--------------------+ +| description | No | None | My awesome picture | ++-------------+-----------+---------------------+--------------------+ +| licence | No | All rights reserved | CC BY-SA 3.0 | ++-------------+-----------+---------------------+--------------------+ + +*Note: licence is not part of the pump.io spec and is a GNU MediaGoblin specific parameter* + +Example URL (with parameters): /api/user/tsyesika/uploads/?qqfile=river.jpg&title=The%20River&description=The%20river%20that%20I%20use%20to%20visit%20as%20a%20child%20licence=CC%20BY-SA%203.0 + +Submit the binary image data in the POST parameter. + diff --git a/mediagoblin/federation/views.py b/mediagoblin/federation/views.py index 0dc32e74..5ae7754c 100644 --- a/mediagoblin/federation/views.py +++ b/mediagoblin/federation/views.py @@ -10,7 +10,7 @@ from mediagoblin.tools.response import redirect, json_response from mediagoblin.meddleware.csrf import csrf_exempt from mediagoblin.submit.lib import new_upload_entry -#@oauth_required +@oauth_required def profile(request, raw=False): """ This is /api/user//profile - This will give profile info """ user = request.matchdict["username"] @@ -29,6 +29,7 @@ def profile(request, raw=False): # user profiles are public so return information return json_response(user.serialize(request)) +@oauth_required def user(request): """ This is /api/user/ - This will get the user """ user, user_profile = profile(request, raw=True) @@ -41,7 +42,7 @@ def user(request): return json_response(data) -#@oauth_required +@oauth_required @csrf_exempt def uploads(request): """ Endpoint for file uploads """ @@ -57,7 +58,7 @@ def uploads(request): # Wrap the data in the werkzeug file wrapper file_data = FileStorage( stream=io.BytesIO(request.data), - filename=request.args.get("qqfile", "unknown.jpg"), + filename=request.args.get("qqfile", "unknown"), content_type=request.headers.get("Content-Type", "application/octal-stream") ) @@ -71,7 +72,7 @@ def uploads(request): return json_response({"error": "Not yet implemented"}, status=400) -#@oauth_required +@oauth_required @csrf_exempt def feed(request): """ Handles the user's outbox - /api/user//feed """ @@ -200,6 +201,7 @@ def object(request, raw_obj=False): return json_response(media.serialize(request)) +@oauth_required def object_comments(request): """ Looks up for the comments on a object """ media = object(request, raw_obj=True) diff --git a/mediagoblin/tests/test_api.py b/mediagoblin/tests/test_api.py index 4e0cbd8f..0ba8a424 100644 --- a/mediagoblin/tests/test_api.py +++ b/mediagoblin/tests/test_api.py @@ -14,80 +14,57 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . - -import logging -import base64 +import urllib import pytest +import mock + +from oauthlib.oauth1 import Client from mediagoblin import mg_globals -from mediagoblin.tools import template, pluginapi from mediagoblin.tests.tools import fixture_add_user -from .resources import GOOD_JPG, GOOD_PNG, EVIL_FILE, EVIL_JPG, EVIL_PNG, \ - BIG_BLUE - - -_log = logging.getLogger(__name__) - +from .resources import GOOD_JPG class TestAPI(object): + def setup(self): self.db = mg_globals.database + self.user = fixture_add_user() - self.user_password = u'4cc355_70k3N' - self.user = fixture_add_user(u'joapi', self.user_password, - privileges=[u'active',u'uploader']) + def test_profile_endpoint(self, test_app): + """ Test that you can successfully get the profile of a user """ + @mock.patch("mediagoblin.decorators.oauth_required") + def _real_test(*args, **kwargs): + profile = test_app.get( + "/api/user/{0}/profile".format(self.user.username) + ).json - def login(self, test_app): - test_app.post( - '/auth/login/', { - 'username': self.user.username, - 'password': self.user_password}) + assert profile["preferredUsername"] == self.user.username + assert profile["objectType"] == "person" - def get_context(self, template_name): - return template.TEMPLATE_TEST_CONTEXT[template_name] + _real_test() - def http_auth_headers(self): - return {'Authorization': 'Basic {0}'.format( - base64.b64encode(':'.join([ - self.user.username, - self.user_password])))} + def test_upload_file(self, test_app): + """ Test that i can upload a file """ + context = { + "title": "Rel", + "description": "ayRel sunu oeru", + "qqfile": "my_picture.jpg", + } + encoded_context = urllib.urlencode(context) + response = test_app.post( + "/api/user/{0}/uploads?{1}".format( + self.user.username, + encoded_context[1:] + ) + ) - def do_post(self, data, test_app, **kwargs): - url = kwargs.pop('url', '/api/submit') - do_follow = kwargs.pop('do_follow', False) - - if not 'headers' in kwargs.keys(): - kwargs['headers'] = self.http_auth_headers() - - response = test_app.post(url, data, **kwargs) - - if do_follow: - response.follow() - - return response - - def upload_data(self, filename): - return {'upload_files': [('file', filename)]} - - def test_1_test_test_view(self, test_app): - self.login(test_app) - - response = test_app.get( - '/api/test', - headers=self.http_auth_headers()) - - assert response.body == \ - '{"username": "joapi", "email": "joapi@example.com"}' - - def test_2_test_submission(self, test_app): - self.login(test_app) - - response = self.do_post( - {'title': 'Great JPG!'}, - test_app, - **self.upload_data(GOOD_JPG)) + picture = self.db.MediaEntry.query.filter_by(title=context["title"]) + picture = picture.first() assert response.status_int == 200 + assert picture + raise Exception(str(dir(picture))) + assert picture.description == context["description"] + - assert self.db.MediaEntry.query.filter_by(title=u'Great JPG!').first() diff --git a/mediagoblin/tests/test_joarapi.py b/mediagoblin/tests/test_joarapi.py new file mode 100644 index 00000000..89cf1026 --- /dev/null +++ b/mediagoblin/tests/test_joarapi.py @@ -0,0 +1,92 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + + +import logging +import base64 + +import pytest + +from mediagoblin import mg_globals +from mediagoblin.tools import template, pluginapi +from mediagoblin.tests.tools import fixture_add_user +from .resources import GOOD_JPG, GOOD_PNG, EVIL_FILE, EVIL_JPG, EVIL_PNG, \ + BIG_BLUE + + +_log = logging.getLogger(__name__) + + +class TestAPI(object): + def setup(self): + self.db = mg_globals.database + + self.user_password = u'4cc355_70k3N' + self.user = fixture_add_user(u'joapi', self.user_password) + + def login(self, test_app): + test_app.post( + '/auth/login/', { + 'username': self.user.username, + 'password': self.user_password}) + + def get_context(self, template_name): + return template.TEMPLATE_TEST_CONTEXT[template_name] + + def http_auth_headers(self): + return {'Authorization': 'Basic {0}'.format( + base64.b64encode(':'.join([ + self.user.username, + self.user_password])))} + + def do_post(self, data, test_app, **kwargs): + url = kwargs.pop('url', '/api/submit') + do_follow = kwargs.pop('do_follow', False) + + if not 'headers' in kwargs.keys(): + kwargs['headers'] = self.http_auth_headers() + + response = test_app.post(url, data, **kwargs) + + if do_follow: + response.follow() + + return response + + def upload_data(self, filename): + return {'upload_files': [('file', filename)]} + + def test_1_test_test_view(self, test_app): + self.login(test_app) + + response = test_app.get( + '/api/test', + headers=self.http_auth_headers()) + + assert response.body == \ + '{"username": "joapi", "email": "joapi@example.com"}' + + def test_2_test_submission(self, test_app): + self.login(test_app) + + response = self.do_post( + {'title': 'Great JPG!'}, + test_app, + **self.upload_data(GOOD_JPG)) + + assert response.status_int == 200 + + assert self.db.MediaEntry.query.filter_by(title=u'Great JPG!').first() From 3d869e82b0d6ebe6a1a2f991a9efb76458704095 Mon Sep 17 00:00:00 2001 From: xray7224 Date: Sun, 12 Jan 2014 18:19:37 +0000 Subject: [PATCH 25/44] Improve the documentation --- docs/source/api/images.rst | 127 ++++++++++++++++++++++++++++++++----- 1 file changed, 110 insertions(+), 17 deletions(-) diff --git a/docs/source/api/images.rst b/docs/source/api/images.rst index 6e4d3d48..8c2653d4 100644 --- a/docs/source/api/images.rst +++ b/docs/source/api/images.rst @@ -15,27 +15,120 @@ Uploading an Image ================== -You must have fully authenticated with oauth to upload an image. +To use any the APIs mentioned in this document you will required :doc:`oauth` -The endpoint is: ``/api/user//uploads/`` (POST endpoint) +Uploading and posting an image requiest you to make two requests, one of which +submits the image to the server, the other of which will post the meta data. -There are four GET parameters available to use, if they're not specified the defaults (listed below) will be used, the parameters are: +To upload an image you should use the URI `/api/user//uploads`. -+-------------+-----------+---------------------+--------------------+ -| Parameter | Required | Default | Example | -+=============+===========+=====================+====================+ -| qqfile | No | unknown | my_picture.jpg | -+-------------+-----------+---------------------+--------------------+ -| title | No | | My Picture! | -+-------------+-----------+---------------------+--------------------+ -| description | No | None | My awesome picture | -+-------------+-----------+---------------------+--------------------+ -| licence | No | All rights reserved | CC BY-SA 3.0 | -+-------------+-----------+---------------------+--------------------+ +A POST request should be made to the image upload URI submitting at least two header: -*Note: licence is not part of the pump.io spec and is a GNU MediaGoblin specific parameter* +* `Content-Type` - This being a valid mimetype for the image. +* `Content-Length` - size in bytes of the image. -Example URL (with parameters): /api/user/tsyesika/uploads/?qqfile=river.jpg&title=The%20River&description=The%20river%20that%20I%20use%20to%20visit%20as%20a%20child%20licence=CC%20BY-SA%203.0 +The binary image data should be submitted as POST data to the image upload URI. +You will get back a JSON encoded response which will look similiar to:: -Submit the binary image data in the POST parameter. + { + "updated": "2014-01-11T09:45:48Z", + "links": { + "self": { + "href": "https:///image/4wiBUV1HT8GRqseyvX8m-w" + } + }, + "fullImage": { + "url": "https:////uploads//2014/1/11/V3cBMw.jpg", + "width": 505, + "height": 600 + }, + "replies": { + "url": "https:////api/image/4wiBUV1HT8GRqseyvX8m-w/replies" + }, + "image": { + "url": "https:///uploads//2014/1/11/V3cBMw_thumb.jpg", + "width": 269, + "height": 320 + }, + "author": { + "preferredUsername": "", + "displayName": "", + "links": { + "activity-outbox": { + "href": "https:///api/user//feed" + }, + "self": { + "href": "https:///api/user//profile" + }, + "activity-inbox": { + "href": "https:///api/user//inbox" + } + }, + "url": "https:///", + "updated": "2013-08-14T10:01:21Z", + "id": "acct:@", + "objectType": "person" + }, + "url": "https:////image/4wiBUV1HT8GRqseyvX8m-w", + "published": "2014-01-11T09:45:48Z", + "id": "https:///api/image/4wiBUV1HT8GRqseyvX8m-w", + "objectType": "image" + } +The main things in this response is `fullImage` which contains `url` (the URL +of the original image - i.e. fullsize) and `image` which contains `url` (the URL +of a thumbnail version). + +Submit to feed +============== + +The next request you will probably wish to make is to post the image to your +feed, this currently in GNU MediaGoblin will just show it visably on the website. +In the future it will allow you to specify whom should see this image. + +The URL you will want to make a POST request to to is `/api/user//feed` + +You first should do a post to the feed URI with some of the information you got +back from the above request (which uploaded the image). The request should look +something like:: + + { + "verb": "post", + "object": { + "id": "https:///api/image/6_K9m-2NQFi37je845c83w", + "objectType": "image" + } + } + +(Any other data submitted **will** be ignored) + +Finally if you wish to set a title, description and licence you will need to do +and update request to the endpoint, the following attributes can be submitted: + ++--------------+---------------------------------------+-------------------+ +| Name | Description | Required/Optional | ++==============+=======================================+===================+ +| displayName | This is the title for the image | Optional | ++--------------+---------------------------------------+-------------------+ +| content | This is the description for the image | Optional | ++--------------+---------------------------------------+-------------------+ +| license | This is the licence to be used | Optional | ++--------------+---------------------------------------+-------------------+ + +.. note:: license attribute is mediagoblin specific, pump.io does not support this attribute + + +The update request should look something similiar to:: + + { + "verb": "update", + "object": { + "displayName": "My super awesome image!", + "content": "The awesome image I took while backpacking to modor", + "license": "creativecommons.org/licenses/by-sa/3.0/", + "id": "https:///api/image/6_K9m-2NQFi37je845c83w", + "objectType": "image" + } + } + +(Again, any other data submitted **will** be ignored). From d70b7a5167fe88989500017912e8ac2053a9d84a Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Fri, 4 Apr 2014 11:47:59 -0500 Subject: [PATCH 26/44] Fix issue where create_uuid doesn't exist nor used --- mediagoblin/db/migrations.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mediagoblin/db/migrations.py b/mediagoblin/db/migrations.py index 242d72d9..88cda6f1 100644 --- a/mediagoblin/db/migrations.py +++ b/mediagoblin/db/migrations.py @@ -29,9 +29,10 @@ from mediagoblin.db.extratypes import JSONEncoded, MutationDict from mediagoblin.db.migration_tools import ( RegisterMigration, inspect_table, replace_table_hack) from mediagoblin.db.models import (MediaEntry, Collection, MediaComment, User, - create_uuid, Privilege) + Privilege) from mediagoblin.db.extratypes import JSONEncoded, MutationDict + MIGRATIONS = {} From 1304a28fa75ff76313e2dcb50d25a87be5b2de95 Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Fri, 4 Apr 2014 12:24:45 -0500 Subject: [PATCH 27/44] Add .jpe file extension recognition --- mediagoblin/media_types/image/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediagoblin/media_types/image/__init__.py b/mediagoblin/media_types/image/__init__.py index 0a77b0ce..ae0bfd11 100644 --- a/mediagoblin/media_types/image/__init__.py +++ b/mediagoblin/media_types/image/__init__.py @@ -26,7 +26,7 @@ from mediagoblin.notifications import add_comment_subscription _log = logging.getLogger(__name__) -ACCEPTED_EXTENSIONS = ["jpg", "jpeg", "png", "gif", "tiff"] +ACCEPTED_EXTENSIONS = ["jpe", "jpg", "jpeg", "png", "gif", "tiff"] MEDIA_TYPE = 'mediagoblin.media_types.image' From 41599bf23c7bfe9b1b6fe88ef3a05d6bac987f81 Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Fri, 4 Apr 2014 12:25:20 -0500 Subject: [PATCH 28/44] Fix image upload problem in API --- mediagoblin/federation/views.py | 8 ++++++-- mediagoblin/media_types/image/__init__.py | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/mediagoblin/federation/views.py b/mediagoblin/federation/views.py index 5ae7754c..7107f4bc 100644 --- a/mediagoblin/federation/views.py +++ b/mediagoblin/federation/views.py @@ -1,5 +1,6 @@ import json import io +import mimetypes from werkzeug.datastructures import FileStorage @@ -56,14 +57,17 @@ def uploads(request): request.user = requested_user[0] if request.method == "POST": # Wrap the data in the werkzeug file wrapper + mimetype = request.headers.get("Content-Type", "application/octal-stream") + filename = mimetypes.guess_all_extensions(mimetype) + filename = 'unknown' + filename[0] if filename else filename file_data = FileStorage( stream=io.BytesIO(request.data), - filename=request.args.get("qqfile", "unknown"), + filename=filename, content_type=request.headers.get("Content-Type", "application/octal-stream") ) # Find media manager - media_type, media_manager = sniff_media(file_data) + media_type, media_manager = sniff_media(file_data, filename) entry = new_upload_entry(request.user) if hasattr(media_manager, "api_upload_request"): return media_manager.api_upload_request(request, file_data, entry) diff --git a/mediagoblin/media_types/image/__init__.py b/mediagoblin/media_types/image/__init__.py index ae0bfd11..7b9296fe 100644 --- a/mediagoblin/media_types/image/__init__.py +++ b/mediagoblin/media_types/image/__init__.py @@ -72,7 +72,7 @@ class ImageMediaManager(MediaManagerBase): queue_file = prepare_queue_task(request.app, entry, file_data.filename) with queue_file: queue_file.write(request.data) - + entry.save() feed_url = request.urlgen( From c3b89febc0a030cc6c6fb1c9dfec5741b598c86b Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Mon, 7 Apr 2014 11:09:08 -0500 Subject: [PATCH 29/44] Fix problem where feed posting wasn't returning correct object --- mediagoblin/federation/views.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/mediagoblin/federation/views.py b/mediagoblin/federation/views.py index 7107f4bc..1e8c3e14 100644 --- a/mediagoblin/federation/views.py +++ b/mediagoblin/federation/views.py @@ -1,3 +1,4 @@ + import json import io import mimetypes @@ -72,9 +73,9 @@ def uploads(request): if hasattr(media_manager, "api_upload_request"): return media_manager.api_upload_request(request, file_data, entry) else: - return json_response({"error": "Not yet implemented"}, status=400) + return json_response({"error": "Not yet implemented"}, status=501) - return json_response({"error": "Not yet implemented"}, status=400) + return json_response({"error": "Not yet implemented"}, status=501) @oauth_required @csrf_exempt @@ -120,7 +121,10 @@ def feed(request): error = "No such 'image' with id '{0}'".format(id=media_id) return json_response(error, status=404) media = media[0] - return json_response(media.serialize(request)) + return json_response({ + "verb": "post", + "object": media.serialize(request) + }) elif obj.get("objectType", None) is None: # They need to tell us what type of object they're giving us. From 6781ff3cb1a26752a0f4bca224813fa374a7f248 Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Tue, 8 Jul 2014 21:27:43 +0100 Subject: [PATCH 30/44] Clean up & Add support to update objects in feed API --- mediagoblin/federation/views.py | 105 ++++++++++++++++++++++++++------ mediagoblin/oauth/oauth.py | 16 +++-- 2 files changed, 93 insertions(+), 28 deletions(-) diff --git a/mediagoblin/federation/views.py b/mediagoblin/federation/views.py index 1e8c3e14..8af5565b 100644 --- a/mediagoblin/federation/views.py +++ b/mediagoblin/federation/views.py @@ -64,7 +64,7 @@ def uploads(request): file_data = FileStorage( stream=io.BytesIO(request.data), filename=filename, - content_type=request.headers.get("Content-Type", "application/octal-stream") + content_type=mimetype ) # Find media manager @@ -90,9 +90,12 @@ def feed(request): return json_response({"error": error}, status=404) request.user = requested_user[0] - - if request.method == "POST": + if request.data: data = json.loads(request.data) + else: + data = {"verb": None, "object": {}} + + if request.method == "POST" and data["verb"] == "post": obj = data.get("object", None) if obj is None: error = {"error": "Could not find 'object' element."} @@ -139,12 +142,74 @@ def feed(request): error = {"error": error_message} return json_response(error, status=400) + elif request.method in ["PUT", "POST"] and data["verb"] == "update": + # Check we've got a valid object + obj = data.get("object", None) + + if obj is None: + error = {"error": "Could not find 'object' element."} + return json_response(error, status=400) + + if "objectType" not in obj: + error = {"error": "No objectType specified."} + return json_response(error, status=400) + + if "id" not in obj: + error = {"error": "Object ID has not been specified."} + return json_response(error, status=400) + + obj_id = obj["id"] + + # Now try and find object + if obj["objectType"] == "comment": + comment = MediaComment.query.filter_by(id=obj_id) + if comment is None: + error = {"error": "No such 'comment' with id '{0}'.".format(obj_id)} + return json_response(error, status=400) + comment = comment[0] + + # TODO: refactor this out to update/setting method on MediaComment + if obj.get("content", None) is not None: + comment.content = obj["content"] + + comment.save() + activity = { + "verb": "update", + "object": comment.serialize(request), + } + return json_response(activity) + + elif obj["objectType"] == "image": + image = MediaEntry.query.filter_by(id=obj_id) + if image is None: + error = {"error": "No such 'image' with the id '{0}'.".format(obj_id)} + return json_response(error, status=400) + + image = image[0] + + # TODO: refactor this out to update/setting method on MediaEntry + if obj.get("displayName", None) is not None: + image.title = obj["displayName"] + + if obj.get("content", None) is not None: + image.description = obj["content"] + + if obj.get("license", None) is not None: + # I think we might need some validation here + image.license = obj["license"] + + image.save() + activity = { + "verb": "update", + "object": image.serialize(request), + } + return json_response(activity) feed_url = request.urlgen( - "mediagoblin.federation.feed", - username=request.user.username, - qualified=True - ) + "mediagoblin.federation.feed", + username=request.user.username, + qualified=True + ) feed = { "displayName": "Activities by {user}@{host}".format( @@ -191,17 +256,17 @@ def feed(request): @oauth_required def object(request, raw_obj=False): """ Lookup for a object type """ - objectType = request.matchdict["objectType"] + object_type = request.matchdict["objectType"] uuid = request.matchdict["uuid"] - if objectType not in ["image"]: - error = "Unknown type: {0}".format(objectType) + if object_type not in ["image"]: + error = "Unknown type: {0}".format(object_type) # not sure why this is 404, maybe ask evan. Maybe 400? return json_response({"error": error}, status=404) media = MediaEntry.query.filter_by(slug=uuid).first() if media is None: # no media found with that uuid - error = "Can't find a {0} with ID = {1}".format(objectType, uuid) + error = "Can't find a {0} with ID = {1}".format(object_type, uuid) return json_response({"error": error}, status=404) if raw_obj: @@ -217,14 +282,16 @@ def object_comments(request): if isinstance(response, MediaEntry): comments = response.serialize(request) comments = comments.get("replies", { - "totalItems": 0, - "items": [], - "url": request.urlgen( - "mediagoblin.federation.object.comments", - objectType=media.objectType, - uuid=media.slug, - qualified=True) - }) + "totalItems": 0, + "items": [], + "url": request.urlgen( + "mediagoblin.federation.object.comments", + objectType=media.objectType, + uuid=media.slug, + qualified=True + ) + }) + comments["displayName"] = "Replies to {0}".format(comments["url"]) comments["links"] = { "first": comments["url"], diff --git a/mediagoblin/oauth/oauth.py b/mediagoblin/oauth/oauth.py index d9defa4b..8a60392c 100644 --- a/mediagoblin/oauth/oauth.py +++ b/mediagoblin/oauth/oauth.py @@ -15,12 +15,10 @@ # along with this program. If not, see . from oauthlib.common import Request -from oauthlib.oauth1 import RequestValidator +from oauthlib.oauth1 import RequestValidator from mediagoblin.db.models import NonceTimestamp, Client, RequestToken, AccessToken - - class GMGRequestValidator(RequestValidator): enforce_ssl = False @@ -63,14 +61,14 @@ class GMGRequestValidator(RequestValidator): """ Currently a stub - called when making AccessTokens """ return list() - def validate_timestamp_and_nonce(self, client_key, timestamp, - nonce, request, request_token=None, + def validate_timestamp_and_nonce(self, client_key, timestamp, + nonce, request, request_token=None, access_token=None): nc = NonceTimestamp.query.filter_by(timestamp=timestamp, nonce=nonce) nc = nc.first() if nc is None: return True - + return False def validate_client_key(self, client_key, request): @@ -78,7 +76,7 @@ class GMGRequestValidator(RequestValidator): client = Client.query.filter_by(id=client_key).first() if client is None: return False - + return True def validate_access_token(self, client_key, token, request): @@ -119,9 +117,9 @@ class GMGRequest(Request): """ def __init__(self, request, *args, **kwargs): - """ + """ :param request: werkzeug request object - + any extra params are passed to oauthlib.common.Request object """ kwargs["uri"] = kwargs.get("uri", request.url) From 24e12cb133ac7b87094f8c6ec7efa03464ce4474 Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Tue, 8 Jul 2014 15:39:24 +0100 Subject: [PATCH 31/44] Fix problem in OAuth views --- mediagoblin/oauth/views.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mediagoblin/oauth/views.py b/mediagoblin/oauth/views.py index f424576b..5ade7a8d 100644 --- a/mediagoblin/oauth/views.py +++ b/mediagoblin/oauth/views.py @@ -252,6 +252,7 @@ def authorize(request): if oauth_request.verifier is None: orequest = GMGRequest(request) + orequest.resource_owner_key = token request_validator = GMGRequestValidator() auth_endpoint = AuthorizationEndpoint(request_validator) verifier = auth_endpoint.create_verifier(orequest, {}) @@ -333,7 +334,7 @@ def access_token(request): error = "Missing required parameter." return json_response({"error": error}, status=400) - + request.resource_owner_key = parsed_tokens["oauth_consumer_key"] request.oauth_token = parsed_tokens["oauth_token"] request_validator = GMGRequestValidator(data) av = AccessTokenEndpoint(request_validator) From 128af9533ffa60c356a187d0f98c370f65876893 Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Tue, 8 Jul 2014 17:27:38 +0100 Subject: [PATCH 32/44] Update documentation on uploading media via API --- docs/source/api/{images.rst => media.rst} | 65 +++++++++++++++-------- 1 file changed, 43 insertions(+), 22 deletions(-) rename docs/source/api/{images.rst => media.rst} (67%) diff --git a/docs/source/api/images.rst b/docs/source/api/media.rst similarity index 67% rename from docs/source/api/images.rst rename to docs/source/api/media.rst index 8c2653d4..bafe43d3 100644 --- a/docs/source/api/images.rst +++ b/docs/source/api/media.rst @@ -11,23 +11,36 @@ Dedication along with this software. If not, see . -================== -Uploading an Image -================== +.. info:: Currently only image uploading is supported. + +=============== +Uploading Media +=============== To use any the APIs mentioned in this document you will required :doc:`oauth` -Uploading and posting an image requiest you to make two requests, one of which -submits the image to the server, the other of which will post the meta data. +Uploading and posting an media requiest you to make two to three requests: -To upload an image you should use the URI `/api/user//uploads`. +1) Uploads the data to the server +2) Post media to feed +3) Update media to have title, description, license, etc. (optional) -A POST request should be made to the image upload URI submitting at least two header: +These steps could be condenced in the future however currently this is how the +pump.io API works. There is currently an issue open, if you would like to change +how this works please contribute upstream: https://github.com/e14n/pump.io/issues/657 -* `Content-Type` - This being a valid mimetype for the image. -* `Content-Length` - size in bytes of the image. +---------------------- +Upload Media to Server +---------------------- -The binary image data should be submitted as POST data to the image upload URI. +To upload media you should use the URI `/api/user//uploads`. + +A POST request should be made to the media upload URI submitting at least two header: + +* `Content-Type` - This being a valid mimetype for the media. +* `Content-Length` - size in bytes of the media. + +The media data should be submitted as POST data to the image upload URI. You will get back a JSON encoded response which will look similiar to:: { @@ -79,17 +92,21 @@ The main things in this response is `fullImage` which contains `url` (the URL of the original image - i.e. fullsize) and `image` which contains `url` (the URL of a thumbnail version). +.. warning:: Media which have been uploaded but not submitted to a feed will + periodically be deleted. + +-------------- Submit to feed -============== +-------------- -The next request you will probably wish to make is to post the image to your -feed, this currently in GNU MediaGoblin will just show it visably on the website. -In the future it will allow you to specify whom should see this image. +This is submitting the media to appear on the website. This will create an +object in your feed which will then appear on the GNU MediaGoblin website so the +user and others can view and interact with the media. -The URL you will want to make a POST request to to is `/api/user//feed` +The URL you need to POST to is `/api/user//feed` You first should do a post to the feed URI with some of the information you got -back from the above request (which uploaded the image). The request should look +back from the above request (which uploaded the media). The request should look something like:: { @@ -100,19 +117,23 @@ something like:: } } -(Any other data submitted **will** be ignored) +.. warning:: Any other data submitted **will** be ignored -Finally if you wish to set a title, description and licence you will need to do +------------------- +Submitting Metadata +------------------- + +Finally if you wish to set a title, description and license you will need to do and update request to the endpoint, the following attributes can be submitted: +--------------+---------------------------------------+-------------------+ | Name | Description | Required/Optional | +==============+=======================================+===================+ -| displayName | This is the title for the image | Optional | +| displayName | This is the title for the media | Optional | +--------------+---------------------------------------+-------------------+ -| content | This is the description for the image | Optional | +| content | This is the description for the media | Optional | +--------------+---------------------------------------+-------------------+ -| license | This is the licence to be used | Optional | +| license | This is the license to be used | Optional | +--------------+---------------------------------------+-------------------+ .. note:: license attribute is mediagoblin specific, pump.io does not support this attribute @@ -131,4 +152,4 @@ The update request should look something similiar to:: } } -(Again, any other data submitted **will** be ignored). +.. warning:: Any other data submitted **will** be ignored. From f751d346cf48dc2c6eeb6fa8dcd07be26715f4de Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Wed, 9 Jul 2014 17:23:57 +0100 Subject: [PATCH 33/44] Add fixtures to provide OAuth client, request and access models --- mediagoblin/tests/tools.py | 64 +++++++++++++++++++++++++++++++++++++- 1 file changed, 63 insertions(+), 1 deletion(-) diff --git a/mediagoblin/tests/tools.py b/mediagoblin/tests/tools.py index 060dfda9..d839373b 100644 --- a/mediagoblin/tests/tools.py +++ b/mediagoblin/tests/tools.py @@ -25,13 +25,16 @@ from webtest import TestApp from mediagoblin import mg_globals from mediagoblin.db.models import User, MediaEntry, Collection, MediaComment, \ - CommentSubscription, CommentNotification, Privilege, CommentReport + CommentSubscription, CommentNotification, Privilege, CommentReport, Client, \ + RequestToken, AccessToken from mediagoblin.tools import testing from mediagoblin.init.config import read_mediagoblin_config from mediagoblin.db.base import Session from mediagoblin.meddleware import BaseMeddleware from mediagoblin.auth import gen_password_hash from mediagoblin.gmg_commands.dbupdate import run_dbupdate +from mediagoblin.oauth.views import OAUTH_ALPHABET +from mediagoblin.tools.crypto import random_string from datetime import datetime @@ -343,3 +346,62 @@ def fixture_add_comment_report(comment=None, reported_user=None, Session.expunge(comment_report) return comment_report + +def fixture_add_oauth_client(client_name=None, client_type="native", + redirect_uri=None, contacts=None): + + client_id = random_string(22, OAUTH_ALPHABET) + client_secret = random_string(43, OAUTH_ALPHABET) + + client = Client( + id=client_id, + secret=client_secret, + expirey=None, + application_type=client_type, + application_name=client_name, + contacts=contacts, + redirect_uri=redirect_uri + ) + client.save() + + return client + +def fixture_add_oauth_request_token(user, client=None): + if client is None: + client = fixture_add_oauth_client() + + rt_token = random_string(22, OAUTH_ALPHABET) + rt_secret = random_string(43, OAUTH_ALPHABET) + rt_verifier = random_string(22, OAUTH_ALPHABET) + + request_token = RequestToken( + token=rt_token, + secret=rt_secret, + user=user.id, + used=True, + authenticated=True, + verifier=rt_verifier, + ) + request_token.save() + + return request_token + +def fixture_add_oauth_access_token(user, client=None, request_token=None): + if client is None: + client = fixture_add_oauth_client() + + if request_token is None: + request_token = fixture_add_oauth_request_token(user) + + at_token = random_string(22, OAUTH_ALPHABET) + at_secret = random_string(43, OAUTH_ALPHABET) + + access_token = AccessToken( + token=at_token, + secret=at_secret, + user=user.id, + request_token=request_token.token + ) + access_token.save() + + return access_token From c9115b89c95fa26344f7688120a29d6b6a3efbf3 Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Wed, 9 Jul 2014 18:01:08 +0100 Subject: [PATCH 34/44] Rename test_joarapi.py => test_legacy_api.py --- mediagoblin/tests/{test_joarapi.py => test_legacy_api.py} | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) rename mediagoblin/tests/{test_joarapi.py => test_legacy_api.py} (98%) diff --git a/mediagoblin/tests/test_joarapi.py b/mediagoblin/tests/test_legacy_api.py similarity index 98% rename from mediagoblin/tests/test_joarapi.py rename to mediagoblin/tests/test_legacy_api.py index 89cf1026..4e0cbd8f 100644 --- a/mediagoblin/tests/test_joarapi.py +++ b/mediagoblin/tests/test_legacy_api.py @@ -35,7 +35,8 @@ class TestAPI(object): self.db = mg_globals.database self.user_password = u'4cc355_70k3N' - self.user = fixture_add_user(u'joapi', self.user_password) + self.user = fixture_add_user(u'joapi', self.user_password, + privileges=[u'active',u'uploader']) def login(self, test_app): test_app.post( From ee9956c3de39854f32207789b223f09eb7bbb20b Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Thu, 10 Jul 2014 17:47:54 +0100 Subject: [PATCH 35/44] Remove unneeded oauth fixtures and add test for image submission --- mediagoblin/tests/test_api.py | 120 +++++++++++++++++++++++----------- 1 file changed, 81 insertions(+), 39 deletions(-) diff --git a/mediagoblin/tests/test_api.py b/mediagoblin/tests/test_api.py index 0ba8a424..e1ca688b 100644 --- a/mediagoblin/tests/test_api.py +++ b/mediagoblin/tests/test_api.py @@ -13,58 +13,100 @@ # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . - import urllib +import json import pytest import mock -from oauthlib.oauth1 import Client - from mediagoblin import mg_globals -from mediagoblin.tests.tools import fixture_add_user from .resources import GOOD_JPG +from mediagoblin.tests.tools import fixture_add_user +from mediagoblin.moderation.tools import take_away_privileges +from .resources import GOOD_JPG, GOOD_PNG, EVIL_FILE, EVIL_JPG, EVIL_PNG, \ + BIG_BLUE + +def mocked_oauth_required(*args, **kwargs): + """ Mocks mediagoblin.decorator.oauth_required to always validate """ + + def oauth_required(controller): + return controller + + return oauth_required class TestAPI(object): - def setup(self): + @pytest.fixture(autouse=True) + def setup(self, test_app): + self.test_app = test_app self.db = mg_globals.database - self.user = fixture_add_user() + self.user = fixture_add_user(privileges=[u'active', u'uploader']) - def test_profile_endpoint(self, test_app): - """ Test that you can successfully get the profile of a user """ - @mock.patch("mediagoblin.decorators.oauth_required") - def _real_test(*args, **kwargs): - profile = test_app.get( - "/api/user/{0}/profile".format(self.user.username) - ).json - - assert profile["preferredUsername"] == self.user.username - assert profile["objectType"] == "person" - - _real_test() - - def test_upload_file(self, test_app): - """ Test that i can upload a file """ - context = { - "title": "Rel", - "description": "ayRel sunu oeru", - "qqfile": "my_picture.jpg", + def test_can_post_image(self, test_app): + """ Tests that an image can be posted to the API """ + # First request we need to do is to upload the image + data = open(GOOD_JPG, "rb").read() + headers = { + "Content-Type": "image/jpeg", + "Content-Length": str(len(data)) } - encoded_context = urllib.urlencode(context) - response = test_app.post( - "/api/user/{0}/uploads?{1}".format( - self.user.username, - encoded_context[1:] + + + with mock.patch("mediagoblin.decorators.oauth_required", new_callable=mocked_oauth_required): + response = test_app.post( + "/api/user/{0}/uploads".format(self.user.username), + data, + headers=headers ) - ) - - picture = self.db.MediaEntry.query.filter_by(title=context["title"]) - picture = picture.first() - - assert response.status_int == 200 - assert picture - raise Exception(str(dir(picture))) - assert picture.description == context["description"] + image = json.loads(response.body) + # I should have got certain things back + assert response.status_code == 200 + + assert "id" in image + assert "fullImage" in image + assert "url" in image["fullImage"] + assert "url" in image + assert "author" in image + assert "published" in image + assert "updated" in image + assert image["objectType"] == "image" + + # Now post this to the feed + activity = { + "verb": "post", + "object": image, + } + response = test_app.post( + "/api/user/{0}/feed".format(self.user.username), + activity + ) + + # Check that we got the response we're expecting + assert response.status_code == 200 + + def test_only_uploaders_post_image(self, test_app): + """ Test that only uploaders can upload images """ + # Remove uploader permissions from user + take_away_privileges(self.user.username, u"uploader") + + # Now try and upload a image + data = open(GOOD_JPG, "rb").read() + headers = { + "Content-Type": "image/jpeg", + "Content-Length": str(len(data)), + } + + with mock.patch("mediagoblin.decorators.oauth_required", new_callable=mocked_oauth_required): + response = test_app.post( + "/api/user/{0}/uploads".format(self.user.username), + data, + headers=headers + ) + + error = json.loads(response.body) + + # Assert that we've got a 403 + assert response.status_code == 403 + assert "error" in error From 967df5eff0c00fe7cd860ebfb297ee1f2e0bcdaf Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Thu, 10 Jul 2014 18:17:47 +0100 Subject: [PATCH 36/44] Require uploader privileges to upload media to API --- mediagoblin/federation/views.py | 3 ++- mediagoblin/tests/test_api.py | 43 ++++++++++++++++++-------------- mediagoblin/tests/test_oauth1.py | 9 +++---- mediagoblin/tools/request.py | 18 +++++++++++-- 4 files changed, 46 insertions(+), 27 deletions(-) diff --git a/mediagoblin/federation/views.py b/mediagoblin/federation/views.py index 8af5565b..6e4d81d4 100644 --- a/mediagoblin/federation/views.py +++ b/mediagoblin/federation/views.py @@ -1,4 +1,3 @@ - import json import io import mimetypes @@ -7,6 +6,7 @@ from werkzeug.datastructures import FileStorage from mediagoblin.media_types import sniff_media from mediagoblin.decorators import oauth_required +from mediagoblin.federation.decorators import user_has_privilege from mediagoblin.db.models import User, MediaEntry, MediaComment from mediagoblin.tools.response import redirect, json_response from mediagoblin.meddleware.csrf import csrf_exempt @@ -46,6 +46,7 @@ def user(request): @oauth_required @csrf_exempt +@user_has_privilege(u'uploader') def uploads(request): """ Endpoint for file uploads """ user = request.matchdict["username"] diff --git a/mediagoblin/tests/test_api.py b/mediagoblin/tests/test_api.py index e1ca688b..21222304 100644 --- a/mediagoblin/tests/test_api.py +++ b/mediagoblin/tests/test_api.py @@ -19,21 +19,16 @@ import json import pytest import mock +from webtest import AppError + from mediagoblin import mg_globals from .resources import GOOD_JPG +from mediagoblin.db.models import User from mediagoblin.tests.tools import fixture_add_user from mediagoblin.moderation.tools import take_away_privileges from .resources import GOOD_JPG, GOOD_PNG, EVIL_FILE, EVIL_JPG, EVIL_PNG, \ BIG_BLUE -def mocked_oauth_required(*args, **kwargs): - """ Mocks mediagoblin.decorator.oauth_required to always validate """ - - def oauth_required(controller): - return controller - - return oauth_required - class TestAPI(object): @pytest.fixture(autouse=True) @@ -42,6 +37,18 @@ class TestAPI(object): self.db = mg_globals.database self.user = fixture_add_user(privileges=[u'active', u'uploader']) + def mocked_oauth_required(self, *args, **kwargs): + """ Mocks mediagoblin.decorator.oauth_required to always validate """ + + def fake_controller(controller, request, *args, **kwargs): + request.user = User.query.filter_by(id=self.user.id).first() + return controller(request, *args, **kwargs) + + def oauth_required(c): + return lambda *args, **kwargs: fake_controller(c, *args, **kwargs) + + return oauth_required + def test_can_post_image(self, test_app): """ Tests that an image can be posted to the API """ # First request we need to do is to upload the image @@ -52,7 +59,7 @@ class TestAPI(object): } - with mock.patch("mediagoblin.decorators.oauth_required", new_callable=mocked_oauth_required): + with mock.patch("mediagoblin.decorators.oauth_required", new_callable=self.mocked_oauth_required): response = test_app.post( "/api/user/{0}/uploads".format(self.user.username), data, @@ -98,15 +105,13 @@ class TestAPI(object): "Content-Length": str(len(data)), } - with mock.patch("mediagoblin.decorators.oauth_required", new_callable=mocked_oauth_required): - response = test_app.post( - "/api/user/{0}/uploads".format(self.user.username), - data, - headers=headers - ) - - error = json.loads(response.body) + with mock.patch("mediagoblin.decorators.oauth_required", new_callable=self.mocked_oauth_required): + with pytest.raises(AppError) as excinfo: + response = test_app.post( + "/api/user/{0}/uploads".format(self.user.username), + data, + headers=headers + ) # Assert that we've got a 403 - assert response.status_code == 403 - assert "error" in error + assert "403 FORBIDDEN" in excinfo.value.message diff --git a/mediagoblin/tests/test_oauth1.py b/mediagoblin/tests/test_oauth1.py index 073c2884..568036e5 100644 --- a/mediagoblin/tests/test_oauth1.py +++ b/mediagoblin/tests/test_oauth1.py @@ -52,8 +52,8 @@ class TestOAuth(object): def register_client(self, **kwargs): """ Regiters a client with the API """ - - kwargs["type"] = "client_associate" + + kwargs["type"] = "client_associate" kwargs["application_type"] = kwargs.get("application_type", "native") return self.test_app.post("/api/client/register", kwargs) @@ -63,7 +63,7 @@ class TestOAuth(object): client_info = response.json client = self.db.Client.query.filter_by(id=client_info["client_id"]).first() - + assert response.status_int == 200 assert client is not None @@ -81,7 +81,7 @@ class TestOAuth(object): client_info = response.json client = self.db.Client.query.filter_by(id=client_info["client_id"]).first() - + assert client is not None assert client.secret == client_info["client_secret"] assert client.application_type == query["application_type"] @@ -163,4 +163,3 @@ class TestOAuth(object): assert request_token.client == client.id assert request_token.used == False assert request_token.callback == request_query["oauth_callback"] - diff --git a/mediagoblin/tools/request.py b/mediagoblin/tools/request.py index 2de0b32f..d2cb0f6a 100644 --- a/mediagoblin/tools/request.py +++ b/mediagoblin/tools/request.py @@ -16,7 +16,9 @@ import json import logging -from mediagoblin.db.models import User + +from mediagoblin.db.models import User, AccessToken +from mediagoblin.oauth.tools.request import decode_authorization_header _log = logging.getLogger(__name__) @@ -31,6 +33,18 @@ def setup_user_in_request(request): Examine a request and tack on a request.user parameter if that's appropriate. """ + # If API request the user will be associated with the access token + authorization = decode_authorization_header(request.headers) + + if authorization.get(u"access_token"): + # Check authorization header. + token = authorization[u"oauth_token"] + token = AccessToken.query.filter_by(token=token).first() + if token is not None: + request.user = token.user + return + + if 'user_id' not in request.session: request.user = None return @@ -46,7 +60,7 @@ def setup_user_in_request(request): def decode_request(request): """ Decodes a request based on MIME-Type """ data = request.data - + if request.content_type == json_encoded: data = json.loads(data) elif request.content_type == form_encoded or request.content_type == "": From 51ab51921e5104f1b71402d38928651562c7134a Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Fri, 11 Jul 2014 15:23:55 +0100 Subject: [PATCH 37/44] Add more tests for federation APIs --- mediagoblin/db/models.py | 15 +- mediagoblin/federation/views.py | 25 ++- mediagoblin/media_types/image/__init__.py | 15 +- mediagoblin/tests/test_api.py | 181 +++++++++++++++++----- mediagoblin/tests/tools.py | 58 ------- 5 files changed, 189 insertions(+), 105 deletions(-) diff --git a/mediagoblin/db/models.py b/mediagoblin/db/models.py index cc5d0afa..27ca74e0 100644 --- a/mediagoblin/db/models.py +++ b/mediagoblin/db/models.py @@ -446,9 +446,8 @@ class MediaEntry(Base, MediaEntryMixin): ) context = { - "id": self.id, + "id": self.id, "author": author.serialize(request), - "displayName": self.title, "objectType": self.objectType, "url": url, "image": { @@ -464,6 +463,15 @@ class MediaEntry(Base, MediaEntryMixin): }, } + if self.title: + context["displayName"] = self.title + + if self.description: + context["content"] = self.description + + if self.license: + context["license"] = self.license + if show_comments: comments = [comment.serialize(request) for comment in self.get_comments()] total = len(comments) @@ -478,7 +486,7 @@ class MediaEntry(Base, MediaEntryMixin): ), } - return context + return context class FileKeynames(Base): """ @@ -630,6 +638,7 @@ class MediaComment(Base, MediaCommentMixin): media = MediaEntry.query.filter_by(id=self.media_entry).first() author = self.get_author context = { + "id": self.id, "objectType": "comment", "content": self.content, "inReplyTo": media.serialize(request, show_comments=False), diff --git a/mediagoblin/federation/views.py b/mediagoblin/federation/views.py index 6e4d81d4..c2b02ec0 100644 --- a/mediagoblin/federation/views.py +++ b/mediagoblin/federation/views.py @@ -116,15 +116,27 @@ def feed(request): elif obj.get("objectType", None) == "image": # Posting an image to the feed - # NB: This is currently just handing the image back until we have an - # to send the image to the actual feed - media_id = int(data["object"]["id"]) media = MediaEntry.query.filter_by(id=media_id) if media is None: error = "No such 'image' with id '{0}'".format(id=media_id) return json_response(error, status=404) - media = media[0] + + media = media.first() + obj = data["object"] + + if "displayName" in obj: + media.title = obj["displayName"] + + if "content" in obj: + media.description = obj["content"] + + if "license" in obj: + media.license = obj["license"] + + media.save() + manager = media.media_manager.api_add_to_feed(request, media) + return json_response({ "verb": "post", "object": media.serialize(request) @@ -206,6 +218,11 @@ def feed(request): } return json_response(activity) + elif request.method != "GET": + # Currently unsupported + error = "Unsupported HTTP method {0}".format(request.method) + return json_response({"error": error}, status=501) + feed_url = request.urlgen( "mediagoblin.federation.feed", username=request.user.username, diff --git a/mediagoblin/media_types/image/__init__.py b/mediagoblin/media_types/image/__init__.py index 7b9296fe..96081068 100644 --- a/mediagoblin/media_types/image/__init__.py +++ b/mediagoblin/media_types/image/__init__.py @@ -63,10 +63,7 @@ class ImageMediaManager(MediaManagerBase): """ This handles a image upload request """ # Use the same kind of method from mediagoblin/submit/views:submit_start entry.media_type = unicode(MEDIA_TYPE) - entry.title = unicode(request.args.get("title", file_data.filename)) - entry.description = unicode(request.args.get("description", "")) - entry.license = request.args.get("license", "") # not part of the standard API - + entry.title = file_data.filename entry.generate_slug() queue_file = prepare_queue_task(request.app, entry, file_data.filename) @@ -74,6 +71,16 @@ class ImageMediaManager(MediaManagerBase): queue_file.write(request.data) entry.save() + return json_response(entry.serialize(request)) + + @staticmethod + def api_add_to_feed(request, entry): + """ Add media to Feed """ + if entry.title: + # Shame we have to do this here but we didn't have the data in + # api_upload_request as no filename is usually specified. + entry.slug = None + entry.generate_slug() feed_url = request.urlgen( 'mediagoblin.user_pages.atom_feed', diff --git a/mediagoblin/tests/test_api.py b/mediagoblin/tests/test_api.py index 21222304..38d4c0d5 100644 --- a/mediagoblin/tests/test_api.py +++ b/mediagoblin/tests/test_api.py @@ -23,11 +23,10 @@ from webtest import AppError from mediagoblin import mg_globals from .resources import GOOD_JPG -from mediagoblin.db.models import User +from mediagoblin.db.models import User, MediaEntry from mediagoblin.tests.tools import fixture_add_user from mediagoblin.moderation.tools import take_away_privileges -from .resources import GOOD_JPG, GOOD_PNG, EVIL_FILE, EVIL_JPG, EVIL_PNG, \ - BIG_BLUE +from .resources import GOOD_JPG class TestAPI(object): @@ -35,8 +34,54 @@ class TestAPI(object): def setup(self, test_app): self.test_app = test_app self.db = mg_globals.database + self.user = fixture_add_user(privileges=[u'active', u'uploader']) + def _activity_to_feed(self, test_app, activity, headers=None): + """ Posts an activity to the user's feed """ + if headers: + headers.setdefault("Content-Type", "application/json") + else: + headers = {"Content-Type": "application/json"} + + with mock.patch("mediagoblin.decorators.oauth_required", new_callable=self.mocked_oauth_required): + response = test_app.post( + "/api/user/{0}/feed".format(self.user.username), + json.dumps(activity), + headers=headers + ) + + return response, json.loads(response.body) + + def _upload_image(self, test_app, image): + """ Uploads and image to MediaGoblin via pump.io API """ + data = open(image, "rb").read() + headers = { + "Content-Type": "image/jpeg", + "Content-Length": str(len(data)) + } + + + with mock.patch("mediagoblin.decorators.oauth_required", new_callable=self.mocked_oauth_required): + response = test_app.post( + "/api/user/{0}/uploads".format(self.user.username), + data, + headers=headers + ) + image = json.loads(response.body) + + return response, image + + def _post_image_to_feed(self, test_app, image): + """ Posts an already uploaded image to feed """ + activity = { + "verb": "post", + "object": image, + } + + return self._activity_to_feed(test_app, activity) + + def mocked_oauth_required(self, *args, **kwargs): """ Mocks mediagoblin.decorator.oauth_required to always validate """ @@ -52,46 +97,63 @@ class TestAPI(object): def test_can_post_image(self, test_app): """ Tests that an image can be posted to the API """ # First request we need to do is to upload the image - data = open(GOOD_JPG, "rb").read() - headers = { - "Content-Type": "image/jpeg", - "Content-Length": str(len(data)) - } + response, image = self._upload_image(test_app, GOOD_JPG) + # I should have got certain things back + assert response.status_code == 200 + + assert "id" in image + assert "fullImage" in image + assert "url" in image["fullImage"] + assert "url" in image + assert "author" in image + assert "published" in image + assert "updated" in image + assert image["objectType"] == "image" + + # Check that we got the response we're expecting + response, _ = self._post_image_to_feed(test_app, image) + assert response.status_code == 200 + + def test_upload_image_with_filename(self, test_app): + """ Tests that you can upload an image with filename and description """ + response, data = self._upload_image(test_app, GOOD_JPG) + response, data = self._post_image_to_feed(test_app, data) + + image = data["object"] + + # Now we need to add a title and description + title = "My image ^_^" + description = "This is my super awesome image :D" + license = "CC-BY-SA" + + image["displayName"] = title + image["content"] = description + image["license"] = license + + activity = {"verb": "update", "object": image} with mock.patch("mediagoblin.decorators.oauth_required", new_callable=self.mocked_oauth_required): - response = test_app.post( - "/api/user/{0}/uploads".format(self.user.username), - data, - headers=headers - ) - image = json.loads(response.body) - - - # I should have got certain things back - assert response.status_code == 200 - - assert "id" in image - assert "fullImage" in image - assert "url" in image["fullImage"] - assert "url" in image - assert "author" in image - assert "published" in image - assert "updated" in image - assert image["objectType"] == "image" - - # Now post this to the feed - activity = { - "verb": "post", - "object": image, - } response = test_app.post( "/api/user/{0}/feed".format(self.user.username), - activity + json.dumps(activity), + headers={"Content-Type": "application/json"} ) - # Check that we got the response we're expecting - assert response.status_code == 200 + image = json.loads(response.body)["object"] + + # Check everything has been set on the media correctly + media = MediaEntry.query.filter_by(id=image["id"]).first() + assert media.title == title + assert media.description == description + assert media.license == license + + # Check we're being given back everything we should on an update + assert image["id"] == media.id + assert image["displayName"] == title + assert image["content"] == description + assert image["license"] == license + def test_only_uploaders_post_image(self, test_app): """ Test that only uploaders can upload images """ @@ -115,3 +177,50 @@ class TestAPI(object): # Assert that we've got a 403 assert "403 FORBIDDEN" in excinfo.value.message + + + def test_post_comment(self, test_app): + """ Tests that I can post an comment media """ + # Upload some media to comment on + response, data = self._upload_image(test_app, GOOD_JPG) + response, data = self._post_image_to_feed(test_app, data) + + content = "Hai this is a comment on this lovely picture ^_^" + + activity = { + "verb": "post", + "object": { + "objectType": "comment", + "content": content, + "inReplyTo": data["object"], + } + } + + response, comment_data = self._activity_to_feed(test_app, activity) + assert response.status_code == 200 + + # Find the objects in the database + media = MediaEntry.query.filter_by(id=data["object"]["id"]).first() + comment = media.get_comments()[0] + + # Tests that it matches in the database + assert comment.author == self.user.id + assert comment.content == content + + # Test that the response is what we should be given + assert comment.id == comment_data["object"]["id"] + assert comment.content == comment_data["object"]["content"] + + def test_profile(self, test_app): + """ Tests profile endpoint """ + uri = "/api/user/{0}/profile".format(self.user.username) + with mock.patch("mediagoblin.decorators.oauth_required", new_callable=self.mocked_oauth_required): + response = test_app.get(uri) + profile = json.loads(response.body) + + assert response.status_code == 200 + + assert profile["preferredUsername"] == self.user.username + assert profile["objectType"] == "person" + + assert "links" in profile diff --git a/mediagoblin/tests/tools.py b/mediagoblin/tests/tools.py index d839373b..57dea7b0 100644 --- a/mediagoblin/tests/tools.py +++ b/mediagoblin/tests/tools.py @@ -347,61 +347,3 @@ def fixture_add_comment_report(comment=None, reported_user=None, return comment_report -def fixture_add_oauth_client(client_name=None, client_type="native", - redirect_uri=None, contacts=None): - - client_id = random_string(22, OAUTH_ALPHABET) - client_secret = random_string(43, OAUTH_ALPHABET) - - client = Client( - id=client_id, - secret=client_secret, - expirey=None, - application_type=client_type, - application_name=client_name, - contacts=contacts, - redirect_uri=redirect_uri - ) - client.save() - - return client - -def fixture_add_oauth_request_token(user, client=None): - if client is None: - client = fixture_add_oauth_client() - - rt_token = random_string(22, OAUTH_ALPHABET) - rt_secret = random_string(43, OAUTH_ALPHABET) - rt_verifier = random_string(22, OAUTH_ALPHABET) - - request_token = RequestToken( - token=rt_token, - secret=rt_secret, - user=user.id, - used=True, - authenticated=True, - verifier=rt_verifier, - ) - request_token.save() - - return request_token - -def fixture_add_oauth_access_token(user, client=None, request_token=None): - if client is None: - client = fixture_add_oauth_client() - - if request_token is None: - request_token = fixture_add_oauth_request_token(user) - - at_token = random_string(22, OAUTH_ALPHABET) - at_secret = random_string(43, OAUTH_ALPHABET) - - access_token = AccessToken( - token=at_token, - secret=at_secret, - user=user.id, - request_token=request_token.token - ) - access_token.save() - - return access_token From 3c8bd177b24cbc53dba9ebc8a03f83370e409c4f Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Sat, 12 Jul 2014 08:42:39 +0100 Subject: [PATCH 38/44] Add test for API object endpoint --- mediagoblin/db/models.py | 11 +++++++++++ mediagoblin/federation/routing.py | 2 +- mediagoblin/federation/views.py | 6 +++--- mediagoblin/tests/test_api.py | 29 +++++++++++++++++++++++++++++ 4 files changed, 44 insertions(+), 4 deletions(-) diff --git a/mediagoblin/db/models.py b/mediagoblin/db/models.py index 27ca74e0..0507161e 100644 --- a/mediagoblin/db/models.py +++ b/mediagoblin/db/models.py @@ -461,6 +461,17 @@ class MediaEntry(Base, MediaEntryMixin): "pump_io": { "shared": False, }, + "links": { + "self": { + "href": request.urlgen( + "mediagoblin.federation.object", + objectType=self.objectType, + slug=self.slug, + qualified=True + ), + }, + + } } if self.title: diff --git a/mediagoblin/federation/routing.py b/mediagoblin/federation/routing.py index b9cc4e2e..544edc68 100644 --- a/mediagoblin/federation/routing.py +++ b/mediagoblin/federation/routing.py @@ -51,7 +51,7 @@ add_route( # object endpoints add_route( "mediagoblin.federation.object", - "/api//", + "/api//", "mediagoblin.federation.views:object" ) add_route( diff --git a/mediagoblin/federation/views.py b/mediagoblin/federation/views.py index c2b02ec0..af81cbcb 100644 --- a/mediagoblin/federation/views.py +++ b/mediagoblin/federation/views.py @@ -275,16 +275,16 @@ def feed(request): def object(request, raw_obj=False): """ Lookup for a object type """ object_type = request.matchdict["objectType"] - uuid = request.matchdict["uuid"] + slug = request.matchdict["slug"] if object_type not in ["image"]: error = "Unknown type: {0}".format(object_type) # not sure why this is 404, maybe ask evan. Maybe 400? return json_response({"error": error}, status=404) - media = MediaEntry.query.filter_by(slug=uuid).first() + media = MediaEntry.query.filter_by(slug=slug).first() if media is None: # no media found with that uuid - error = "Can't find a {0} with ID = {1}".format(object_type, uuid) + error = "Can't find a {0} with ID = {1}".format(object_type, slug) return json_response({"error": error}, status=404) if raw_obj: diff --git a/mediagoblin/tests/test_api.py b/mediagoblin/tests/test_api.py index 38d4c0d5..07c34d04 100644 --- a/mediagoblin/tests/test_api.py +++ b/mediagoblin/tests/test_api.py @@ -178,6 +178,35 @@ class TestAPI(object): # Assert that we've got a 403 assert "403 FORBIDDEN" in excinfo.value.message + def test_object_endpoint(self, test_app): + """ Tests that object can be looked up at endpoint """ + # Post an image + response, data = self._upload_image(test_app, GOOD_JPG) + response, data = self._post_image_to_feed(test_app, data) + + # Now lookup image to check that endpoint works. + image = data["object"] + + assert "links" in image + assert "self" in image["links"] + + # Get URI and strip testing host off + object_uri = image["links"]["self"]["href"] + object_uri = object_uri.replace("http://localhost:80", "") + + with mock.patch("mediagoblin.decorators.oauth_required", new_callable=self.mocked_oauth_required): + request = test_app.get(object_uri) + + image = json.loads(request.body) + entry = MediaEntry.query.filter_by(id=image["id"]).first() + + assert request.status_code == 200 + assert entry.id == image["id"] + + assert "image" in image + assert "fullImage" in image + assert "pump_io" in image + assert "links" in image def test_post_comment(self, test_app): """ Tests that I can post an comment media """ From 161cf125f06ae6e0f7f1f1b719ce708dbc70ab4c Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Sat, 12 Jul 2014 09:04:40 +0100 Subject: [PATCH 39/44] Add documentation for interacting with media entires --- docs/source/api/media_interaction.rst | 65 +++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 docs/source/api/media_interaction.rst diff --git a/docs/source/api/media_interaction.rst b/docs/source/api/media_interaction.rst new file mode 100644 index 00000000..41114a71 --- /dev/null +++ b/docs/source/api/media_interaction.rst @@ -0,0 +1,65 @@ +.. MediaGoblin Documentation + + Written in 2011, 2012 by MediaGoblin contributors + + To the extent possible under law, the author(s) have dedicated all + copyright and related and neighboring rights to this software to + the public domain worldwide. This software is distributed without + any warranty. + + You should have received a copy of the CC0 Public Domain + Dedication along with this software. If not, see + . + +Pump.io supports a number of different interactions that can happen against +media. Theser are commenting, liking/favoriting and (re-)sharing. Currently +MediaGoblin supports just commenting although other interactions will come at +a later date. + +-------------- +How to comment +-------------- + +.. warning:: Commenting on a comment currently is NOT supported. + +Commenting is done by posting a comment activity to the users feed. The +activity should look similiar to:: + + { + "verb": "post", + "object": { + "objectType": "comment", + "inReplyTo": + } + } + +This is where `` is the media object you have got with from the server. + +---------------- +Getting comments +---------------- + +The media object you get back should have a `replies` section. This should +be an object which contains the number of replies and if there are any (i.e. +number of replies > 0) then `items` will include an array of every item:: + + { + "totalItems": 2, + "items: [ + { + "id": 1, + "objectType": "comment", + "content": "I'm a comment ^_^", + "author": + }, + { + "id": 4, + "objectType": "comment", + "content": "Another comment! Blimey!", + "author": + } + ], + "url": "http://some.server/api/images/1/comments/" + } + + From 0e283215bd2938f665930f3c481a6003d74bb845 Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Sat, 12 Jul 2014 09:15:16 +0100 Subject: [PATCH 40/44] oops - add decorators for federated APIs --- mediagoblin/federation/decorators.py | 51 ++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 mediagoblin/federation/decorators.py diff --git a/mediagoblin/federation/decorators.py b/mediagoblin/federation/decorators.py new file mode 100644 index 00000000..f515af42 --- /dev/null +++ b/mediagoblin/federation/decorators.py @@ -0,0 +1,51 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +from functools import wraps + +from mediagoblin.db.models import User +from mediagoblin.decorators import require_active_login +from mediagoblin.tools.response import json_response + +def user_has_privilege(privilege_name): + """ + Requires that a user have a particular privilege in order to access a page. + In order to require that a user have multiple privileges, use this + decorator twice on the same view. This decorator also makes sure that the + user is not banned, or else it redirects them to the "You are Banned" page. + + :param privilege_name A unicode object that is that represents + the privilege object. This object is + the name of the privilege, as assigned + in the Privilege.privilege_name column + """ + + def user_has_privilege_decorator(controller): + @wraps(controller) + @require_active_login + def wrapper(request, *args, **kwargs): + user_id = request.user.id + if not request.user.has_privilege(privilege_name): + error = "User '{0}' needs '{1}' privilege".format( + request.user.username, + privilege_name + ) + return json_response({"error": error}, status=403) + + return controller(request, *args, **kwargs) + + return wrapper + return user_has_privilege_decorator + From 0679545f192d8d45a4d98c65bf731e236d73b418 Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Tue, 15 Jul 2014 21:24:25 +0100 Subject: [PATCH 41/44] Add garbage collection task --- mediagoblin.ini | 4 +++ mediagoblin/federation/routing.py | 18 +++++------ mediagoblin/federation/task.py | 49 +++++++++++++++++++++++++++++ mediagoblin/federation/views.py | 31 +++++++++++++----- mediagoblin/init/celery/__init__.py | 13 ++++++++ 5 files changed, 98 insertions(+), 17 deletions(-) create mode 100755 mediagoblin/federation/task.py diff --git a/mediagoblin.ini b/mediagoblin.ini index 5e2477a4..6ccfa4f7 100644 --- a/mediagoblin.ini +++ b/mediagoblin.ini @@ -23,6 +23,10 @@ allow_registration = true # Set to false to disable the ability for users to report offensive content allow_reporting = true +# Frequency garbage collection will run (setting to 0 or false to disable) +# Setting units are minutes. +garbage_collection = 60 + ## Uncomment this to put some user-overriding templates here # local_templates = %(here)s/user_dev/templates/ diff --git a/mediagoblin/federation/routing.py b/mediagoblin/federation/routing.py index 544edc68..c5fa5ce8 100644 --- a/mediagoblin/federation/routing.py +++ b/mediagoblin/federation/routing.py @@ -21,32 +21,32 @@ add_route( "mediagoblin.federation.user", "/api/user//", "mediagoblin.federation.views:user" - ) +) add_route( "mediagoblin.federation.user.profile", "/api/user//profile", "mediagoblin.federation.views:profile" - ) +) # Inbox and Outbox (feed) add_route( "mediagoblin.federation.feed", "/api/user//feed", "mediagoblin.federation.views:feed" - ) +) add_route( "mediagoblin.federation.user.uploads", "/api/user//uploads", "mediagoblin.federation.views:uploads" - ) +) add_route( "mediagoblin.federation.inbox", "/api/user//inbox", "mediagoblin.federation.views:feed" - ) +) # object endpoints add_route( @@ -58,22 +58,22 @@ add_route( "mediagoblin.federation.object.comments", "/api///comments", "mediagoblin.federation.views:object_comments" - ) +) add_route( "mediagoblin.webfinger.well-known.host-meta", "/.well-known/host-meta", "mediagoblin.federation.views:host_meta" - ) +) add_route( "mediagoblin.webfinger.well-known.host-meta.json", "/.well-known/host-meta.json", "mediagoblin.federation.views:host_meta" - ) +) add_route( "mediagoblin.webfinger.whoami", "/api/whoami", "mediagoblin.federation.views:whoami" - ) +) diff --git a/mediagoblin/federation/task.py b/mediagoblin/federation/task.py new file mode 100755 index 00000000..1d42e851 --- /dev/null +++ b/mediagoblin/federation/task.py @@ -0,0 +1,49 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import celery +import datetime +import logging +import pytz + +from mediagoblin.db.models import MediaEntry + +_log = logging.getLogger(__name__) +logging.basicConfig() +_log.setLevel(logging.DEBUG) + +@celery.task() +def collect_garbage(): + """ + Garbage collection to clean up media + + This will look for all critera on models to clean + up. This is primerally written to clean up media that's + entered a erroneous state. + """ + _log.info("Garbage collection is running.") + now = datetime.datetime.now(pytz.UTC) - datetime.timedelta(days=1) + + garbage = MediaEntry.query.filter(MediaEntry.created > now) + garbage = garbage.filter(MediaEntry.state == "unprocessed") + + for entry in garbage.all(): + _log.info("Garbage media found with ID '{0}'".format(entry.id)) + entry.delete() + + + + diff --git a/mediagoblin/federation/views.py b/mediagoblin/federation/views.py index af81cbcb..c383b3ef 100644 --- a/mediagoblin/federation/views.py +++ b/mediagoblin/federation/views.py @@ -1,3 +1,19 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + import json import io import mimetypes @@ -135,7 +151,7 @@ def feed(request): media.license = obj["license"] media.save() - manager = media.media_manager.api_add_to_feed(request, media) + media.media_manager.api_add_to_feed(request, media) return json_response({ "verb": "post", @@ -263,7 +279,7 @@ def feed(request): "actor": request.user.serialize(request), "content": "{0} posted a picture".format(request.user.username), "id": 1, - }) + }) feed["items"][-1]["updated"] = feed["items"][-1]["object"]["updated"] feed["items"][-1]["published"] = feed["items"][-1]["object"]["published"] feed["items"][-1]["url"] = feed["items"][-1]["object"]["url"] @@ -319,7 +335,6 @@ def object_comments(request): return response - ## # Well known ## @@ -331,19 +346,19 @@ def host_meta(request): links.append({ "ref": "registration_endpoint", "href": request.urlgen("mediagoblin.oauth.client_register", qualified=True), - }) + }) links.append({ "ref": "http://apinamespace.org/oauth/request_token", "href": request.urlgen("mediagoblin.oauth.request_token", qualified=True), - }) + }) links.append({ "ref": "http://apinamespace.org/oauth/authorize", "href": request.urlgen("mediagoblin.oauth.authorize", qualified=True), - }) + }) links.append({ "ref": "http://apinamespace.org/oauth/access_token", "href": request.urlgen("mediagoblin.oauth.access_token", qualified=True), - }) + }) return json_response({"links": links}) @@ -353,6 +368,6 @@ def whoami(request): "mediagoblin.federation.user.profile", username=request.user.username, qualified=True - ) + ) return redirect(request, location=profile) diff --git a/mediagoblin/init/celery/__init__.py b/mediagoblin/init/celery/__init__.py index 57242bf6..214d00c3 100644 --- a/mediagoblin/init/celery/__init__.py +++ b/mediagoblin/init/celery/__init__.py @@ -16,6 +16,7 @@ import os import sys +import datetime import logging from celery import Celery @@ -58,6 +59,18 @@ def get_celery_settings_dict(app_config, global_config, celery_settings['CELERY_ALWAYS_EAGER'] = True celery_settings['CELERY_EAGER_PROPAGATES_EXCEPTIONS'] = True + # Garbage collection periodic task + frequency = app_config.get('garbage_collection', 60) + if frequency: + frequency = int(app_config['garbage_collection']) + celery_settings['CELERYBEAT_SCHEDULE'] = { + 'garbage-collection': { + 'task': 'mediagoblin.federation.task.garbage_collection', + 'schedule': datetime.timedelta(minutes=frequency), + } + } + celery_settings['BROKER_HEARTBEAT'] = 1 + return celery_settings From d8f55f2b412507d4c75ebd249a824fdaee66c6dd Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Wed, 16 Jul 2014 17:59:03 +0100 Subject: [PATCH 42/44] Add unseralize for API objects --- lazystarter.sh | 2 +- mediagoblin/db/models.py | 31 ++++++++++++++++++ mediagoblin/federation/views.py | 51 +++++++++-------------------- mediagoblin/init/celery/__init__.py | 2 +- 4 files changed, 49 insertions(+), 37 deletions(-) diff --git a/lazystarter.sh b/lazystarter.sh index d3770194..41994015 100755 --- a/lazystarter.sh +++ b/lazystarter.sh @@ -76,7 +76,7 @@ case "$selfname" in lazycelery.sh) MEDIAGOBLIN_CONFIG="${ini_file}" \ CELERY_CONFIG_MODULE=mediagoblin.init.celery.from_celery \ - $starter "$@" + $starter -B "$@" ;; *) exit 1 ;; esac diff --git a/mediagoblin/db/models.py b/mediagoblin/db/models.py index 0507161e..8ea16b80 100644 --- a/mediagoblin/db/models.py +++ b/mediagoblin/db/models.py @@ -499,6 +499,19 @@ class MediaEntry(Base, MediaEntryMixin): return context + def unserialize(self, data): + """ Takes API objects and unserializes on existing MediaEntry """ + if "displayName" in data: + self.title = data["displayName"] + + if "content" in data: + self.description = data["content"] + + if "license" in data: + self.license = data["license"] + + return True + class FileKeynames(Base): """ keywords for various places. @@ -658,6 +671,24 @@ class MediaComment(Base, MediaCommentMixin): return context + def unserialize(self, data): + """ Takes API objects and unserializes on existing comment """ + # Do initial checks to verify the object is correct + required_attributes = ["content", "inReplyTo"] + for attr in required_attributes: + if attr not in data: + return False + + # Validate inReplyTo has ID + if "id" not in data["inReplyTo"]: + return False + + self.media_entry = data["inReplyTo"]["id"] + self.content = data["content"] + return True + + + class Collection(Base, CollectionMixin): """An 'album' or 'set' of media by a user. diff --git a/mediagoblin/federation/views.py b/mediagoblin/federation/views.py index c383b3ef..8db04f3a 100644 --- a/mediagoblin/federation/views.py +++ b/mediagoblin/federation/views.py @@ -55,8 +55,8 @@ def user(request): "nickname": user.username, "updated": user.created.isoformat(), "published": user.created.isoformat(), - "profile": user_profile - } + "profile": user_profile, + } return json_response(data) @@ -120,12 +120,8 @@ def feed(request): if obj.get("objectType", None) == "comment": # post a comment - media = int(data["object"]["inReplyTo"]["id"]) - comment = MediaComment( - media_entry=media, - author=request.user.id, - content=data["object"]["content"] - ) + comment = MediaComment(author=request.user.id) + comment.unserialize(data["object"]) comment.save() data = {"verb": "post", "object": comment.serialize(request)} return json_response(data) @@ -139,17 +135,9 @@ def feed(request): return json_response(error, status=404) media = media.first() - obj = data["object"] - - if "displayName" in obj: - media.title = obj["displayName"] - - if "content" in obj: - media.description = obj["content"] - - if "license" in obj: - media.license = obj["license"] - + if not media.unserialize(data["object"]): + error = {"error": "Invalid 'image' with id '{0}'".format(obj_id)} + return json_response(error, status=400) media.save() media.media_manager.api_add_to_feed(request, media) @@ -195,13 +183,14 @@ def feed(request): if comment is None: error = {"error": "No such 'comment' with id '{0}'.".format(obj_id)} return json_response(error, status=400) - comment = comment[0] - # TODO: refactor this out to update/setting method on MediaComment - if obj.get("content", None) is not None: - comment.content = obj["content"] + comment = comment[0] + if not comment.unserialize(data["object"]): + error = {"error": "Invalid 'comment' with id '{0}'".format(obj_id)} + return json_response(error, status=400) comment.save() + activity = { "verb": "update", "object": comment.serialize(request), @@ -215,19 +204,11 @@ def feed(request): return json_response(error, status=400) image = image[0] - - # TODO: refactor this out to update/setting method on MediaEntry - if obj.get("displayName", None) is not None: - image.title = obj["displayName"] - - if obj.get("content", None) is not None: - image.description = obj["content"] - - if obj.get("license", None) is not None: - # I think we might need some validation here - image.license = obj["license"] - + if not image.unserialize(obj): + error = {"error": "Invalid 'image' with id '{0}'".format(obj_id)} + return json_response(error, status=400) image.save() + activity = { "verb": "update", "object": image.serialize(request), diff --git a/mediagoblin/init/celery/__init__.py b/mediagoblin/init/celery/__init__.py index 214d00c3..2f2c40d3 100644 --- a/mediagoblin/init/celery/__init__.py +++ b/mediagoblin/init/celery/__init__.py @@ -62,7 +62,7 @@ def get_celery_settings_dict(app_config, global_config, # Garbage collection periodic task frequency = app_config.get('garbage_collection', 60) if frequency: - frequency = int(app_config['garbage_collection']) + frequency = int(frequency) celery_settings['CELERYBEAT_SCHEDULE'] = { 'garbage-collection': { 'task': 'mediagoblin.federation.task.garbage_collection', From 8ac7a653d93831ab5e297e31a94f1056bede05a4 Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Thu, 17 Jul 2014 11:39:24 +0100 Subject: [PATCH 43/44] Create test for garbage collection --- mediagoblin/tests/test_api.py | 41 +++++++++++++++++++++++++++++++---- 1 file changed, 37 insertions(+), 4 deletions(-) diff --git a/mediagoblin/tests/test_api.py b/mediagoblin/tests/test_api.py index 07c34d04..7142ef39 100644 --- a/mediagoblin/tests/test_api.py +++ b/mediagoblin/tests/test_api.py @@ -13,20 +13,24 @@ # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -import urllib import json +import datetime -import pytest import mock +import pytz +import pytest from webtest import AppError +from werkzeug.datastructures import FileStorage +from .resources import GOOD_JPG from mediagoblin import mg_globals -from .resources import GOOD_JPG +from mediagoblin.media_types import sniff_media from mediagoblin.db.models import User, MediaEntry +from mediagoblin.submit.lib import new_upload_entry from mediagoblin.tests.tools import fixture_add_user +from mediagoblin.federation.task import collect_garbage from mediagoblin.moderation.tools import take_away_privileges -from .resources import GOOD_JPG class TestAPI(object): @@ -253,3 +257,32 @@ class TestAPI(object): assert profile["objectType"] == "person" assert "links" in profile + + def test_garbage_collection_task(self, test_app): + """ Test old media entry are removed by GC task """ + # Create a media entry that's unprocessed and over an hour old. + entry_id = 72 + file_data = FileStorage( + stream=open(GOOD_JPG, "rb"), + filename="mah_test.jpg", + content_type="image/jpeg" + ) + + # Find media manager + media_type, media_manager = sniff_media(file_data, "mah_test.jpg") + entry = new_upload_entry(self.user) + entry.id = entry_id + entry.title = "Mah Image" + entry.slug = "slugy-slug-slug" + entry.media_type = 'image' + entry.uploaded = datetime.datetime.now(pytz.UTC) - datetime.timedelta(days=2) + entry.save() + + # Validate the model exists + assert MediaEntry.query.filter_by(id=entry_id).first() is not None + + # Call the garbage collection task + collect_garbage() + + # Now validate the image has been deleted + assert MediaEntry.query.filter_by(id=entry_id).first() is None From a14d90c2db5ff96bdd72009a07f1afc0e8ef3595 Mon Sep 17 00:00:00 2001 From: Jessica Tallon Date: Mon, 21 Jul 2014 18:32:47 +0100 Subject: [PATCH 44/44] Switch from slug to ID and clean up style to conform to PEP-8 --- mediagoblin/db/models.py | 4 +- mediagoblin/federation/__init__.py | 15 ++++++ mediagoblin/federation/routing.py | 4 +- mediagoblin/federation/views.py | 80 ++++++++++++++++++++---------- mediagoblin/tests/test_api.py | 24 ++++++--- 5 files changed, 88 insertions(+), 39 deletions(-) diff --git a/mediagoblin/db/models.py b/mediagoblin/db/models.py index 8ea16b80..aaceb599 100644 --- a/mediagoblin/db/models.py +++ b/mediagoblin/db/models.py @@ -466,7 +466,7 @@ class MediaEntry(Base, MediaEntryMixin): "href": request.urlgen( "mediagoblin.federation.object", objectType=self.objectType, - slug=self.slug, + id=self.id, qualified=True ), }, @@ -492,7 +492,7 @@ class MediaEntry(Base, MediaEntryMixin): "url": request.urlgen( "mediagoblin.federation.object.comments", objectType=self.objectType, - uuid=self.slug, + id=self.id, qualified=True ), } diff --git a/mediagoblin/federation/__init__.py b/mediagoblin/federation/__init__.py index e69de29b..621845ba 100644 --- a/mediagoblin/federation/__init__.py +++ b/mediagoblin/federation/__init__.py @@ -0,0 +1,15 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . diff --git a/mediagoblin/federation/routing.py b/mediagoblin/federation/routing.py index c5fa5ce8..2993b388 100644 --- a/mediagoblin/federation/routing.py +++ b/mediagoblin/federation/routing.py @@ -51,12 +51,12 @@ add_route( # object endpoints add_route( "mediagoblin.federation.object", - "/api//", + "/api//", "mediagoblin.federation.views:object" ) add_route( "mediagoblin.federation.object.comments", - "/api///comments", + "/api///comments", "mediagoblin.federation.views:object_comments" ) diff --git a/mediagoblin/federation/views.py b/mediagoblin/federation/views.py index 8db04f3a..86670857 100644 --- a/mediagoblin/federation/views.py +++ b/mediagoblin/federation/views.py @@ -75,7 +75,10 @@ def uploads(request): request.user = requested_user[0] if request.method == "POST": # Wrap the data in the werkzeug file wrapper - mimetype = request.headers.get("Content-Type", "application/octal-stream") + if "Content-Type" not in request.headers: + error = "Must supply 'Content-Type' header to upload media." + return json_response({"error": error}, status=400) + mimetype = request.headers["Content-Type"] filename = mimetypes.guess_all_extensions(mimetype) filename = 'unknown' + filename[0] if filename else filename file_data = FileStorage( @@ -136,8 +139,8 @@ def feed(request): media = media.first() if not media.unserialize(data["object"]): - error = {"error": "Invalid 'image' with id '{0}'".format(obj_id)} - return json_response(error, status=400) + error = "Invalid 'image' with id '{0}'".format(media_id) + return json_response({"error": error}, status=400) media.save() media.media_manager.api_add_to_feed(request, media) @@ -181,13 +184,13 @@ def feed(request): if obj["objectType"] == "comment": comment = MediaComment.query.filter_by(id=obj_id) if comment is None: - error = {"error": "No such 'comment' with id '{0}'.".format(obj_id)} - return json_response(error, status=400) + error = "No such 'comment' with id '{0}'.".format(obj_id) + return json_response({"error": error}, status=400) comment = comment[0] if not comment.unserialize(data["object"]): - error = {"error": "Invalid 'comment' with id '{0}'".format(obj_id)} - return json_response(error, status=400) + error = "Invalid 'comment' with id '{0}'".format(obj_id) + return json_response({"error": error}, status=400) comment.save() @@ -200,13 +203,13 @@ def feed(request): elif obj["objectType"] == "image": image = MediaEntry.query.filter_by(id=obj_id) if image is None: - error = {"error": "No such 'image' with the id '{0}'.".format(obj_id)} - return json_response(error, status=400) + error = "No such 'image' with the id '{0}'.".format(obj_id) + return json_response({"error": error}, status=400) image = image[0] if not image.unserialize(obj): - error = {"error": "Invalid 'image' with id '{0}'".format(obj_id)} - return json_response(error, status=400) + "Invalid 'image' with id '{0}'".format(obj_id) + return json_response({"error": error}, status=400) image.save() activity = { @@ -254,16 +257,17 @@ def feed(request): # Now lookup the user's feed. for media in MediaEntry.query.all(): - feed["items"].append({ + item = { "verb": "post", "object": media.serialize(request), "actor": request.user.serialize(request), "content": "{0} posted a picture".format(request.user.username), "id": 1, - }) - feed["items"][-1]["updated"] = feed["items"][-1]["object"]["updated"] - feed["items"][-1]["published"] = feed["items"][-1]["object"]["published"] - feed["items"][-1]["url"] = feed["items"][-1]["object"]["url"] + } + item["updated"] = item["object"]["updated"] + item["published"] = item["object"]["published"] + item["url"] = item["object"]["url"] + feed["items"].append(item) feed["totalItems"] = len(feed["items"]) return json_response(feed) @@ -272,16 +276,27 @@ def feed(request): def object(request, raw_obj=False): """ Lookup for a object type """ object_type = request.matchdict["objectType"] - slug = request.matchdict["slug"] + try: + object_id = int(request.matchdict["id"]) + except ValueError: + error = "Invalid object ID '{0}' for '{1}'".format( + request.matchdict["id"], + object_type + ) + return json_response({"error": error}, status=400) + if object_type not in ["image"]: error = "Unknown type: {0}".format(object_type) # not sure why this is 404, maybe ask evan. Maybe 400? return json_response({"error": error}, status=404) - media = MediaEntry.query.filter_by(slug=slug).first() + media = MediaEntry.query.filter_by(id=object_id).first() if media is None: # no media found with that uuid - error = "Can't find a {0} with ID = {1}".format(object_type, slug) + error = "Can't find '{0}' with ID '{1}'".format( + object_type, + object_id + ) return json_response({"error": error}, status=404) if raw_obj: @@ -302,7 +317,7 @@ def object_comments(request): "url": request.urlgen( "mediagoblin.federation.object.comments", objectType=media.objectType, - uuid=media.slug, + uuid=media.id, qualified=True ) }) @@ -320,31 +335,42 @@ def object_comments(request): # Well known ## def host_meta(request): - """ This is /.well-known/host-meta - provides URL's to resources on server """ + """ /.well-known/host-meta - provide URLs to resources """ links = [] - # Client registration links links.append({ "ref": "registration_endpoint", - "href": request.urlgen("mediagoblin.oauth.client_register", qualified=True), + "href": request.urlgen( + "mediagoblin.oauth.client_register", + qualified=True + ), }) links.append({ "ref": "http://apinamespace.org/oauth/request_token", - "href": request.urlgen("mediagoblin.oauth.request_token", qualified=True), + "href": request.urlgen( + "mediagoblin.oauth.request_token", + qualified=True + ), }) links.append({ "ref": "http://apinamespace.org/oauth/authorize", - "href": request.urlgen("mediagoblin.oauth.authorize", qualified=True), + "href": request.urlgen( + "mediagoblin.oauth.authorize", + qualified=True + ), }) links.append({ "ref": "http://apinamespace.org/oauth/access_token", - "href": request.urlgen("mediagoblin.oauth.access_token", qualified=True), + "href": request.urlgen( + "mediagoblin.oauth.access_token", + qualified=True + ), }) return json_response({"links": links}) def whoami(request): - """ This is /api/whoami - This is a HTTP redirect to api profile """ + """ /api/whoami - HTTP redirect to API profile """ profile = request.urlgen( "mediagoblin.federation.user.profile", username=request.user.username, diff --git a/mediagoblin/tests/test_api.py b/mediagoblin/tests/test_api.py index 7142ef39..55228edc 100644 --- a/mediagoblin/tests/test_api.py +++ b/mediagoblin/tests/test_api.py @@ -33,6 +33,7 @@ from mediagoblin.federation.task import collect_garbage from mediagoblin.moderation.tools import take_away_privileges class TestAPI(object): + """ Test mediagoblin's pump.io complient APIs """ @pytest.fixture(autouse=True) def setup(self, test_app): @@ -48,7 +49,8 @@ class TestAPI(object): else: headers = {"Content-Type": "application/json"} - with mock.patch("mediagoblin.decorators.oauth_required", new_callable=self.mocked_oauth_required): + with mock.patch("mediagoblin.decorators.oauth_required", + new_callable=self.mocked_oauth_required): response = test_app.post( "/api/user/{0}/feed".format(self.user.username), json.dumps(activity), @@ -66,7 +68,8 @@ class TestAPI(object): } - with mock.patch("mediagoblin.decorators.oauth_required", new_callable=self.mocked_oauth_required): + with mock.patch("mediagoblin.decorators.oauth_required", + new_callable=self.mocked_oauth_required): response = test_app.post( "/api/user/{0}/uploads".format(self.user.username), data, @@ -137,7 +140,8 @@ class TestAPI(object): activity = {"verb": "update", "object": image} - with mock.patch("mediagoblin.decorators.oauth_required", new_callable=self.mocked_oauth_required): + with mock.patch("mediagoblin.decorators.oauth_required", + new_callable=self.mocked_oauth_required): response = test_app.post( "/api/user/{0}/feed".format(self.user.username), json.dumps(activity), @@ -171,9 +175,10 @@ class TestAPI(object): "Content-Length": str(len(data)), } - with mock.patch("mediagoblin.decorators.oauth_required", new_callable=self.mocked_oauth_required): + with mock.patch("mediagoblin.decorators.oauth_required", + new_callable=self.mocked_oauth_required): with pytest.raises(AppError) as excinfo: - response = test_app.post( + test_app.post( "/api/user/{0}/uploads".format(self.user.username), data, headers=headers @@ -198,7 +203,8 @@ class TestAPI(object): object_uri = image["links"]["self"]["href"] object_uri = object_uri.replace("http://localhost:80", "") - with mock.patch("mediagoblin.decorators.oauth_required", new_callable=self.mocked_oauth_required): + with mock.patch("mediagoblin.decorators.oauth_required", + new_callable=self.mocked_oauth_required): request = test_app.get(object_uri) image = json.loads(request.body) @@ -247,7 +253,8 @@ class TestAPI(object): def test_profile(self, test_app): """ Tests profile endpoint """ uri = "/api/user/{0}/profile".format(self.user.username) - with mock.patch("mediagoblin.decorators.oauth_required", new_callable=self.mocked_oauth_required): + with mock.patch("mediagoblin.decorators.oauth_required", + new_callable=self.mocked_oauth_required): response = test_app.get(uri) profile = json.loads(response.body) @@ -262,6 +269,7 @@ class TestAPI(object): """ Test old media entry are removed by GC task """ # Create a media entry that's unprocessed and over an hour old. entry_id = 72 + now = datetime.datetime.now(pytz.UTC) file_data = FileStorage( stream=open(GOOD_JPG, "rb"), filename="mah_test.jpg", @@ -275,7 +283,7 @@ class TestAPI(object): entry.title = "Mah Image" entry.slug = "slugy-slug-slug" entry.media_type = 'image' - entry.uploaded = datetime.datetime.now(pytz.UTC) - datetime.timedelta(days=2) + entry.uploaded = now - datetime.timedelta(days=2) entry.save() # Validate the model exists