Merge branch 'master' into derek-moore-bug405_email_notifications_for_comments

Conflicts:
	mediagoblin/db/mongo/migrations.py
This commit is contained in:
Christopher Allan Webber
2012-03-18 12:12:41 -05:00
37 changed files with 4497 additions and 76 deletions

View File

@@ -120,7 +120,7 @@ def login(request):
login_failed = False
if request.method == 'POST' and login_form.validate():
user = request.db.User.one(
user = request.db.User.find_one(
{'username': request.POST['username'].lower()})
if user and user.check_login(request.POST['password']):

View File

@@ -2,6 +2,9 @@
# HTML title of the pages
html_title = string(default="GNU MediaGoblin")
# link to source for this MediaGoblin site
source_link = string(default="https://gitorious.org/mediagoblin/mediagoblin")
# Enabled media types
media_types = string_list(default=list("mediagoblin.media_types.image"))

View File

@@ -155,6 +155,22 @@ def convert_video_media_data(database):
collection.save(document)
@RegisterMigration(11)
def convert_gps_media_data(database):
"""
Move media_data["gps"]["*"] to media_data["gps_*"].
In preparation for media_data.gps_*
"""
collection = database['media_entries']
target = collection.find(
{'media_data.gps': {'$exists': True}})
for document in target:
for key, value in document['media_data']['gps'].iteritems():
document['media_data']['gps_' + key] = value
del document['media_data']['gps']
collection.save(document)
@RegisterMigration(12)
def user_add_wants_comment_notification(database):
"""
Add wants_comment_notification to user model

View File

@@ -310,3 +310,9 @@ def check_media_slug_used(db, uploader_id, slug, ignore_m_id):
existing_user_slug_entries = db.MediaEntry.find(
query_dict).count()
return existing_user_slug_entries
def media_entries_for_tag_slug(db, tag_slug):
return db.MediaEntry.find(
{u'state': u'processed',
u'tags.slug': tag_slug})

View File

@@ -14,12 +14,14 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from copy import copy
from mediagoblin.init import setup_global_and_app_config, setup_database
from mediagoblin.db.mongo.util import ObjectId
from mediagoblin.db.sql.models import (Base, User, MediaEntry, MediaComment,
Tag, MediaTag, MediaFile, MediaAttachmentFile)
Tag, MediaTag, MediaFile, MediaAttachmentFile, MigrationData)
from mediagoblin.media_types.image.models import ImageData
from mediagoblin.media_types.video.models import VideoData
from mediagoblin.db.sql.open import setup_connection_and_db_from_config as \
sql_connect
@@ -106,6 +108,38 @@ def convert_media_entries(mk_db):
session.close()
def convert_image(mk_db):
session = Session()
for media in mk_db.MediaEntry.find(
{'media_type': 'mediagoblin.media_types.image'}).sort('created'):
media_data = copy(media.media_data)
# TODO: Fix after exif is migrated
media_data.pop('exif', None)
if len(media_data):
media_data_row = ImageData(**media_data)
media_data_row.media_entry = obj_id_table[media._id]
session.add(media_data_row)
session.commit()
session.close()
def convert_video(mk_db):
session = Session()
for media in mk_db.MediaEntry.find(
{'media_type': 'mediagoblin.media_types.video'}).sort('created'):
media_data_row = VideoData(**media.media_data)
media_data_row.media_entry = obj_id_table[media._id]
session.add(media_data_row)
session.commit()
session.close()
def convert_media_tags(mk_db):
session = Session()
session.autoflush = False
@@ -155,6 +189,20 @@ def convert_media_comments(mk_db):
session.close()
def convert_add_migration_versions():
session = Session()
for name in ("__main__",
"mediagoblin.media_types.image",
"mediagoblin.media_types.video",
):
m = MigrationData(name=name, version=0)
session.add(m)
session.commit()
session.close()
def run_conversion(config_name):
global_config, app_config = setup_global_and_app_config(config_name)
@@ -167,10 +215,16 @@ def run_conversion(config_name):
Session.remove()
convert_media_entries(mk_db)
Session.remove()
convert_image(mk_db)
Session.remove()
convert_video(mk_db)
Session.remove()
convert_media_tags(mk_db)
Session.remove()
convert_media_comments(mk_db)
Session.remove()
convert_add_migration_versions()
Session.remove()
if __name__ == '__main__':

View File

@@ -20,6 +20,7 @@ TODO: indexes on foreignkeys, where useful.
import datetime
import sys
from sqlalchemy import (
Column, Integer, Unicode, UnicodeText, DateTime, Boolean, ForeignKey,
@@ -28,10 +29,12 @@ from sqlalchemy.orm import relationship
from sqlalchemy.orm.collections import attribute_mapped_collection
from sqlalchemy.sql.expression import desc
from sqlalchemy.ext.associationproxy import association_proxy
from sqlalchemy.util import memoized_property
from mediagoblin.db.sql.extratypes import PathTupleWithSlashes, JSONEncoded
from mediagoblin.db.sql.base import Base, DictReadAttrProxy
from mediagoblin.db.mixin import UserMixin, MediaEntryMixin, MediaCommentMixin
from mediagoblin.db.sql.base import Session
# It's actually kind of annoying how sqlalchemy-migrate does this, if
# I understand it right, but whatever. Anyway, don't remove this :P
@@ -58,7 +61,7 @@ class User(Base, UserMixin):
TODO: We should consider moving some rarely used fields
into some sort of "shadow" table.
"""
__tablename__ = "users"
__tablename__ = "core__users"
id = Column(Integer, primary_key=True)
username = Column(Unicode, nullable=False, unique=True)
@@ -85,10 +88,10 @@ class MediaEntry(Base, MediaEntryMixin):
"""
TODO: Consider fetching the media_files using join
"""
__tablename__ = "media_entries"
__tablename__ = "core__media_entries"
id = Column(Integer, primary_key=True)
uploader = Column(Integer, ForeignKey('users.id'), nullable=False)
uploader = Column(Integer, ForeignKey('core__users.id'), nullable=False)
title = Column(Unicode, nullable=False)
slug = Column(Unicode)
created = Column(DateTime, nullable=False, default=datetime.datetime.now)
@@ -168,14 +171,39 @@ class MediaEntry(Base, MediaEntryMixin):
if media is not None:
return media.url_for_self(urlgen)
#@memoized_property
@property
def media_data(self):
# TODO: Replace with proper code to read the correct table
return {}
session = Session()
return session.query(self.media_data_table).filter_by(
media_entry=self.id).first()
def media_data_init(self, **kwargs):
# TODO: Implement this
pass
"""
Initialize or update the contents of a media entry's media_data row
"""
session = Session()
media_data = session.query(self.media_data_table).filter_by(
media_entry=self.id).first()
# No media data, so actually add a new one
if not media_data:
media_data = self.media_data_table(
**kwargs)
session.add(media_data)
# Update old media data
else:
for field, value in kwargs.iteritems():
setattr(media_data, field, value)
@memoized_property
def media_data_table(self):
# TODO: memoize this
models_module = self.media_type + '.models'
__import__(models_module)
return sys.modules[models_module].DATA_MODEL
class FileKeynames(Base):
@@ -203,7 +231,7 @@ class MediaFile(Base):
TODO: Highly consider moving "name" into a new table.
TODO: Consider preloading said table in software
"""
__tablename__ = "mediafiles"
__tablename__ = "core__mediafiles"
media_entry = Column(
Integer, ForeignKey(MediaEntry.id),
@@ -242,7 +270,7 @@ class MediaAttachmentFile(Base):
class Tag(Base):
__tablename__ = "tags"
__tablename__ = "core__tags"
id = Column(Integer, primary_key=True)
slug = Column(Unicode, nullable=False, unique=True)
@@ -259,13 +287,13 @@ class Tag(Base):
class MediaTag(Base):
__tablename__ = "media_tags"
__tablename__ = "core__media_tags"
id = Column(Integer, primary_key=True)
media_entry = Column(
Integer, ForeignKey(MediaEntry.id),
nullable=False)
tag = Column(Integer, ForeignKey('tags.id'), nullable=False)
tag = Column(Integer, ForeignKey('core__tags.id'), nullable=False)
name = Column(Unicode)
# created = Column(DateTime, nullable=False, default=datetime.datetime.now)
@@ -292,12 +320,12 @@ class MediaTag(Base):
class MediaComment(Base, MediaCommentMixin):
__tablename__ = "media_comments"
__tablename__ = "core__media_comments"
id = Column(Integer, primary_key=True)
media_entry = Column(
Integer, ForeignKey('media_entries.id'), nullable=False)
author = Column(Integer, ForeignKey('users.id'), nullable=False)
Integer, ForeignKey('core__media_entries.id'), nullable=False)
author = Column(Integer, ForeignKey('core__users.id'), nullable=False)
created = Column(DateTime, nullable=False, default=datetime.datetime.now)
content = Column(UnicodeText, nullable=False)
@@ -319,7 +347,7 @@ MODELS = [
######################################################
class MigrationData(Base):
__tablename__ = "migrations"
__tablename__ = "core__migrations"
name = Column(Unicode, primary_key=True)
version = Column(Integer, nullable=False, default=0)

View File

@@ -294,6 +294,15 @@ def check_media_slug_used(dummy_db, uploader_id, slug, ignore_m_id):
return does_exist
def media_entries_for_tag_slug(dummy_db, tag_slug):
return MediaEntry.query \
.join(MediaEntry.tags_helper) \
.join(MediaTag.tag_helper) \
.filter(
(MediaEntry.state == u'processed')
& (Tag.slug == tag_slug))
def clean_orphan_tags():
q1 = Session.query(Tag).outerjoin(MediaTag).filter(MediaTag.id==None)
for t in q1:

View File

@@ -21,7 +21,9 @@ except ImportError:
if use_sql:
from mediagoblin.db.sql.fake import ObjectId, InvalidId, DESCENDING
from mediagoblin.db.sql.util import atomic_update, check_media_slug_used
from mediagoblin.db.sql.util import atomic_update, check_media_slug_used, \
media_entries_for_tag_slug
else:
from mediagoblin.db.mongo.util import \
ObjectId, InvalidId, DESCENDING, atomic_update, check_media_slug_used
ObjectId, InvalidId, DESCENDING, atomic_update, \
check_media_slug_used, media_entries_for_tag_slug

View File

@@ -17,7 +17,7 @@
import sys
from mediagoblin.db.mongo import util as db_util
from mediagoblin.db.open import setup_connection_and_db_from_config
from mediagoblin.db.mongo.open import setup_connection_and_db_from_config
from mediagoblin.init import setup_global_and_app_config
# This MUST be imported so as to set up the appropriate migrations!
@@ -41,7 +41,12 @@ def _print_finished_migration(migration_number, migration_func):
def migrate(args):
global_config, app_config = setup_global_and_app_config(args.conf_file)
run_migrate(args.conf_file)
def run_migrate(conf_file):
global_config, app_config = setup_global_and_app_config(conf_file)
connection, db = setup_connection_and_db_from_config(
app_config, use_pymongo=True)
migration_manager = db_util.MigrationManager(db)

View File

@@ -14,12 +14,15 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from mediagoblin.db.sql.convert import run_conversion
def mongosql_parser_setup(subparser):
pass
def mongosql(args):
# First, make sure our mongo migrations are up to date...
from mediagoblin.gmg_commands.migrate import run_migrate
run_migrate(args.conf_file)
from mediagoblin.db.sql.convert import run_conversion
run_conversion(args.conf_file)

View File

@@ -22,7 +22,9 @@ from mediagoblin.gmg_commands import util as commands_util
def shell_parser_setup(subparser):
pass
subparser.add_argument(
'--ipython', help='Use ipython',
action="store_true")
SHELL_BANNER = """\
@@ -34,16 +36,42 @@ Available vars:
- db: database instance
"""
def py_shell(**user_namespace):
"""
Run a shell using normal python shell.
"""
code.interact(
banner=SHELL_BANNER,
local=user_namespace)
def ipython_shell(**user_namespace):
"""
Run a shell for the user using ipython.
"""
try:
from IPython import embed
except:
print "IPython not available... exiting!"
return
embed(
banner1=SHELL_BANNER,
user_ns=user_namespace)
def shell(args):
"""
Setup a shell for the user
either a normal Python shell
or an IPython one
"""
mgoblin_app = commands_util.setup_app(args)
user_namespace = {
'mg_globals': mg_globals,
'mgoblin_app': commands_util.setup_app(args),
'db': mg_globals.database}
code.interact(
banner=SHELL_BANNER,
local={
'mgoblin_app': mgoblin_app,
'mg_globals': mg_globals,
'db': mg_globals.database})
if args.ipython:
ipython_shell(**user_namespace)
else:
py_shell(**user_namespace)

View File

@@ -14,7 +14,7 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from mediagoblin.db.util import DESCENDING
from mediagoblin.db.util import media_entries_for_tag_slug, DESCENDING
from mediagoblin.tools.pagination import Pagination
from mediagoblin.tools.response import render_to_response
@@ -29,11 +29,16 @@ def _get_tag_name_from_entries(media_entries, tag_slug):
"""
# ... this is slightly hacky looking :\
tag_name = tag_slug
if media_entries.count():
for tag in media_entries[0]['tags']:
for entry in media_entries:
for tag in entry.tags:
if tag['slug'] == tag_slug:
tag_name == tag['name']
tag_name = tag['name']
break
break
# TODO: Remove after SQL-switch, it's mongo specific
if hasattr(media_entries, "rewind"):
media_entries.rewind()
return tag_name
@@ -43,9 +48,7 @@ def tag_listing(request, page):
"""'Gallery'/listing for this tag slug"""
tag_slug = request.matchdict[u'tag']
cursor = request.db.MediaEntry.find(
{u'state': u'processed',
u'tags.slug': tag_slug})
cursor = media_entries_for_tag_slug(request.db, tag_slug)
cursor = cursor.sort('created', DESCENDING)
pagination = Pagination(page, cursor)

View File

@@ -23,11 +23,11 @@ from sqlalchemy import (
class AsciiData(Base):
__tablename__ = "ascii_data"
__tablename__ = "ascii__mediadata"
id = Column(Integer, primary_key=True)
media_entry = Column(
Integer, ForeignKey('media_entries.id'), nullable=False)
Integer, ForeignKey('core__media_entries.id'), nullable=False)
DATA_MODEL = AsciiData

View File

@@ -5,15 +5,17 @@ from sqlalchemy import (
class ImageData(Base):
__tablename__ = "image_data"
__tablename__ = "image__mediadata"
# The primary key *and* reference to the main media_entry
media_entry = Column(Integer, ForeignKey('media_entries.id'),
media_entry = Column(Integer, ForeignKey('core__media_entries.id'),
primary_key=True)
width = Column(Integer)
height = Column(Integer)
gps_longitude = Column(Float)
gps_latitude = Column(Float)
gps_altitude = Column(Float)
gps_direction = Column(Float)
DATA_MODEL = ImageData

View File

@@ -115,11 +115,18 @@ def process_image(entry):
# Insert exif data into database
media_data = entry.setdefault('media_data', {})
media_data['exif'] = {
'clean': clean_exif(exif_tags)}
media_data['exif']['useful'] = get_useful(
media_data['exif']['clean'])
media_data['gps'] = gps_data
# TODO: Fix for sql media_data, when exif is in sql
if media_data is not None:
media_data['exif'] = {
'clean': clean_exif(exif_tags)}
media_data['exif']['useful'] = get_useful(
media_data['exif']['clean'])
if len(gps_data):
for key in list(gps_data.keys()):
gps_data['gps_' + key] = gps_data.pop(key)
entry.media_data_init(**gps_data)
# clean up workbench
workbench.destroy_self()

View File

@@ -22,10 +22,10 @@ from sqlalchemy import (
class VideoData(Base):
__tablename__ = "video_data"
__tablename__ = "video__mediadata"
# The primary key *and* reference to the main media_entry
media_entry = Column(Integer, ForeignKey('media_entries.id'),
media_entry = Column(Integer, ForeignKey('core__media_entries.id'),
primary_key=True)
width = Column(SmallInteger)
height = Column(SmallInteger)

View File

@@ -234,13 +234,14 @@ text-align: center;
height: 0;
}
h3.sidedata {
border: none;
background-color: #212121;
border-radius: 4px 4px 0 0;
padding: 3px 8px;
margin: 20px 0 5px 0;
.media_sidebar h3 {
font-size: 1em;
margin: 0 0 5px;
border: none;
}
.media_sidebar p {
padding-left: 8px;
}
/* forms */

View File

@@ -0,0 +1 @@
../../../../extlib/video-js

View File

@@ -82,7 +82,10 @@
{% block mediagoblin_footer %}
<footer>
{% trans -%}
Powered by <a href="http://mediagoblin.org">MediaGoblin</a>, a <a href="http://gnu.org/">GNU</a> project
Powered by <a href="http://mediagoblin.org">MediaGoblin</a>, a <a href="http://gnu.org/">GNU</a> project.
{%- endtrans %}
{% trans source_link=app_config['source_link'] -%}
Released under the <a href="http://www.fsf.org/licensing/licenses/agpl-3.0.html">AGPL</a>. <a href="{{ source_link }}">Source code</a> available.
{%- endtrans %}
</footer>
{% endblock mediagoblin_footer %}

View File

@@ -18,6 +18,13 @@
{% extends 'mediagoblin/user_pages/media.html' %}
{% block mediagoblin_head %}
{{ super() }}
<script type="text/javascript"
src="{{ request.staticdirect('/js/extlib/video-js/video.js') }}"></script>
<link href="{{ request.staticdirect('/js/extlib/video-js/video-js.css') }}" rel="stylesheet">
{% endblock %}
{% block mediagoblin_media %}
<div class="video-player" style="position: relative;">
<video class="video-js vjs-default-skin"

View File

@@ -172,7 +172,7 @@
</div>
<div class="media_sidebar">
{% trans date=media.created.strftime("%Y-%m-%d") -%}
<h3 class="sidedata">Added on</h3>
<h3>Added on</h3>
<p>{{ date }}</p>
{%- endtrans %}
{% if media.tags %}

View File

@@ -17,10 +17,10 @@
#}
{% block exif_content %}
{% if media.media_data.has_key('exif')
and app_config['exif_visible']
{% if app_config['exif_visible']
and media.media_data.exif is defined
and media.media_data.exif.has_key('useful') %}
<h3 class="sidedata">EXIF</h3>
<h3>EXIF</h3>
<table>
{% for key, tag in media.media_data.exif.useful.items() %}
<tr>

View File

@@ -17,24 +17,27 @@
#}
{% block geolocation_map %}
{% if media.media_data.has_key('gps')
and app_config['geolocation_map_visible']
and media.media_data.gps %}
<h3 class="sidedata">Location</h3>
{% if app_config['geolocation_map_visible']
and media.media_data.gps_latitude is defined
and media.media_data.gps_latitude
and media.media_data.gps_longitude is defined
and media.media_data.gps_longitude %}
<h3>{% trans %}Location{% endtrans %}</h3>
<div>
{% set gps = media.media_data.gps %}
{%- set lon = media.media_data.gps_longitude %}
{%- set lat = media.media_data.gps_latitude %}
{%- set osm_url = "http://openstreetmap.org/?mlat={lat}&mlon={lon}".format(lat=lat, lon=lon) %}
<div id="tile-map" style="width: 100%; height: 196px;">
<input type="hidden" id="gps-longitude"
value="{{ gps.longitude }}" />
value="{{ lon }}" />
<input type="hidden" id="gps-latitude"
value="{{ gps.latitude }}" />
value="{{ lat }}" />
</div>
<p>
<small>
View on
<a href="http://openstreetmap.org/?mlat={{ gps.latitude }}&mlon={{ gps.longitude }}">
OpenStreetMap
</a>
{% trans -%}
View on <a href="{{ osm_url }}">OpenStreetMap</a>
{%- endtrans %}
</small>
</p>
</div>

View File

@@ -17,7 +17,7 @@
#}
{% block license_content -%}
<h3 class="sidedata">{% trans %}License{% endtrans %}</h3>
<h3>{% trans %}License{% endtrans %}</h3>
<p>
{% if media.license %}
<a href="{{ media.license }}">{{ media.get_license_data().abbreviation }}</a>

View File

@@ -17,7 +17,7 @@
#}
{% block tags_content -%}
<h3 class="sidedata">Tagged with</h3>
<h3>{% trans %}Tagged with{% endtrans %}</h3>
<p>
{% for tag in media.tags %}
{% if loop.last %}

View File

@@ -180,6 +180,18 @@ class TestSubmission:
# Does media entry exist?
assert_true(media)
# Add a comment, so we can test for its deletion later.
get_comments = lambda: list(
request.db.MediaComment.find({'media_entry': media._id}))
assert_false(get_comments())
response = self.test_app.post(
request.urlgen('mediagoblin.user_pages.media_post_comment',
user=self.test_user.username,
media=media._id),
{'comment_content': 'i love this test'})
response.follow()
assert_true(get_comments())
# Do not confirm deletion
# ---------------------------------------------------
response = self.test_app.post(
@@ -219,6 +231,9 @@ class TestSubmission:
request.db.MediaEntry.find(
{'_id': media._id}).count())
# How about the comment?
assert_false(get_comments())
def test_malicious_uploads(self):
# Test non-suppoerted file with non-supported extension
# -----------------------------------------------------

View File

@@ -14,8 +14,12 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import logging
from mediagoblin.db.util import ObjectId, InvalidId
_log = logging.getLogger(__name__)
def setup_user_in_request(request):
"""
Examine a request and tack on a request.user parameter if that's
@@ -30,12 +34,12 @@ def setup_user_in_request(request):
except InvalidId:
user = None
else:
user = request.db.User.one({'_id': oid})
user = request.db.User.find_one({'_id': oid})
if not user:
# Something's wrong... this user doesn't exist? Invalidate
# this session.
print "Killing session for %r" % request.session['user_id']
_log.warn("Killing session for user id %r", request.session['user_id'])
request.session.invalidate()
request.user = user

View File

@@ -21,7 +21,7 @@ from mediagoblin.tools.translate import fake_ugettext_passthrough as _
class MediaCommentForm(wtforms.Form):
comment_content = wtforms.TextAreaField(
_(''),
'',
[wtforms.validators.Required()])

View File

@@ -180,6 +180,10 @@ def media_confirm_delete(request, media):
if form.confirm.data is True:
username = media.get_uploader.username
# Delete all the associated comments
for comment in media.get_comments():
comment.delete()
# Delete all files on the public storage
delete_media_files(media)