heckyel/mediagoblin
0
0
Fork 0
Code Issues Pull Requests Releases Activity

Merged changes with upstream

Browse Source
This commit is contained in:
Aaron Williamson
2012-01-17 00:59:21 -05:00
parent 25b48323a8 8c7701f9f1
commit 99a270e952
233 changed files with 20405 additions and 6364 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
mediagoblin/db/__init__.py
View File

@@ -23,7 +23,7 @@ Database Abstraction/Wrapper Layer
pymongo. Read beow for why, but note that nobody is actually doing
this and there's no proof that we'll ever support more than
MongoDB... it would be a huge amount of work to do so.
If you really want to prove that possible, jump on IRC and talk to
us about making such a branch. In the meanwhile, it doesn't hurt to
have things as they are... if it ever makes it hard for us to

92
mediagoblin/db/mixin.py Normal file
View File

@@ -0,0 +1,92 @@
# GNU MediaGoblin -- federated, autonomous media hosting
# Copyright (C) 2011,2012 MediaGoblin contributors. See AUTHORS.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
This module contains some Mixin classes for the db objects.
A bunch of functions on the db objects are really more like
"utility functions": They could live outside the classes
and be called "by hand" passing the appropiate reference.
They usually only use the public API of the object and
rarely use database related stuff.
These functions now live here and get "mixed in" into the
real objects.
"""
from mediagoblin.auth import lib as auth_lib
from mediagoblin.tools import common
class UserMixin(object):
def check_login(self, password):
"""
See if a user can login with this password
"""
return auth_lib.bcrypt_check_password(
password, self.pw_hash)
class MediaEntryMixin(object):
def get_display_media(self, media_map,
fetch_order=common.DISPLAY_IMAGE_FETCHING_ORDER):
"""
Find the best media for display.
Args:
- media_map: a dict like
{u'image_size': [u'dir1', u'dir2', u'image.jpg']}
- fetch_order: the order we should try fetching images in
Returns:
(media_size, media_path)
"""
media_sizes = media_map.keys()
for media_size in common.DISPLAY_IMAGE_FETCHING_ORDER:
if media_size in media_sizes:
return media_map[media_size]
def main_mediafile(self):
pass
def url_for_self(self, urlgen, **extra_args):
"""
Generate an appropriate url for ourselves
Use a slug if we have one, else use our '_id'.
"""
uploader = self.get_uploader
if self.get('slug'):
return urlgen(
'mediagoblin.user_pages.media_home',
user=uploader.username,
media=self.slug,
**extra_args)
else:
return urlgen(
'mediagoblin.user_pages.media_home',
user=uploader.username,
media=unicode(self._id),
**extra_args)
def get_fail_exception(self):
"""
Get the exception that's appropriate for this error
"""
if self['fail_error']:
return common.import_component(self['fail_error'])

15
mediagoblin/db/mongo/__init__.py Normal file
View File

@@ -0,0 +1,15 @@
# GNU MediaGoblin -- federated, autonomous media hosting
# Copyright (C) 2011 MediaGoblin contributors. See AUTHORS.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.

5
mediagoblin/db/indexes.py → mediagoblin/db/mongo/indexes.py
View File

@@ -93,8 +93,9 @@ MEDIAENTRY_INDEXES = {
('created', DESCENDING)]},
'state_uploader_tags_created': {
# Indexing on processed?, media uploader, associated tags, and timestamp
# Used for showing media items matching a tag search, most recent first.
# Indexing on processed?, media uploader, associated tags, and
# timestamp Used for showing media items matching a tag
# search, most recent first.
'index': [('state', ASCENDING),
('uploader', ASCENDING),
('tags.slug', DESCENDING),

10
mediagoblin/db/migrations.py → mediagoblin/db/mongo/migrations.py
View File

@@ -14,7 +14,7 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from mediagoblin.db.util import RegisterMigration
from mediagoblin.db.mongo.util import RegisterMigration
from mediagoblin.tools.text import cleaned_markdown_conversion
@@ -100,3 +100,11 @@ def user_add_forgot_password_token_and_expires(database):
"""
add_table_field(database, 'users', 'fp_verification_key', None)
add_table_field(database, 'users', 'fp_token_expire', None)
@RegisterMigration(7)
def media_type_image_to_multimedia_type_image(database):
database['media_entries'].update(
{'media_type': 'image'},
{'$set': {'media_type': 'mediagoblin.media_types.image'}},
multi=True)

0
mediagoblin/db/models.py → mediagoblin/db/mongo/models.py
View File

78
mediagoblin/db/mongo/open.py Normal file
View File

@@ -0,0 +1,78 @@
# GNU MediaGoblin -- federated, autonomous media hosting
# Copyright (C) 2011 MediaGoblin contributors. See AUTHORS.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import pymongo
import mongokit
from paste.deploy.converters import asint
from mediagoblin.db.mongo import models
from mediagoblin.db.mongo.util import MigrationManager
def connect_database_from_config(app_config, use_pymongo=False):
"""
Connect to the main database, take config from app_config
Optionally use pymongo instead of mongokit for the connection.
"""
port = app_config.get('db_port')
if port:
port = asint(port)
if use_pymongo:
connection = pymongo.Connection(
app_config.get('db_host'), port)
else:
connection = mongokit.Connection(
app_config.get('db_host'), port)
return connection
def setup_connection_and_db_from_config(app_config, use_pymongo=False):
"""
Setup connection and database from config.
Optionally use pymongo instead of mongokit.
"""
connection = connect_database_from_config(app_config, use_pymongo)
database_path = app_config['db_name']
db = connection[database_path]
if not use_pymongo:
models.register_models(connection)
return (connection, db)
def check_db_migrations_current(db):
# This MUST be imported so as to set up the appropriate migrations!
from mediagoblin.db.mongo import migrations
# Init the migration number if necessary
migration_manager = MigrationManager(db)
migration_manager.install_migration_version_if_missing()
# Tiny hack to warn user if our migration is out of date
if not migration_manager.database_at_latest_migration():
db_migration_num = migration_manager.database_current_migration()
latest_migration_num = migration_manager.latest_migration()
if db_migration_num < latest_migration_num:
print (
"*WARNING:* Your migrations are out of date, "
"maybe run ./bin/gmg migrate?")
elif db_migration_num > latest_migration_num:
print (
"*WARNING:* Your migrations are out of date... "
"in fact they appear to be from the future?!")

292
mediagoblin/db/mongo/util.py Normal file
View File

@@ -0,0 +1,292 @@
# GNU MediaGoblin -- federated, autonomous media hosting
# Copyright (C) 2011 MediaGoblin contributors. See AUTHORS.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
Utilities for database operations.
Some note on migration and indexing tools:
We store information about what the state of the database is in the
'mediagoblin' document of the 'app_metadata' collection. Keys in that
document relevant to here:
- 'migration_number': The integer representing the current state of
the migrations
"""
import copy
# Imports that other modules might use
from pymongo import ASCENDING, DESCENDING
from pymongo.errors import InvalidId
from mongokit import ObjectId
from mediagoblin.db.mongo.indexes import ACTIVE_INDEXES, DEPRECATED_INDEXES
################
# Indexing tools
################
def add_new_indexes(database, active_indexes=ACTIVE_INDEXES):
"""
Add any new indexes to the database.
Args:
- database: pymongo or mongokit database instance.
- active_indexes: indexes to possibly add in the pattern of:
{'collection_name': {
'identifier': {
'index': [index_foo_goes_here],
'unique': True}}
where 'index' is the index to add and all other options are
arguments for collection.create_index.
Returns:
A list of indexes added in form ('collection', 'index_name')
"""
indexes_added = []
for collection_name, indexes in active_indexes.iteritems():
collection = database[collection_name]
collection_indexes = collection.index_information().keys()
for index_name, index_data in indexes.iteritems():
if not index_name in collection_indexes:
# Get a copy actually so we don't modify the actual
# structure
index_data = copy.copy(index_data)
index = index_data.pop('index')
collection.create_index(
index, name=index_name, **index_data)
indexes_added.append((collection_name, index_name))
return indexes_added
def remove_deprecated_indexes(database, deprecated_indexes=DEPRECATED_INDEXES):
"""
Remove any deprecated indexes from the database.
Args:
- database: pymongo or mongokit database instance.
- deprecated_indexes: the indexes to deprecate in the pattern of:
{'collection_name': {
'identifier': {
'index': [index_foo_goes_here],
'unique': True}}
(... although we really only need the 'identifier' here, as the
rest of the information isn't used in this case. But it's kept
around so we can remember what it was)
Returns:
A list of indexes removed in form ('collection', 'index_name')
"""
indexes_removed = []
for collection_name, indexes in deprecated_indexes.iteritems():
collection = database[collection_name]
collection_indexes = collection.index_information().keys()
for index_name, index_data in indexes.iteritems():
if index_name in collection_indexes:
collection.drop_index(index_name)
indexes_removed.append((collection_name, index_name))
return indexes_removed
#################
# Migration tools
#################
# The default migration registry...
#
# Don't set this yourself! RegisterMigration will automatically fill
# this with stuff via decorating methods in migrations.py
class MissingCurrentMigration(Exception):
pass
MIGRATIONS = {}
class RegisterMigration(object):
"""
Tool for registering migrations
Call like:
@RegisterMigration(33)
def update_dwarves(database):
[...]
This will register your migration with the default migration
registry. Alternately, to specify a very specific
migration_registry, you can pass in that as the second argument.
Note, the number of your migration should NEVER be 0 or less than
0. 0 is the default "no migrations" state!
"""
def __init__(self, migration_number, migration_registry=MIGRATIONS):
assert migration_number > 0, "Migration number must be > 0!"
assert migration_number not in migration_registry, \
"Duplicate migration numbers detected! That's not allowed!"
self.migration_number = migration_number
self.migration_registry = migration_registry
def __call__(self, migration):
self.migration_registry[self.migration_number] = migration
return migration
class MigrationManager(object):
"""
Migration handling tool.
Takes information about a database, lets you update the database
to the latest migrations, etc.
"""
def __init__(self, database, migration_registry=MIGRATIONS):
"""
Args:
- database: database we're going to migrate
- migration_registry: where we should find all migrations to
run
"""
self.database = database
self.migration_registry = migration_registry
self._sorted_migrations = None
def _ensure_current_migration_record(self):
"""
If there isn't a database[u'app_metadata'] mediagoblin entry
with the 'current_migration', throw an error.
"""
if self.database_current_migration() is None:
raise MissingCurrentMigration(
"Tried to call function which requires "
"'current_migration' set in database")
@property
def sorted_migrations(self):
"""
Sort migrations if necessary and store in self._sorted_migrations
"""
if not self._sorted_migrations:
self._sorted_migrations = sorted(
self.migration_registry.items(),
# sort on the key... the migration number
key=lambda migration_tuple: migration_tuple[0])
return self._sorted_migrations
def latest_migration(self):
"""
Return a migration number for the latest migration, or 0 if
there are no migrations.
"""
if self.sorted_migrations:
return self.sorted_migrations[-1][0]
else:
# If no migrations have been set, we start at 0.
return 0
def set_current_migration(self, migration_number):
"""
Set the migration in the database to migration_number
"""
# Add the mediagoblin migration if necessary
self.database[u'app_metadata'].update(
{u'_id': u'mediagoblin'},
{u'$set': {u'current_migration': migration_number}},
upsert=True)
def install_migration_version_if_missing(self):
"""
Sets the migration to the latest version if no migration
version at all is set.
"""
mgoblin_metadata = self.database[u'app_metadata'].find_one(
{u'_id': u'mediagoblin'})
if not mgoblin_metadata:
latest_migration = self.latest_migration()
self.set_current_migration(latest_migration)
def database_current_migration(self):
"""
Return the current migration in the database.
"""
mgoblin_metadata = self.database[u'app_metadata'].find_one(
{u'_id': u'mediagoblin'})
if not mgoblin_metadata:
return None
else:
return mgoblin_metadata[u'current_migration']
def database_at_latest_migration(self):
"""
See if the database is at the latest migration.
Returns a boolean.
"""
current_migration = self.database_current_migration()
return current_migration == self.latest_migration()
def migrations_to_run(self):
"""
Get a list of migrations to run still, if any.
Note that calling this will set your migration version to the
latest version if it isn't installed to anything yet!
"""
self._ensure_current_migration_record()
db_current_migration = self.database_current_migration()
return [
(migration_number, migration_func)
for migration_number, migration_func in self.sorted_migrations
if migration_number > db_current_migration]
def migrate_new(self, pre_callback=None, post_callback=None):
"""
Run all migrations.
Includes two optional args:
- pre_callback: if called, this is a callback on something to
run pre-migration. Takes (migration_number, migration_func)
as arguments
- pre_callback: if called, this is a callback on something to
run post-migration. Takes (migration_number, migration_func)
as arguments
"""
# If we aren't set to any version number, presume we're at the
# latest (which means we'll do nothing here...)
self.install_migration_version_if_missing()
for migration_number, migration_func in self.migrations_to_run():
if pre_callback:
pre_callback(migration_number, migration_func)
migration_func(self.database)
self.set_current_migration(migration_number)
if post_callback:
post_callback(migration_number, migration_func)

41
mediagoblin/db/open.py
View File

@@ -14,42 +14,5 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import pymongo
import mongokit
from paste.deploy.converters import asint
from mediagoblin.db import models
def connect_database_from_config(app_config, use_pymongo=False):
"""
Connect to the main database, take config from app_config
Optionally use pymongo instead of mongokit for the connection.
"""
port = app_config.get('db_port')
if port:
port = asint(port)
if use_pymongo:
connection = pymongo.Connection(
app_config.get('db_host'), port)
else:
connection = mongokit.Connection(
app_config.get('db_host'), port)
return connection
def setup_connection_and_db_from_config(app_config, use_pymongo=False):
"""
Setup connection and database from config.
Optionally use pymongo instead of mongokit.
"""
connection = connect_database_from_config(app_config, use_pymongo)
database_path = app_config['db_name']
db = connection[database_path]
if not use_pymongo:
models.register_models(connection)
return (connection, db)
from mediagoblin.db.mongo.open import \
setup_connection_and_db_from_config, check_db_migrations_current

15
mediagoblin/db/sql/__init__.py Normal file
View File

@@ -0,0 +1,15 @@
# GNU MediaGoblin -- federated, autonomous media hosting
# Copyright (C) 2011 MediaGoblin contributors. See AUTHORS.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.

38
mediagoblin/db/sql/base.py Normal file
View File

@@ -0,0 +1,38 @@
from sqlalchemy.orm import scoped_session, sessionmaker, object_session
Session = scoped_session(sessionmaker())
def _fix_query_dict(query_dict):
if '_id' in query_dict:
query_dict['id'] = query_dict.pop('_id')
class GMGTableBase(object):
query = Session.query_property()
@classmethod
def find(cls, query_dict={}):
_fix_query_dict(query_dict)
return cls.query.filter_by(**query_dict)
@classmethod
def find_one(cls, query_dict={}):
_fix_query_dict(query_dict)
return cls.query.filter_by(**query_dict).first()
@classmethod
def one(cls, query_dict):
return cls.find(query_dict).one()
def get(self, key):
return getattr(self, key)
def save(self, validate = True):
assert validate
sess = object_session(self)
if sess is None:
sess = Session()
sess.add(self)
sess.commit()

151
mediagoblin/db/sql/convert.py Normal file
View File

@@ -0,0 +1,151 @@
from mediagoblin.init import setup_global_and_app_config, setup_database
from mediagoblin.db.mongo.util import ObjectId
from mediagoblin.db.sql.models import (Base, User, MediaEntry, MediaComment,
Tag, MediaTag, MediaFile)
from mediagoblin.db.sql.open import setup_connection_and_db_from_config as \
sql_connect
from mediagoblin.db.mongo.open import setup_connection_and_db_from_config as \
mongo_connect
from mediagoblin.db.sql.base import Session
obj_id_table = dict()
def add_obj_ids(entry, new_entry):
global obj_id_table
print "%r -> %r" % (entry._id, new_entry.id)
obj_id_table[entry._id] = new_entry.id
def copy_attrs(entry, new_entry, attr_list):
for a in attr_list:
val = entry[a]
setattr(new_entry, a, val)
def copy_reference_attr(entry, new_entry, ref_attr):
val = entry[ref_attr]
val = obj_id_table[val]
setattr(new_entry, ref_attr, val)
def convert_users(mk_db):
session = Session()
for entry in mk_db.User.find():
print entry.username
new_entry = User()
copy_attrs(entry, new_entry,
('username', 'email', 'created', 'pw_hash', 'email_verified',
'status', 'verification_key', 'is_admin', 'url',
'bio', 'bio_html',
'fp_verification_key', 'fp_token_expire',))
# new_entry.fp_verification_expire = entry.fp_token_expire
session.add(new_entry)
session.flush()
add_obj_ids(entry, new_entry)
session.commit()
session.close()
def convert_media_entries(mk_db):
session = Session()
for entry in mk_db.MediaEntry.find():
print repr(entry.title)
new_entry = MediaEntry()
copy_attrs(entry, new_entry,
('title', 'slug', 'created',
'description', 'description_html',
'media_type', 'state',
'fail_error',
'queued_task_id',))
copy_reference_attr(entry, new_entry, "uploader")
session.add(new_entry)
session.flush()
add_obj_ids(entry, new_entry)
for key, value in entry.media_files.iteritems():
new_file = MediaFile(name=key, file_path=value)
new_file.media_entry = new_entry.id
Session.add(new_file)
session.commit()
session.close()
def convert_media_tags(mk_db):
session = Session()
session.autoflush = False
for media in mk_db.MediaEntry.find():
print repr(media.title)
for otag in media.tags:
print " ", repr((otag["slug"], otag["name"]))
nslug = session.query(Tag).filter_by(slug=otag["slug"]).first()
print " ", repr(nslug)
if nslug is None:
nslug = Tag(slug=otag["slug"])
session.add(nslug)
session.flush()
print " ", repr(nslug), nslug.id
ntag = MediaTag()
ntag.tag = nslug.id
ntag.name = otag["name"]
ntag.media_entry = obj_id_table[media._id]
session.add(ntag)
session.commit()
session.close()
def convert_media_comments(mk_db):
session = Session()
for entry in mk_db.MediaComment.find():
print repr(entry.content)
new_entry = MediaComment()
copy_attrs(entry, new_entry,
('created',
'content', 'content_html',))
copy_reference_attr(entry, new_entry, "media_entry")
copy_reference_attr(entry, new_entry, "author")
session.add(new_entry)
session.flush()
add_obj_ids(entry, new_entry)
session.commit()
session.close()
def main():
global_config, app_config = setup_global_and_app_config("mediagoblin.ini")
sql_conn, sql_db = sql_connect({'sql_engine': 'sqlite:///mediagoblin.db'})
mk_conn, mk_db = mongo_connect(app_config)
Base.metadata.create_all(sql_db.engine)
convert_users(mk_db)
Session.remove()
convert_media_entries(mk_db)
Session.remove()
convert_media_tags(mk_db)
Session.remove()
convert_media_comments(mk_db)
Session.remove()
if __name__ == '__main__':
main()

18
mediagoblin/db/sql/extratypes.py Normal file
View File

@@ -0,0 +1,18 @@
from sqlalchemy.types import TypeDecorator, Unicode
class PathTupleWithSlashes(TypeDecorator):
"Represents a Tuple of strings as a slash separated string."
impl = Unicode
def process_bind_param(self, value, dialect):
if value is not None:
assert len(value), "Does not support empty lists"
value = '/'.join(value)
return value
def process_result_value(self, value, dialect):
if value is not None:
value = tuple(value.split('/'))
return value

153
mediagoblin/db/sql/models.py Normal file
View File

@@ -0,0 +1,153 @@
import datetime
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy import (
Column, Integer, Unicode, UnicodeText, DateTime, Boolean, ForeignKey,
UniqueConstraint)
from sqlalchemy.orm import relationship
from sqlalchemy.orm.collections import attribute_mapped_collection
from sqlalchemy.ext.associationproxy import association_proxy
from mediagoblin.db.sql.extratypes import PathTupleWithSlashes
from mediagoblin.db.sql.base import GMGTableBase
from mediagoblin.db.mixin import UserMixin, MediaEntryMixin
Base = declarative_base(cls=GMGTableBase)
class SimpleFieldAlias(object):
"""An alias for any field"""
def __init__(self, fieldname):
self.fieldname = fieldname
def __get__(self, instance, cls):
return getattr(instance, self.fieldname)
def __set__(self, instance, val):
setattr(instance, self.fieldname, val)
class User(Base, UserMixin):
__tablename__ = "users"
id = Column(Integer, primary_key=True)
username = Column(Unicode, nullable=False, unique=True)
email = Column(Unicode, nullable=False)
created = Column(DateTime, nullable=False, default=datetime.datetime.now)
pw_hash = Column(Unicode, nullable=False)
email_verified = Column(Boolean)
status = Column(Unicode, default=u"needs_email_verification", nullable=False)
verification_key = Column(Unicode)
is_admin = Column(Boolean, default=False, nullable=False)
url = Column(Unicode)
bio = Column(UnicodeText) # ??
bio_html = Column(UnicodeText) # ??
fp_verification_key = Column(Unicode)
fp_token_expire = Column(DateTime)
## TODO
# plugin data would be in a separate model
_id = SimpleFieldAlias("id")
class MediaEntry(Base, MediaEntryMixin):
__tablename__ = "media_entries"
id = Column(Integer, primary_key=True)
uploader = Column(Integer, ForeignKey('users.id'), nullable=False)
title = Column(Unicode, nullable=False)
slug = Column(Unicode, nullable=False)
created = Column(DateTime, nullable=False, default=datetime.datetime.now)
description = Column(UnicodeText) # ??
description_html = Column(UnicodeText) # ??
media_type = Column(Unicode, nullable=False)
state = Column(Unicode, nullable=False) # or use sqlalchemy.types.Enum?
fail_error = Column(Unicode)
fail_metadata = Column(UnicodeText)
queued_media_file = Column(PathTupleWithSlashes)
queued_task_id = Column(Unicode)
__table_args__ = (
UniqueConstraint('uploader', 'slug'),
{})
get_uploader = relationship(User)
media_files_helper = relationship("MediaFile",
collection_class=attribute_mapped_collection("name"),
cascade="all, delete-orphan"
)
media_files = association_proxy('media_files_helper', 'file_path',
creator=lambda k,v: MediaFile(name=k, file_path=v)
)
## TODO
# media_data
# attachment_files
# fail_error
class MediaFile(Base):
__tablename__ = "mediafiles"
media_entry = Column(
Integer, ForeignKey(MediaEntry.id),
nullable=False, primary_key=True)
name = Column(Unicode, primary_key=True)
file_path = Column(PathTupleWithSlashes)
def __repr__(self):
return "<MediaFile %s: %r>" % (self.name, self.file_path)
class Tag(Base):
__tablename__ = "tags"
id = Column(Integer, primary_key=True)
slug = Column(Unicode, nullable=False, unique=True)
class MediaTag(Base):
__tablename__ = "media_tags"
id = Column(Integer, primary_key=True)
tag = Column(Integer, ForeignKey('tags.id'), nullable=False)
name = Column(Unicode)
media_entry = Column(
Integer, ForeignKey('media_entries.id'),
nullable=False)
# created = Column(DateTime, nullable=False, default=datetime.datetime.now)
__table_args__ = (
UniqueConstraint('tag', 'media_entry'),
{})
class MediaComment(Base):
__tablename__ = "media_comments"
id = Column(Integer, primary_key=True)
media_entry = Column(
Integer, ForeignKey('media_entries.id'), nullable=False)
author = Column(Integer, ForeignKey('users.id'), nullable=False)
created = Column(DateTime, nullable=False, default=datetime.datetime.now)
content = Column(UnicodeText, nullable=False)
content_html = Column(UnicodeText)
get_author = relationship(User)
def show_table_init():
from sqlalchemy import create_engine
engine = create_engine('sqlite:///:memory:', echo=True)
Base.metadata.create_all(engine)
if __name__ == '__main__':
show_table_init()

33
mediagoblin/db/sql/open.py Normal file
View File

@@ -0,0 +1,33 @@
from sqlalchemy import create_engine
from mediagoblin.db.sql.base import Session
from mediagoblin.db.sql.models import Base
class DatabaseMaster(object):
def __init__(self, engine):
self.engine = engine
for k,v in Base._decl_class_registry.iteritems():
setattr(self, k, v)
def commit(self):
Session.commit()
def save(self, obj):
Session.add(obj)
Session.flush()
def reset_after_request(self):
Session.remove()
def setup_connection_and_db_from_config(app_config):
engine = create_engine(app_config['sql_engine'], echo=True)
Session.configure(bind=engine)
return "dummy conn", DatabaseMaster(engine)
def check_db_migrations_current(db):
pass

277
mediagoblin/db/util.py
View File

@@ -14,278 +14,5 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
Utilities for database operations.
Some note on migration and indexing tools:
We store information about what the state of the database is in the
'mediagoblin' document of the 'app_metadata' collection. Keys in that
document relevant to here:
- 'migration_number': The integer representing the current state of
the migrations
"""
import copy
# Imports that other modules might use
from pymongo import ASCENDING, DESCENDING
from pymongo.errors import InvalidId
from mongokit import ObjectId
from mediagoblin.db.indexes import ACTIVE_INDEXES, DEPRECATED_INDEXES
################
# Indexing tools
################
def add_new_indexes(database, active_indexes=ACTIVE_INDEXES):
"""
Add any new indexes to the database.
Args:
- database: pymongo or mongokit database instance.
- active_indexes: indexes to possibly add in the pattern of:
{'collection_name': {
'identifier': {
'index': [index_foo_goes_here],
'unique': True}}
where 'index' is the index to add and all other options are
arguments for collection.create_index.
Returns:
A list of indexes added in form ('collection', 'index_name')
"""
indexes_added = []
for collection_name, indexes in active_indexes.iteritems():
collection = database[collection_name]
collection_indexes = collection.index_information().keys()
for index_name, index_data in indexes.iteritems():
if not index_name in collection_indexes:
# Get a copy actually so we don't modify the actual
# structure
index_data = copy.copy(index_data)
index = index_data.pop('index')
collection.create_index(
index, name=index_name, **index_data)
indexes_added.append((collection_name, index_name))
return indexes_added
def remove_deprecated_indexes(database, deprecated_indexes=DEPRECATED_INDEXES):
"""
Remove any deprecated indexes from the database.
Args:
- database: pymongo or mongokit database instance.
- deprecated_indexes: the indexes to deprecate in the pattern of:
{'collection_name': {
'identifier': {
'index': [index_foo_goes_here],
'unique': True}}
(... although we really only need the 'identifier' here, as the
rest of the information isn't used in this case. But it's kept
around so we can remember what it was)
Returns:
A list of indexes removed in form ('collection', 'index_name')
"""
indexes_removed = []
for collection_name, indexes in deprecated_indexes.iteritems():
collection = database[collection_name]
collection_indexes = collection.index_information().keys()
for index_name, index_data in indexes.iteritems():
if index_name in collection_indexes:
collection.drop_index(index_name)
indexes_removed.append((collection_name, index_name))
return indexes_removed
#################
# Migration tools
#################
# The default migration registry...
#
# Don't set this yourself! RegisterMigration will automatically fill
# this with stuff via decorating methods in migrations.py
class MissingCurrentMigration(Exception): pass
MIGRATIONS = {}
class RegisterMigration(object):
"""
Tool for registering migrations
Call like:
@RegisterMigration(33)
def update_dwarves(database):
[...]
This will register your migration with the default migration
registry. Alternately, to specify a very specific
migration_registry, you can pass in that as the second argument.
Note, the number of your migration should NEVER be 0 or less than
0. 0 is the default "no migrations" state!
"""
def __init__(self, migration_number, migration_registry=MIGRATIONS):
assert migration_number > 0, "Migration number must be > 0!"
assert not migration_registry.has_key(migration_number), \
"Duplicate migration numbers detected! That's not allowed!"
self.migration_number = migration_number
self.migration_registry = migration_registry
def __call__(self, migration):
self.migration_registry[self.migration_number] = migration
return migration
class MigrationManager(object):
"""
Migration handling tool.
Takes information about a database, lets you update the database
to the latest migrations, etc.
"""
def __init__(self, database, migration_registry=MIGRATIONS):
"""
Args:
- database: database we're going to migrate
- migration_registry: where we should find all migrations to
run
"""
self.database = database
self.migration_registry = migration_registry
self._sorted_migrations = None
def _ensure_current_migration_record(self):
"""
If there isn't a database[u'app_metadata'] mediagoblin entry
with the 'current_migration', throw an error.
"""
if self.database_current_migration() is None:
raise MissingCurrentMigration(
"Tried to call function which requires "
"'current_migration' set in database")
@property
def sorted_migrations(self):
"""
Sort migrations if necessary and store in self._sorted_migrations
"""
if not self._sorted_migrations:
self._sorted_migrations = sorted(
self.migration_registry.items(),
# sort on the key... the migration number
key=lambda migration_tuple: migration_tuple[0])
return self._sorted_migrations
def latest_migration(self):
"""
Return a migration number for the latest migration, or 0 if
there are no migrations.
"""
if self.sorted_migrations:
return self.sorted_migrations[-1][0]
else:
# If no migrations have been set, we start at 0.
return 0
def set_current_migration(self, migration_number):
"""
Set the migration in the database to migration_number
"""
# Add the mediagoblin migration if necessary
self.database[u'app_metadata'].update(
{u'_id': u'mediagoblin'},
{u'$set': {u'current_migration': migration_number}},
upsert=True)
def install_migration_version_if_missing(self):
"""
Sets the migration to the latest version if no migration
version at all is set.
"""
mgoblin_metadata = self.database[u'app_metadata'].find_one(
{u'_id': u'mediagoblin'})
if not mgoblin_metadata:
latest_migration = self.latest_migration()
self.set_current_migration(latest_migration)
def database_current_migration(self):
"""
Return the current migration in the database.
"""
mgoblin_metadata = self.database[u'app_metadata'].find_one(
{u'_id': u'mediagoblin'})
if not mgoblin_metadata:
return None
else:
return mgoblin_metadata[u'current_migration']
def database_at_latest_migration(self):
"""
See if the database is at the latest migration.
Returns a boolean.
"""
current_migration = self.database_current_migration()
return current_migration == self.latest_migration()
def migrations_to_run(self):
"""
Get a list of migrations to run still, if any.
Note that calling this will set your migration version to the
latest version if it isn't installed to anything yet!
"""
self._ensure_current_migration_record()
db_current_migration = self.database_current_migration()
return [
(migration_number, migration_func)
for migration_number, migration_func in self.sorted_migrations
if migration_number > db_current_migration]
def migrate_new(self, pre_callback=None, post_callback=None):
"""
Run all migrations.
Includes two optional args:
- pre_callback: if called, this is a callback on something to
run pre-migration. Takes (migration_number, migration_func)
as arguments
- pre_callback: if called, this is a callback on something to
run post-migration. Takes (migration_number, migration_func)
as arguments
"""
# If we aren't set to any version number, presume we're at the
# latest (which means we'll do nothing here...)
self.install_migration_version_if_missing()
for migration_number, migration_func in self.migrations_to_run():
if pre_callback:
pre_callback(migration_number, migration_func)
migration_func(self.database)
self.set_current_migration(migration_number)
if post_callback:
post_callback(migration_number, migration_func)
from mediagoblin.db.mongo.util import (ObjectId, InvalidId,
DESCENDING)