heckyel/mediagoblin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Move db.sql.util to db.util

Browse Source 
Now that sqlalchemy is providing the database abstractions, there is no
need to hide everything in db.sql. sub-modules. It complicates the code
and provides a futher layer of indirection.

Move the db.sql.util.py to db.util.py and adapt the importers.
This commit is contained in:
Sebastian Spaeth 2012-11-30 09:49:45 +01:00
parent 0efe9e2796
commit 1e46dc2537
6 changed files with 313 additions and 334 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
mediagoblin/db/migrations.py
View File

@ -23,7 +23,7 @@ from sqlalchemy.exc import ProgrammingError
from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.ext.declarative import declarative_base
from migrate.changeset.constraint import UniqueConstraint from migrate.changeset.constraint import UniqueConstraint
from mediagoblin.db.sql.util import RegisterMigration from mediagoblin.db.util import RegisterMigration
from mediagoblin.db.sql.models import MediaEntry, Collection, User from mediagoblin.db.sql.models import MediaEntry, Collection, User
MIGRATIONS = {} MIGRATIONS = {}

327
mediagoblin/db/sql/util.py
View File

@ -1,327 +0,0 @@
# GNU MediaGoblin -- federated, autonomous media hosting
# Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import sys
from mediagoblin.db.sql.base import Session
from mediagoblin.db.sql.models import MediaEntry, Tag, MediaTag, Collection
from mediagoblin.tools.common import simple_printer
class MigrationManager(object):
"""
Migration handling tool.
Takes information about a database, lets you update the database
to the latest migrations, etc.
"""
def __init__(self, name, models, migration_registry, session,
printer=simple_printer):
"""
Args:
- name: identifier of this section of the database
- session: session we're going to migrate
- migration_registry: where we should find all migrations to
run
"""
self.name = unicode(name)
self.models = models
self.session = session
self.migration_registry = migration_registry
self._sorted_migrations = None
self.printer = printer
# For convenience
from mediagoblin.db.sql.models import MigrationData
self.migration_model = MigrationData
self.migration_table = MigrationData.__table__
@property
def sorted_migrations(self):
"""
Sort migrations if necessary and store in self._sorted_migrations
"""
if not self._sorted_migrations:
self._sorted_migrations = sorted(
self.migration_registry.items(),
# sort on the key... the migration number
key=lambda migration_tuple: migration_tuple[0])
return self._sorted_migrations
@property
def migration_data(self):
"""
Get the migration row associated with this object, if any.
"""
return self.session.query(
self.migration_model).filter_by(name=self.name).first()
@property
def latest_migration(self):
"""
Return a migration number for the latest migration, or 0 if
there are no migrations.
"""
if self.sorted_migrations:
return self.sorted_migrations[-1][0]
else:
# If no migrations have been set, we start at 0.
return 0
@property
def database_current_migration(self):
"""
Return the current migration in the database.
"""
# If the table doesn't even exist, return None.
if not self.migration_table.exists(self.session.bind):
return None
# Also return None if self.migration_data is None.
if self.migration_data is None:
return None
return self.migration_data.version
def set_current_migration(self, migration_number=None):
"""
Set the migration in the database to migration_number
(or, the latest available)
"""
self.migration_data.version = migration_number or self.latest_migration
self.session.commit()
def migrations_to_run(self):
"""
Get a list of migrations to run still, if any.
Note that this will fail if there's no migration record for
this class!
"""
assert self.database_current_migration is not None
db_current_migration = self.database_current_migration
return [
(migration_number, migration_func)
for migration_number, migration_func in self.sorted_migrations
if migration_number > db_current_migration]
def init_tables(self):
"""
Create all tables relative to this package
"""
# sanity check before we proceed, none of these should be created
for model in self.models:
# Maybe in the future just print out a "Yikes!" or something?
assert not model.__table__.exists(self.session.bind)
self.migration_model.metadata.create_all(
self.session.bind,
tables=[model.__table__ for model in self.models])
def create_new_migration_record(self):
"""
Create a new migration record for this migration set
"""
migration_record = self.migration_model(
name=self.name,
version=self.latest_migration)
self.session.add(migration_record)
self.session.commit()
def dry_run(self):
"""
Print out a dry run of what we would have upgraded.
"""
if self.database_current_migration is None:
self.printer(
u'~> Woulda initialized: %s\n' % self.name_for_printing())
return u'inited'
migrations_to_run = self.migrations_to_run()
if migrations_to_run:
self.printer(
u'~> Woulda updated %s:\n' % self.name_for_printing())
for migration_number, migration_func in migrations_to_run():
self.printer(
u' + Would update %s, "%s"\n' % (
migration_number, migration_func.func_name))
return u'migrated'
def name_for_printing(self):
if self.name == u'__main__':
return u"main mediagoblin tables"
else:
# TODO: Use the friendlier media manager "human readable" name
return u'media type "%s"' % self.name
def init_or_migrate(self):
"""
Initialize the database or migrate if appropriate.
Returns information about whether or not we initialized
('inited'), migrated ('migrated'), or did nothing (None)
"""
assure_migrations_table_setup(self.session)
# Find out what migration number, if any, this database data is at,
# and what the latest is.
migration_number = self.database_current_migration
# Is this our first time? Is there even a table entry for
# this identifier?
# If so:
# - create all tables
# - create record in migrations registry
# - print / inform the user
# - return 'inited'
if migration_number is None:
self.printer(u"-> Initializing %s... " % self.name_for_printing())
self.init_tables()
# auto-set at latest migration number
self.create_new_migration_record()
self.printer(u"done.\n")
self.set_current_migration()
return u'inited'
# Run migrations, if appropriate.
migrations_to_run = self.migrations_to_run()
if migrations_to_run:
self.printer(
u'-> Updating %s:\n' % self.name_for_printing())
for migration_number, migration_func in migrations_to_run:
self.printer(
u' + Running migration %s, "%s"... ' % (
migration_number, migration_func.func_name))
migration_func(self.session)
self.set_current_migration(migration_number)
self.printer('done.\n')
return u'migrated'
# Otherwise return None. Well it would do this anyway, but
# for clarity... ;)
return None
class RegisterMigration(object):
"""
Tool for registering migrations
Call like:
@RegisterMigration(33)
def update_dwarves(database):
[...]
This will register your migration with the default migration
registry. Alternately, to specify a very specific
migration_registry, you can pass in that as the second argument.
Note, the number of your migration should NEVER be 0 or less than
0. 0 is the default "no migrations" state!
"""
def __init__(self, migration_number, migration_registry):
assert migration_number > 0, "Migration number must be > 0!"
assert migration_number not in migration_registry, \
"Duplicate migration numbers detected! That's not allowed!"
self.migration_number = migration_number
self.migration_registry = migration_registry
def __call__(self, migration):
self.migration_registry[self.migration_number] = migration
return migration
def assure_migrations_table_setup(db):
"""
Make sure the migrations table is set up in the database.
"""
from mediagoblin.db.sql.models import MigrationData
if not MigrationData.__table__.exists(db.bind):
MigrationData.metadata.create_all(
db.bind, tables=[MigrationData.__table__])
##########################
# Random utility functions
##########################
def atomic_update(table, query_dict, update_values):
table.find(query_dict).update(update_values,
synchronize_session=False)
Session.commit()
def check_media_slug_used(dummy_db, uploader_id, slug, ignore_m_id):
filt = (MediaEntry.uploader == uploader_id) \
& (MediaEntry.slug == slug)
if ignore_m_id is not None:
filt = filt & (MediaEntry.id != ignore_m_id)
does_exist = Session.query(MediaEntry.id).filter(filt).first() is not None
return does_exist
def media_entries_for_tag_slug(dummy_db, tag_slug):
return MediaEntry.query \
.join(MediaEntry.tags_helper) \
.join(MediaTag.tag_helper) \
.filter(
(MediaEntry.state == u'processed')
& (Tag.slug == tag_slug))
def clean_orphan_tags(commit=True):
"""Search for unused MediaTags and delete them"""
q1 = Session.query(Tag).outerjoin(MediaTag).filter(MediaTag.id==None)
for t in q1:
Session.delete(t)
# The "let the db do all the work" version:
# q1 = Session.query(Tag.id).outerjoin(MediaTag).filter(MediaTag.id==None)
# q2 = Session.query(Tag).filter(Tag.id.in_(q1))
# q2.delete(synchronize_session = False)
if commit:
Session.commit()
def check_collection_slug_used(dummy_db, creator_id, slug, ignore_c_id):
filt = (Collection.creator == creator_id) \
& (Collection.slug == slug)
if ignore_c_id is not None:
filt = filt & (Collection.id != ignore_c_id)
does_exist = Session.query(Collection.id).filter(filt).first() is not None
return does_exist
if __name__ == '__main__':
from mediagoblin.db.open import setup_connection_and_db_from_config
db = setup_connection_and_db_from_config({'sql_engine':'sqlite:///mediagoblin.db'})
clean_orphan_tags()

312
mediagoblin/db/util.py
View File

@ -13,7 +13,313 @@
# #
# You should have received a copy of the GNU Affero General Public License # You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>. # along with this program. If not, see <http://www.gnu.org/licenses/>.
import sys
from mediagoblin.db.sql.base import Session
from mediagoblin.db.sql.models import MediaEntry, Tag, MediaTag, Collection
#TODO: check now after mongo removal if we can't rip out a layer of abstraction from mediagoblin.tools.common import simple_printer
from mediagoblin.db.sql.util import atomic_update, check_media_slug_used, \
media_entries_for_tag_slug, check_collection_slug_used
class MigrationManager(object):
"""
Migration handling tool.
Takes information about a database, lets you update the database
to the latest migrations, etc.
"""
def __init__(self, name, models, migration_registry, session,
printer=simple_printer):
"""
Args:
- name: identifier of this section of the database
- session: session we're going to migrate
- migration_registry: where we should find all migrations to
run
"""
self.name = unicode(name)
self.models = models
self.session = session
self.migration_registry = migration_registry
self._sorted_migrations = None
self.printer = printer
# For convenience
from mediagoblin.db.sql.models import MigrationData
self.migration_model = MigrationData
self.migration_table = MigrationData.__table__
@property
def sorted_migrations(self):
"""
Sort migrations if necessary and store in self._sorted_migrations
"""
if not self._sorted_migrations:
self._sorted_migrations = sorted(
self.migration_registry.items(),
# sort on the key... the migration number
key=lambda migration_tuple: migration_tuple[0])
return self._sorted_migrations
@property
def migration_data(self):
"""
Get the migration row associated with this object, if any.
"""
return self.session.query(
self.migration_model).filter_by(name=self.name).first()
@property
def latest_migration(self):
"""
Return a migration number for the latest migration, or 0 if
there are no migrations.
"""
if self.sorted_migrations:
return self.sorted_migrations[-1][0]
else:
# If no migrations have been set, we start at 0.
return 0
@property
def database_current_migration(self):
"""
Return the current migration in the database.
"""
# If the table doesn't even exist, return None.
if not self.migration_table.exists(self.session.bind):
return None
# Also return None if self.migration_data is None.
if self.migration_data is None:
return None
return self.migration_data.version
def set_current_migration(self, migration_number=None):
"""
Set the migration in the database to migration_number
(or, the latest available)
"""
self.migration_data.version = migration_number or self.latest_migration
self.session.commit()
def migrations_to_run(self):
"""
Get a list of migrations to run still, if any.
Note that this will fail if there's no migration record for
this class!
"""
assert self.database_current_migration is not None
db_current_migration = self.database_current_migration
return [
(migration_number, migration_func)
for migration_number, migration_func in self.sorted_migrations
if migration_number > db_current_migration]
def init_tables(self):
"""
Create all tables relative to this package
"""
# sanity check before we proceed, none of these should be created
for model in self.models:
# Maybe in the future just print out a "Yikes!" or something?
assert not model.__table__.exists(self.session.bind)
self.migration_model.metadata.create_all(
self.session.bind,
tables=[model.__table__ for model in self.models])
def create_new_migration_record(self):
"""
Create a new migration record for this migration set
"""
migration_record = self.migration_model(
name=self.name,
version=self.latest_migration)
self.session.add(migration_record)
self.session.commit()
def dry_run(self):
"""
Print out a dry run of what we would have upgraded.
"""
if self.database_current_migration is None:
self.printer(
u'~> Woulda initialized: %s\n' % self.name_for_printing())
return u'inited'
migrations_to_run = self.migrations_to_run()
if migrations_to_run:
self.printer(
u'~> Woulda updated %s:\n' % self.name_for_printing())
for migration_number, migration_func in migrations_to_run():
self.printer(
u' + Would update %s, "%s"\n' % (
migration_number, migration_func.func_name))
return u'migrated'
def name_for_printing(self):
if self.name == u'__main__':
return u"main mediagoblin tables"
else:
# TODO: Use the friendlier media manager "human readable" name
return u'media type "%s"' % self.name
def init_or_migrate(self):
"""
Initialize the database or migrate if appropriate.
Returns information about whether or not we initialized
('inited'), migrated ('migrated'), or did nothing (None)
"""
assure_migrations_table_setup(self.session)
# Find out what migration number, if any, this database data is at,
# and what the latest is.
migration_number = self.database_current_migration
# Is this our first time? Is there even a table entry for
# this identifier?
# If so:
# - create all tables
# - create record in migrations registry
# - print / inform the user
# - return 'inited'
if migration_number is None:
self.printer(u"-> Initializing %s... " % self.name_for_printing())
self.init_tables()
# auto-set at latest migration number
self.create_new_migration_record()
self.printer(u"done.\n")
self.set_current_migration()
return u'inited'
# Run migrations, if appropriate.
migrations_to_run = self.migrations_to_run()
if migrations_to_run:
self.printer(
u'-> Updating %s:\n' % self.name_for_printing())
for migration_number, migration_func in migrations_to_run:
self.printer(
u' + Running migration %s, "%s"... ' % (
migration_number, migration_func.func_name))
migration_func(self.session)
self.set_current_migration(migration_number)
self.printer('done.\n')
return u'migrated'
# Otherwise return None. Well it would do this anyway, but
# for clarity... ;)
return None
class RegisterMigration(object):
"""
Tool for registering migrations
Call like:
@RegisterMigration(33)
def update_dwarves(database):
[...]
This will register your migration with the default migration
registry. Alternately, to specify a very specific
migration_registry, you can pass in that as the second argument.
Note, the number of your migration should NEVER be 0 or less than
0. 0 is the default "no migrations" state!
"""
def __init__(self, migration_number, migration_registry):
assert migration_number > 0, "Migration number must be > 0!"
assert migration_number not in migration_registry, \
"Duplicate migration numbers detected! That's not allowed!"
self.migration_number = migration_number
self.migration_registry = migration_registry
def __call__(self, migration):
self.migration_registry[self.migration_number] = migration
return migration
def assure_migrations_table_setup(db):
"""
Make sure the migrations table is set up in the database.
"""
from mediagoblin.db.sql.models import MigrationData
if not MigrationData.__table__.exists(db.bind):
MigrationData.metadata.create_all(
db.bind, tables=[MigrationData.__table__])
##########################
# Random utility functions
##########################
def atomic_update(table, query_dict, update_values):
table.find(query_dict).update(update_values,
synchronize_session=False)
Session.commit()
def check_media_slug_used(dummy_db, uploader_id, slug, ignore_m_id):
filt = (MediaEntry.uploader == uploader_id) \
& (MediaEntry.slug == slug)
if ignore_m_id is not None:
filt = filt & (MediaEntry.id != ignore_m_id)
does_exist = Session.query(MediaEntry.id).filter(filt).first() is not None
return does_exist
def media_entries_for_tag_slug(dummy_db, tag_slug):
return MediaEntry.query \
.join(MediaEntry.tags_helper) \
.join(MediaTag.tag_helper) \
.filter(
(MediaEntry.state == u'processed')
& (Tag.slug == tag_slug))
def clean_orphan_tags(commit=True):
"""Search for unused MediaTags and delete them"""
q1 = Session.query(Tag).outerjoin(MediaTag).filter(MediaTag.id==None)
for t in q1:
Session.delete(t)
# The "let the db do all the work" version:
# q1 = Session.query(Tag.id).outerjoin(MediaTag).filter(MediaTag.id==None)
# q2 = Session.query(Tag).filter(Tag.id.in_(q1))
# q2.delete(synchronize_session = False)
if commit:
Session.commit()
def check_collection_slug_used(dummy_db, creator_id, slug, ignore_c_id):
filt = (Collection.creator == creator_id) \
& (Collection.slug == slug)
if ignore_c_id is not None:
filt = filt & (Collection.id != ignore_c_id)
does_exist = Session.query(Collection.id).filter(filt).first() is not None
return does_exist
if __name__ == '__main__':
from mediagoblin.db.open import setup_connection_and_db_from_config
db = setup_connection_and_db_from_config({'sql_engine':'sqlite:///mediagoblin.db'})
clean_orphan_tags()

2
mediagoblin/gmg_commands/dbupdate.py
View File

@ -19,7 +19,7 @@ import logging
from sqlalchemy.orm import sessionmaker from sqlalchemy.orm import sessionmaker
from mediagoblin.db.open import setup_connection_and_db_from_config from mediagoblin.db.open import setup_connection_and_db_from_config
from mediagoblin.db.sql.util import MigrationManager from mediagoblin.db.util import MigrationManager
from mediagoblin.init import setup_global_and_app_config from mediagoblin.init import setup_global_and_app_config
from mediagoblin.tools.common import import_component from mediagoblin.tools.common import import_component

2
mediagoblin/plugins/oauth/migrations.py
View File

@ -19,7 +19,7 @@ from sqlalchemy import (MetaData, Table, Column,
Integer, Unicode, Enum, DateTime, ForeignKey) Integer, Unicode, Enum, DateTime, ForeignKey)
from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.ext.declarative import declarative_base
from mediagoblin.db.sql.util import RegisterMigration from mediagoblin.db.util import RegisterMigration
from mediagoblin.db.sql.models import User from mediagoblin.db.sql.models import User

2
mediagoblin/tests/test_sql_migrations.py
View File

@ -26,7 +26,7 @@ from sqlalchemy.sql import select, insert
from migrate import changeset from migrate import changeset
from mediagoblin.db.sql.base import GMGTableBase from mediagoblin.db.sql.base import GMGTableBase
from mediagoblin.db.sql.util import MigrationManager, RegisterMigration from mediagoblin.db.util import MigrationManager, RegisterMigration
from mediagoblin.tools.common import CollectingPrinter from mediagoblin.tools.common import CollectingPrinter