From f2b2008da51ff554df1af0e5a14a73aff4d89c33 Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Mon, 29 Jul 2013 16:36:06 -0400 Subject: [PATCH 1/5] This was a very simple ticket actually. I created a list called FOUNDATIONS in mediagoblin/db/models.py. This list holds all of the information about rows that should be created at database initialization. Read the documentation near the FOUNDATIONS list to understand the proper format for this list. All of the work is done through a new method on MigrationManager in mediagoblin/db/migrations_tools.py. This method, `populate_table_foundations` parses the FOUNDATIONS list and creates the foundations based on the data incl- uded. This only ever happens when the database is initialized. Migrations to releases with new Foundations should be very easy just using the basic database functionality. --- mediagoblin/db/migration_tools.py | 16 +++++++++++++++- mediagoblin/db/models.py | 15 +++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/mediagoblin/db/migration_tools.py b/mediagoblin/db/migration_tools.py index aa22ef94..1192836d 100644 --- a/mediagoblin/db/migration_tools.py +++ b/mediagoblin/db/migration_tools.py @@ -140,6 +140,18 @@ class MigrationManager(object): self.session.bind, tables=[model.__table__ for model in self.models]) + def populate_table_foundations(self): + """ + Create the table foundations (default rows) as layed out in FOUNDATIONS + in mediagoblin.db.models + """ + from mediagoblin.db.models import FOUNDATIONS as MAIN_FOUNDATIONS + for Model, rows in MAIN_FOUNDATIONS.items(): + print u'\n--> Laying foundations for %s table' % Model.__name__ + for parameters in rows: + row = Model(**parameters) + row.save() + def create_new_migration_record(self): """ Create a new migration record for this migration set @@ -202,7 +214,9 @@ class MigrationManager(object): self.init_tables() # auto-set at latest migration number - self.create_new_migration_record() + self.create_new_migration_record() + if self.name==u'__main__': + self.populate_table_foundations() self.printer(u"done.\n") self.set_current_migration() diff --git a/mediagoblin/db/models.py b/mediagoblin/db/models.py index 826d47ba..7448f5ce 100644 --- a/mediagoblin/db/models.py +++ b/mediagoblin/db/models.py @@ -585,6 +585,21 @@ MODELS = [ Notification, CommentNotification, ProcessingNotification, CommentSubscription] +""" + Foundations are the default rows that are created immediately after the tables + are initialized. Each entry to this dictionary should be in the format of: + ModelConstructorObject:List of Dictionaries + (Each Dictionary represents a row on the Table to be created, containing each + of the columns' names as a key string, and each of the columns' values as a + value) + + ex. [NOTE THIS IS NOT BASED OFF OF OUR USER TABLE] + user_foundations = [{'name':u'Joanna', 'age':24}, + {'name':u'Andrea', 'age':41}] + + FOUNDATIONS = {User:user_foundations} +""" +FOUNDATIONS = {} ###################################################### # Special, migrations-tracking table From 08cd10d84fd89df3f0cad3835ba8ab8b8000d4b2 Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Mon, 29 Jul 2013 17:15:29 -0400 Subject: [PATCH 2/5] I actually had to do a bit more work than I thought, because I needed to account for plugins. In this commit I changed the MigrationManager and DatabaseData ob- jects to account for FOUNDATIONS in any plugin's (or main program's) models.py file. --- mediagoblin/db/migration_tools.py | 15 +++++++-------- mediagoblin/gmg_commands/dbupdate.py | 25 ++++++++++++++++++++----- 2 files changed, 27 insertions(+), 13 deletions(-) diff --git a/mediagoblin/db/migration_tools.py b/mediagoblin/db/migration_tools.py index 1192836d..ad137683 100644 --- a/mediagoblin/db/migration_tools.py +++ b/mediagoblin/db/migration_tools.py @@ -29,7 +29,7 @@ class MigrationManager(object): to the latest migrations, etc. """ - def __init__(self, name, models, migration_registry, session, + def __init__(self, name, models, foundations, migration_registry, session, printer=simple_printer): """ Args: @@ -40,6 +40,7 @@ class MigrationManager(object): """ self.name = unicode(name) self.models = models + self.foundations = foundations self.session = session self.migration_registry = migration_registry self._sorted_migrations = None @@ -145,12 +146,11 @@ class MigrationManager(object): Create the table foundations (default rows) as layed out in FOUNDATIONS in mediagoblin.db.models """ - from mediagoblin.db.models import FOUNDATIONS as MAIN_FOUNDATIONS - for Model, rows in MAIN_FOUNDATIONS.items(): - print u'\n--> Laying foundations for %s table' % Model.__name__ + for Model, rows in self.foundations.items(): + print u'\n + Laying foundations for %s table' % (Model.__name__) for parameters in rows: - row = Model(**parameters) - row.save() + new_row = Model(**parameters) + new_row.save() def create_new_migration_record(self): """ @@ -215,8 +215,7 @@ class MigrationManager(object): self.init_tables() # auto-set at latest migration number self.create_new_migration_record() - if self.name==u'__main__': - self.populate_table_foundations() + self.populate_table_foundations() self.printer(u"done.\n") self.set_current_migration() diff --git a/mediagoblin/gmg_commands/dbupdate.py b/mediagoblin/gmg_commands/dbupdate.py index 00007567..bad3e352 100644 --- a/mediagoblin/gmg_commands/dbupdate.py +++ b/mediagoblin/gmg_commands/dbupdate.py @@ -32,14 +32,15 @@ def dbupdate_parse_setup(subparser): class DatabaseData(object): - def __init__(self, name, models, migrations): + def __init__(self, name, models, foundations, migrations): self.name = name self.models = models + self.foundations = foundations self.migrations = migrations def make_migration_manager(self, session): return MigrationManager( - self.name, self.models, self.migrations, session) + self.name, self.models, self.foundations, self.migrations, session) def gather_database_data(plugins): @@ -54,10 +55,11 @@ def gather_database_data(plugins): # Add main first from mediagoblin.db.models import MODELS as MAIN_MODELS from mediagoblin.db.migrations import MIGRATIONS as MAIN_MIGRATIONS + from mediagoblin.db.models import FOUNDATIONS as MAIN_FOUNDATIONS managed_dbdata.append( DatabaseData( - u'__main__', MAIN_MODELS, MAIN_MIGRATIONS)) + u'__main__', MAIN_MODELS, MAIN_FOUNDATIONS, MAIN_MIGRATIONS)) for plugin in plugins: try: @@ -83,13 +85,26 @@ forgotten to add it? ({1})'.format(plugin, exc)) migrations = {} except AttributeError as exc: - _log.debug('Cloud not find MIGRATIONS in {0}.migrations, have you \ + _log.debug('Could not find MIGRATIONS in {0}.migrations, have you \ forgotten to add it? ({1})'.format(plugin, exc)) migrations = {} + try: + foundations = import_component('{0}.models:FOUNDATIONS'.format(plugin)) + except ImportError as exc: + _log.debug('No foundations found for {0}: {1}'.format( + plugin, + exc)) + + foundations = [] + except AttributeError as exc: + _log.debug('Could not find FOUNDATIONS in {0}.models, have you \ +forgotten to add it? ({1})'.format(plugin, exc)) + foundations = {} + if models: managed_dbdata.append( - DatabaseData(plugin, models, migrations)) + DatabaseData(plugin, models, foundations, migrations)) return managed_dbdata From 63c3ca28abab7f12592bc3e8bcd0b05749cd0053 Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Tue, 30 Jul 2013 19:06:26 -0400 Subject: [PATCH 3/5] Starting to write unit tests... --- mediagoblin/db/migration_tools.py | 8 ++++---- mediagoblin/tests/test_sql_migrations.py | 13 ++++++++----- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/mediagoblin/db/migration_tools.py b/mediagoblin/db/migration_tools.py index ad137683..e75f3757 100644 --- a/mediagoblin/db/migration_tools.py +++ b/mediagoblin/db/migration_tools.py @@ -147,10 +147,11 @@ class MigrationManager(object): in mediagoblin.db.models """ for Model, rows in self.foundations.items(): - print u'\n + Laying foundations for %s table' % (Model.__name__) + self.printer(u' + Laying foundations for %s table\n' % + (Model.__name__)) for parameters in rows: new_row = Model(**parameters) - new_row.save() + self.session.add(new_row) def create_new_migration_record(self): """ @@ -215,9 +216,8 @@ class MigrationManager(object): self.init_tables() # auto-set at latest migration number self.create_new_migration_record() - self.populate_table_foundations() - self.printer(u"done.\n") + self.populate_table_foundations() self.set_current_migration() return u'inited' diff --git a/mediagoblin/tests/test_sql_migrations.py b/mediagoblin/tests/test_sql_migrations.py index 2fc4c043..86bb989a 100644 --- a/mediagoblin/tests/test_sql_migrations.py +++ b/mediagoblin/tests/test_sql_migrations.py @@ -58,6 +58,9 @@ class Level1(Base1): SET1_MODELS = [Creature1, Level1] +SET1_FOUNDATIONS = {Creature1:[{'name':u'goblin','num_legs':2,'is_demon':False}, + {'name':u'cerberus','num_legs':4,'is_demon':True}]} + SET1_MIGRATIONS = {} ####################################################### @@ -542,7 +545,6 @@ def _insert_migration3_objects(session): session.commit() - def create_test_engine(): from sqlalchemy import create_engine engine = create_engine('sqlite:///:memory:', echo=False) @@ -572,7 +574,7 @@ def test_set1_to_set3(): printer = CollectingPrinter() migration_manager = MigrationManager( - u'__main__', SET1_MODELS, SET1_MIGRATIONS, Session(), + u'__main__', SET1_MODELS, SET1_FOUNDATIONS, SET1_MIGRATIONS, Session(), printer) # Check latest migration and database current migration @@ -585,7 +587,8 @@ def test_set1_to_set3(): assert result == u'inited' # Check output assert printer.combined_string == ( - "-> Initializing main mediagoblin tables... done.\n") + "-> Initializing main mediagoblin tables... done.\n" + \ + " + Laying foundations for Creature1 table\n" ) # Check version in database assert migration_manager.latest_migration == 0 assert migration_manager.database_current_migration == 0 @@ -597,8 +600,8 @@ def test_set1_to_set3(): # Try to "re-migrate" with same manager settings... nothing should happen migration_manager = MigrationManager( - u'__main__', SET1_MODELS, SET1_MIGRATIONS, Session(), - printer) + u'__main__', SET1_MODELS, SET1_FOUNDATIONS, SET1_MIGRATIONS, + Session(), printer) assert migration_manager.init_or_migrate() == None # Check version in database From 860fa806ee3af0260b9f8f6f02dd606fb55678f5 Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Tue, 30 Jul 2013 19:42:26 -0400 Subject: [PATCH 4/5] In this commit I added a few unittests to account for Foundations. There were only a few tests I had to add to mediagoblin/tests/test_sql_migrations.py beca- -use the foundation creation only happens at database initialization. --- mediagoblin/tests/test_sql_migrations.py | 27 ++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/mediagoblin/tests/test_sql_migrations.py b/mediagoblin/tests/test_sql_migrations.py index 86bb989a..a004aa43 100644 --- a/mediagoblin/tests/test_sql_migrations.py +++ b/mediagoblin/tests/test_sql_migrations.py @@ -58,8 +58,8 @@ class Level1(Base1): SET1_MODELS = [Creature1, Level1] -SET1_FOUNDATIONS = {Creature1:[{'name':u'goblin','num_legs':2,'is_demon':False}, - {'name':u'cerberus','num_legs':4,'is_demon':True}]} +FOUNDATIONS = {Creature1:[{'name':u'goblin','num_legs':2,'is_demon':False}, + {'name':u'cerberus','num_legs':4,'is_demon':True}]} SET1_MIGRATIONS = {} @@ -574,7 +574,7 @@ def test_set1_to_set3(): printer = CollectingPrinter() migration_manager = MigrationManager( - u'__main__', SET1_MODELS, SET1_FOUNDATIONS, SET1_MIGRATIONS, Session(), + u'__main__', SET1_MODELS, FOUNDATIONS, SET1_MIGRATIONS, Session(), printer) # Check latest migration and database current migration @@ -593,6 +593,7 @@ def test_set1_to_set3(): assert migration_manager.latest_migration == 0 assert migration_manager.database_current_migration == 0 + # Install the initial set # ----------------------- @@ -600,7 +601,7 @@ def test_set1_to_set3(): # Try to "re-migrate" with same manager settings... nothing should happen migration_manager = MigrationManager( - u'__main__', SET1_MODELS, SET1_FOUNDATIONS, SET1_MIGRATIONS, + u'__main__', SET1_MODELS, FOUNDATIONS, SET1_MIGRATIONS, Session(), printer) assert migration_manager.init_or_migrate() == None @@ -642,6 +643,20 @@ def test_set1_to_set3(): # Now check to see if stuff seems to be in there. session = Session() + # Check the creation of the foundation rows on the creature table + creature = session.query(Creature1).filter_by( + name=u'goblin').one() + assert creature.num_legs == 2 + assert creature.is_demon == False + + creature = session.query(Creature1).filter_by( + name=u'cerberus').one() + assert creature.num_legs == 4 + assert creature.is_demon == True + + + # Check the creation of the inserted rows on the creature and levels tables + creature = session.query(Creature1).filter_by( name=u'centipede').one() assert creature.num_legs == 100 @@ -682,7 +697,7 @@ def test_set1_to_set3(): # isn't said to be updated yet printer = CollectingPrinter() migration_manager = MigrationManager( - u'__main__', SET3_MODELS, SET3_MIGRATIONS, Session(), + u'__main__', SET3_MODELS, FOUNDATIONS, SET3_MIGRATIONS, Session(), printer) assert migration_manager.latest_migration == 8 @@ -709,7 +724,7 @@ def test_set1_to_set3(): # Make sure version matches expected migration_manager = MigrationManager( - u'__main__', SET3_MODELS, SET3_MIGRATIONS, Session(), + u'__main__', SET3_MODELS, FOUNDATIONS, SET3_MIGRATIONS, Session(), printer) assert migration_manager.latest_migration == 8 assert migration_manager.database_current_migration == 8 From 84c1cd7c52e7a07f2e0605433f255d946fe2737a Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Wed, 31 Jul 2013 20:50:38 -0400 Subject: [PATCH 5/5] I added a few more unitests in this commit. It now confirms that even after mi- -gration, there is only one of each Foundation object. --- mediagoblin/tests/test_sql_migrations.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/mediagoblin/tests/test_sql_migrations.py b/mediagoblin/tests/test_sql_migrations.py index a004aa43..3d67fdf6 100644 --- a/mediagoblin/tests/test_sql_migrations.py +++ b/mediagoblin/tests/test_sql_migrations.py @@ -59,7 +59,8 @@ class Level1(Base1): SET1_MODELS = [Creature1, Level1] FOUNDATIONS = {Creature1:[{'name':u'goblin','num_legs':2,'is_demon':False}, - {'name':u'cerberus','num_legs':4,'is_demon':True}]} + {'name':u'cerberus','num_legs':4,'is_demon':True}] + } SET1_MIGRATIONS = {} @@ -790,6 +791,15 @@ def test_set1_to_set3(): # Now check to see if stuff seems to be in there. session = Session() + + + # Start with making sure that the foundations did not run again + assert session.query(Creature3).filter_by( + name=u'goblin').count() == 1 + assert session.query(Creature3).filter_by( + name=u'cerberus').count() == 1 + + # Then make sure the models have been migrated correctly creature = session.query(Creature3).filter_by( name=u'centipede').one() assert creature.num_limbs == 100.0