From ed7970696e3194fdfc9fb597342e2cecacc75935 Mon Sep 17 00:00:00 2001 From: Elrond Date: Sun, 31 Jul 2011 13:54:07 +0200 Subject: [PATCH 1/5] Storage Config: Use own section Instead of configuring storage X by parameters in the main section "X_class = backend" and "X_param = value", use a new section in the config: "[storage:X]" and use "class = backend" and "param = value" there. This is the beginning, it includes a try at being backward compatible. But that try isn't really fully useful anyway. --- mediagoblin.ini | 10 +++++++--- mediagoblin/config_spec.ini | 12 +++++++----- mediagoblin/init/__init__.py | 18 ++++++++++++++++-- mediagoblin/storage.py | 7 ++++++- 4 files changed, 36 insertions(+), 11 deletions(-) diff --git a/mediagoblin.ini b/mediagoblin.ini index e889646a..c22d12d7 100644 --- a/mediagoblin.ini +++ b/mediagoblin.ini @@ -1,7 +1,4 @@ [mediagoblin] -queuestore_base_dir = %(here)s/user_dev/media/queue -publicstore_base_dir = %(here)s/user_dev/media/public -publicstore_base_url = /mgoblin_media/ direct_remote_path = /mgoblin_static/ email_sender_address = "notice@mediagoblin.example.org" @@ -14,5 +11,12 @@ allow_registration = true ## Uncomment this to put some user-overriding templates here #local_templates = %(here)s/user_dev/templates/ +[storage:queuestore] +base_dir = %(here)s/user_dev/media/queue + +[storage:publicstore] +base_dir = %(here)s/user_dev/media/public +base_url = /mgoblin_media/ + [celery] # Put celery stuff here diff --git a/mediagoblin/config_spec.ini b/mediagoblin/config_spec.ini index bbc1f7d6..6e0d52b4 100644 --- a/mediagoblin/config_spec.ini +++ b/mediagoblin/config_spec.ini @@ -4,15 +4,10 @@ db_host = string() db_name = string(default="mediagoblin") db_port = integer() -# -queuestore_base_dir = string(default="%(here)s/user_dev/media/queue") -publicstore_base_dir = string(default="%(here)s/user_dev/media/public") # Where temporary files used in processing and etc are kept workbench_path = string(default="%(here)s/user_dev/media/workbench") -# -publicstore_base_url = string(default="/mgoblin_media/") # Where mediagoblin-builtin static assets are kept direct_remote_path = string(default="/mgoblin_static/") @@ -37,6 +32,13 @@ local_templates = string() # itself) celery_setup_elsewhere = boolean(default=False) +[storage:publicstore] +base_dir = string(default="%(here)s/user_dev/media/public") +base_url = string(default="/mgoblin_media/") + +[storage:queuestore] +base_dir = string(default="%(here)s/user_dev/media/queue") + [celery] # known booleans celery_result_persistent = boolean() diff --git a/mediagoblin/init/__init__.py b/mediagoblin/init/__init__.py index ff005703..b87bd7a4 100644 --- a/mediagoblin/init/__init__.py +++ b/mediagoblin/init/__init__.py @@ -113,9 +113,23 @@ def get_staticdirector(app_config): def setup_storage(): app_config = mg_globals.app_config + global_config = mg_globals.global_config - public_store = storage_system_from_config(app_config, 'publicstore') - queue_store = storage_system_from_config(app_config, 'queuestore') + key_short = 'publicstore' + key_long = "storage:" + key_short + if global_config.has_key(key_long): + print "New style" + public_store = storage_system_from_config(global_config[key_long], None) + else: + print "old style" + public_store = storage_system_from_config(app_config, key_short) + + key_short = 'queuestore' + key_long = "storage:" + key_short + if global_config.has_key(key_long): + queue_store = storage_system_from_config(global_config[key_long], None) + else: + queue_store = storage_system_from_config(app_config, key_short) setup_globals( public_store = public_store, diff --git a/mediagoblin/storage.py b/mediagoblin/storage.py index 88c748ce..46a0c040 100644 --- a/mediagoblin/storage.py +++ b/mediagoblin/storage.py @@ -472,7 +472,10 @@ def storage_system_from_config(paste_config, storage_prefix): base_url='/media/', base_dir='/var/whatever/media') """ - prefix_re = re.compile('^%s_(.+)$' % re.escape(storage_prefix)) + if storage_prefix is not None: + prefix_re = re.compile('^%s_(.+)$' % re.escape(storage_prefix)) + else: + prefix_re = re.compile('^(.+)$') config_params = dict( [(prefix_re.match(key).groups()[0], value) @@ -485,5 +488,7 @@ def storage_system_from_config(paste_config, storage_prefix): else: storage_class = "mediagoblin.storage:BasicFileStorage" + print storage_class, repr(config_params) + storage_class = util.import_component(storage_class) return storage_class(**config_params) From f7d73aeb34ed16c7d1aa06d977ffb4c9c7a44051 Mon Sep 17 00:00:00 2001 From: Elrond Date: Sun, 7 Aug 2011 17:41:24 +0200 Subject: [PATCH 2/5] storage Config: Stop being backward compatible (beginning). Chris Webber says "no backward compatibility for this". So start removing the backward compat stuff. --- mediagoblin/init/__init__.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/mediagoblin/init/__init__.py b/mediagoblin/init/__init__.py index b87bd7a4..32f0194c 100644 --- a/mediagoblin/init/__init__.py +++ b/mediagoblin/init/__init__.py @@ -117,19 +117,11 @@ def setup_storage(): key_short = 'publicstore' key_long = "storage:" + key_short - if global_config.has_key(key_long): - print "New style" - public_store = storage_system_from_config(global_config[key_long], None) - else: - print "old style" - public_store = storage_system_from_config(app_config, key_short) + public_store = storage_system_from_config(global_config[key_long], None) key_short = 'queuestore' key_long = "storage:" + key_short - if global_config.has_key(key_long): - queue_store = storage_system_from_config(global_config[key_long], None) - else: - queue_store = storage_system_from_config(app_config, key_short) + queue_store = storage_system_from_config(global_config[key_long], None) setup_globals( public_store = public_store, From 56fc71865919792159ce4307101ef246488b4194 Mon Sep 17 00:00:00 2001 From: Elrond Date: Tue, 9 Aug 2011 13:11:34 +0200 Subject: [PATCH 3/5] Storage config: Drop all Backward Compatibility Chris Webber says not to care about backward compatibility at this stage. So drop the last bits. --- mediagoblin/init/__init__.py | 5 ++--- mediagoblin/storage.py | 14 ++++---------- 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/mediagoblin/init/__init__.py b/mediagoblin/init/__init__.py index 32f0194c..44f604b1 100644 --- a/mediagoblin/init/__init__.py +++ b/mediagoblin/init/__init__.py @@ -112,16 +112,15 @@ def get_staticdirector(app_config): def setup_storage(): - app_config = mg_globals.app_config global_config = mg_globals.global_config key_short = 'publicstore' key_long = "storage:" + key_short - public_store = storage_system_from_config(global_config[key_long], None) + public_store = storage_system_from_config(global_config[key_long]) key_short = 'queuestore' key_long = "storage:" + key_short - queue_store = storage_system_from_config(global_config[key_long], None) + queue_store = storage_system_from_config(global_config[key_long]) setup_globals( public_store = public_store, diff --git a/mediagoblin/storage.py b/mediagoblin/storage.py index 46a0c040..66acc1d4 100644 --- a/mediagoblin/storage.py +++ b/mediagoblin/storage.py @@ -443,7 +443,7 @@ def clean_listy_filepath(listy_filepath): return cleaned_filepath -def storage_system_from_config(paste_config, storage_prefix): +def storage_system_from_config(config): """ Utility for setting up a storage system from the paste app config. @@ -472,15 +472,9 @@ def storage_system_from_config(paste_config, storage_prefix): base_url='/media/', base_dir='/var/whatever/media') """ - if storage_prefix is not None: - prefix_re = re.compile('^%s_(.+)$' % re.escape(storage_prefix)) - else: - prefix_re = re.compile('^(.+)$') - - config_params = dict( - [(prefix_re.match(key).groups()[0], value) - for key, value in paste_config.iteritems() - if prefix_re.match(key)]) + # This construct is needed, because dict(config) does + # not replace the variables in the config items. + config_params = dict(config.iteritems()) if 'storage_class' in config_params: storage_class = config_params['storage_class'] From f9b68f6efa8624a425464a62d5a836167f5e000e Mon Sep 17 00:00:00 2001 From: Elrond Date: Tue, 9 Aug 2011 13:40:20 +0200 Subject: [PATCH 4/5] Storage Config: Finally drop debug --- mediagoblin/storage.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/mediagoblin/storage.py b/mediagoblin/storage.py index 66acc1d4..d338fb31 100644 --- a/mediagoblin/storage.py +++ b/mediagoblin/storage.py @@ -482,7 +482,5 @@ def storage_system_from_config(config): else: storage_class = "mediagoblin.storage:BasicFileStorage" - print storage_class, repr(config_params) - storage_class = util.import_component(storage_class) return storage_class(**config_params) From 758eb746547cc8c6c52135fd88bee11b5f81d3d4 Mon Sep 17 00:00:00 2001 From: Elrond Date: Fri, 12 Aug 2011 13:01:41 +0200 Subject: [PATCH 5/5] Document changes to storage_system_from_config Chris suggested changing the docs for storage_system_from_config: - The only param is a config section. - The format of that section is much simpler, no prefix. --- mediagoblin/storage.py | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/mediagoblin/storage.py b/mediagoblin/storage.py index d338fb31..ec3bc1b5 100644 --- a/mediagoblin/storage.py +++ b/mediagoblin/storage.py @@ -15,7 +15,6 @@ # along with this program. If not, see . import os -import re import shutil import urlparse import uuid @@ -443,29 +442,25 @@ def clean_listy_filepath(listy_filepath): return cleaned_filepath -def storage_system_from_config(config): +def storage_system_from_config(config_section): """ - Utility for setting up a storage system from the paste app config. + Utility for setting up a storage system from a config section. - Note that a special argument may be passed in to the paste_config - which is "${storage_prefix}_storage_class" which will provide an + Note that a special argument may be passed in to + the config_section which is "storage_class" which will provide an import path to a storage system. This defaults to "mediagoblin.storage:BasicFileStorage" if otherwise undefined. Arguments: - - paste_config: dictionary of config parameters - - storage_prefix: the storage system we're setting up / will be - getting keys/arguments from. For example 'publicstore' will - grab all arguments that are like 'publicstore_FOO'. + - config_section: dictionary of config parameters Returns: An instantiated storage system. Example: storage_system_from_config( - {'publicstore_base_url': '/media/', - 'publicstore_base_dir': '/var/whatever/media/'}, - 'publicstore') + {'base_url': '/media/', + 'base_dir': '/var/whatever/media/'}) Will return: BasicFileStorage( @@ -474,7 +469,7 @@ def storage_system_from_config(config): """ # This construct is needed, because dict(config) does # not replace the variables in the config items. - config_params = dict(config.iteritems()) + config_params = dict(config_section.iteritems()) if 'storage_class' in config_params: storage_class = config_params['storage_class']