Merge branch 'configobj'
This commit is contained in:
Christopher Allan Webber
commit
fa4e6b279b
@ -125,7 +125,7 @@ Running the server
|
||||
|
||||
Run::
|
||||
|
||||
./bin/paster serve mediagoblin.ini --reload
|
||||
./bin/paster serve server.ini --reload
|
||||
|
||||
|
||||
Running celeryd
|
||||
@ -143,7 +143,7 @@ Run::
|
||||
Too much work? Don't want to run an http server and celeryd at the
|
||||
same time? For development purposes there's a shortcut::
|
||||
|
||||
CELERY_ALWAYS_EAGER=true ./bin/paster serve mediagoblin.ini --reload
|
||||
CELERY_ALWAYS_EAGER=true ./bin/paster serve server.ini --reload
|
||||
|
||||
This way the web server will block on processing items until they are
|
||||
done, but you don't need to run celery separately (which is probably
|
||||
@ -156,7 +156,7 @@ Running the test suite
|
||||
|
||||
Run::
|
||||
|
||||
CELERY_CONFIG_MODULE=mediagoblin.celery_setup.from_tests ./bin/nosetests
|
||||
./bin/nosetests
|
||||
|
||||
|
||||
Running a shell
|
||||
|
||||
@ -1,42 +1,15 @@
|
||||
[DEFAULT]
|
||||
debug = true
|
||||
|
||||
[composite:main]
|
||||
use = egg:Paste#urlmap
|
||||
/ = mediagoblin
|
||||
/mgoblin_media/ = publicstore_serve
|
||||
/mgoblin_static/ = mediagoblin_static
|
||||
|
||||
[app:mediagoblin]
|
||||
use = egg:mediagoblin#app
|
||||
filter-with = beaker
|
||||
[mediagoblin]
|
||||
queuestore_base_dir = %(here)s/user_dev/media/queue
|
||||
publicstore_base_dir = %(here)s/user_dev/media/public
|
||||
publicstore_base_url = /mgoblin_media/
|
||||
direct_remote_path = /mgoblin_static/
|
||||
email_sender_address = "notice@mediagoblin.example.org"
|
||||
|
||||
# set to false to enable sending notices
|
||||
email_debug_mode = true
|
||||
|
||||
## Uncomment this to put some user-overriding templates here
|
||||
#local_templates = %(here)s/user_dev/templates/
|
||||
|
||||
[app:publicstore_serve]
|
||||
use = egg:Paste#static
|
||||
document_root = %(here)s/user_dev/media/public
|
||||
|
||||
[app:mediagoblin_static]
|
||||
use = egg:Paste#static
|
||||
document_root = %(here)s/mediagoblin/static/
|
||||
|
||||
[filter:beaker]
|
||||
use = egg:Beaker#beaker_session
|
||||
cache_dir = %(here)s/user_dev/beaker
|
||||
beaker.session.key = mediagoblin
|
||||
# beaker.session.secret = somesupersecret
|
||||
beaker.session.data_dir = %(here)s/user_dev/beaker/sessions/data
|
||||
beaker.session.lock_dir = %(here)s/user_dev/beaker/sessions/lock
|
||||
|
||||
[server:main]
|
||||
use = egg:Paste#http
|
||||
host = 127.0.0.1
|
||||
port = 6543
|
||||
[celery]
|
||||
# Put celery stuff here
|
||||
|
||||
@ -18,14 +18,15 @@ import os
|
||||
import urllib
|
||||
|
||||
import routes
|
||||
from paste.deploy.converters import asbool
|
||||
from webob import Request, exc
|
||||
|
||||
from mediagoblin import routing, util, storage, staticdirect
|
||||
from mediagoblin.config import (
|
||||
read_mediagoblin_config, generate_validation_report)
|
||||
from mediagoblin.db.open import setup_connection_and_db_from_config
|
||||
from mediagoblin.mg_globals import setup_globals
|
||||
from mediagoblin.celery_setup import setup_celery_from_config
|
||||
from mediagoblin.workbench import WorkbenchManager, DEFAULT_WORKBENCH_DIR
|
||||
from mediagoblin.workbench import WorkbenchManager
|
||||
|
||||
|
||||
class Error(Exception): pass
|
||||
@ -34,42 +35,102 @@ class ImproperlyConfigured(Error): pass
|
||||
|
||||
class MediaGoblinApp(object):
|
||||
"""
|
||||
Really basic wsgi app using routes and WebOb.
|
||||
WSGI application of MediaGoblin
|
||||
|
||||
... this is the heart of the program!
|
||||
"""
|
||||
def __init__(self, connection, db,
|
||||
public_store, queue_store,
|
||||
staticdirector,
|
||||
email_sender_address, email_debug_mode,
|
||||
user_template_path=None,
|
||||
workbench_path=DEFAULT_WORKBENCH_DIR):
|
||||
def __init__(self, config_path, setup_celery=True):
|
||||
"""
|
||||
Initialize the application based on a configuration file.
|
||||
|
||||
Arguments:
|
||||
- config_path: path to the configuration file we're opening.
|
||||
- setup_celery: whether or not to setup celery during init.
|
||||
(Note: setting 'celery_setup_elsewhere' also disables
|
||||
setting up celery.)
|
||||
"""
|
||||
##############
|
||||
# Setup config
|
||||
##############
|
||||
|
||||
# Open and setup the config
|
||||
global_config, validation_result = read_mediagoblin_config(config_path)
|
||||
app_config = global_config['mediagoblin']
|
||||
# report errors if necessary
|
||||
validation_report = generate_validation_report(
|
||||
global_config, validation_result)
|
||||
if validation_report:
|
||||
raise ImproperlyConfigured(validation_report)
|
||||
|
||||
##########################################
|
||||
# Setup other connections / useful objects
|
||||
##########################################
|
||||
|
||||
# Set up the database
|
||||
self.connection, self.db = setup_connection_and_db_from_config(
|
||||
app_config)
|
||||
|
||||
# Get the template environment
|
||||
self.template_loader = util.get_jinja_loader(user_template_path)
|
||||
self.template_loader = util.get_jinja_loader(
|
||||
app_config.get('user_template_path'))
|
||||
|
||||
# Set up storage systems
|
||||
self.public_store = public_store
|
||||
self.queue_store = queue_store
|
||||
|
||||
# Set up database
|
||||
self.connection = connection
|
||||
self.db = db
|
||||
self.public_store = storage.storage_system_from_config(
|
||||
app_config, 'publicstore')
|
||||
self.queue_store = storage.storage_system_from_config(
|
||||
app_config, 'queuestore')
|
||||
|
||||
# set up routing
|
||||
self.routing = routing.get_mapper()
|
||||
|
||||
# set up staticdirector tool
|
||||
self.staticdirector = staticdirector
|
||||
if app_config.has_key('direct_remote_path'):
|
||||
self.staticdirector = staticdirect.RemoteStaticDirect(
|
||||
app_config['direct_remote_path'].strip())
|
||||
elif app_config.has_key('direct_remote_paths'):
|
||||
direct_remote_path_lines = app_config[
|
||||
'direct_remote_paths'].strip().splitlines()
|
||||
self.staticdirector = staticdirect.MultiRemoteStaticDirect(
|
||||
dict([line.strip().split(' ', 1)
|
||||
for line in direct_remote_path_lines]))
|
||||
else:
|
||||
raise ImproperlyConfigured(
|
||||
"One of direct_remote_path or "
|
||||
"direct_remote_paths must be provided")
|
||||
|
||||
# Setup celery, if appropriate
|
||||
if setup_celery and not app_config.get('celery_setup_elsewhere'):
|
||||
if os.environ.get('CELERY_ALWAYS_EAGER'):
|
||||
setup_celery_from_config(
|
||||
app_config, global_config,
|
||||
force_celery_always_eager=True)
|
||||
else:
|
||||
setup_celery_from_config(app_config, global_config)
|
||||
|
||||
#######################################################
|
||||
# Insert appropriate things into mediagoblin.mg_globals
|
||||
#
|
||||
# certain properties need to be accessed globally eg from
|
||||
# validators, etc, which might not access to the request
|
||||
# object.
|
||||
#######################################################
|
||||
|
||||
setup_globals(
|
||||
email_sender_address=email_sender_address,
|
||||
email_debug_mode=email_debug_mode,
|
||||
db_connection=connection,
|
||||
app_config=app_config,
|
||||
global_config=global_config,
|
||||
|
||||
# TODO: No need to set these two up as globals, we could
|
||||
# just read them out of mg_globals.app_config
|
||||
email_sender_address=app_config['email_sender_address'],
|
||||
email_debug_mode=app_config['email_debug_mode'],
|
||||
|
||||
# Actual, useful to everyone objects
|
||||
app=self,
|
||||
db_connection=self.connection,
|
||||
database=self.db,
|
||||
public_store=self.public_store,
|
||||
queue_store=self.queue_store,
|
||||
workbench_manager=WorkbenchManager(workbench_path))
|
||||
workbench_manager=WorkbenchManager(app_config['workbench_path']))
|
||||
|
||||
def __call__(self, environ, start_response):
|
||||
request = Request(environ)
|
||||
@ -119,45 +180,6 @@ class MediaGoblinApp(object):
|
||||
|
||||
|
||||
def paste_app_factory(global_config, **app_config):
|
||||
# Get the database connection
|
||||
connection, db = setup_connection_and_db_from_config(app_config)
|
||||
|
||||
# Set up the storage systems.
|
||||
public_store = storage.storage_system_from_paste_config(
|
||||
app_config, 'publicstore')
|
||||
queue_store = storage.storage_system_from_paste_config(
|
||||
app_config, 'queuestore')
|
||||
|
||||
# Set up the staticdirect system
|
||||
if app_config.has_key('direct_remote_path'):
|
||||
staticdirector = staticdirect.RemoteStaticDirect(
|
||||
app_config['direct_remote_path'].strip())
|
||||
elif app_config.has_key('direct_remote_paths'):
|
||||
direct_remote_path_lines = app_config[
|
||||
'direct_remote_paths'].strip().splitlines()
|
||||
staticdirector = staticdirect.MultiRemoteStaticDirect(
|
||||
dict([line.strip().split(' ', 1)
|
||||
for line in direct_remote_path_lines]))
|
||||
else:
|
||||
raise ImproperlyConfigured(
|
||||
"One of direct_remote_path or direct_remote_paths must be provided")
|
||||
|
||||
if not asbool(app_config.get('celery_setup_elsewhere')):
|
||||
if asbool(os.environ.get('CELERY_ALWAYS_EAGER')):
|
||||
setup_celery_from_config(
|
||||
app_config, global_config,
|
||||
force_celery_always_eager=True)
|
||||
else:
|
||||
setup_celery_from_config(app_config, global_config)
|
||||
|
||||
mgoblin_app = MediaGoblinApp(
|
||||
connection, db,
|
||||
public_store=public_store, queue_store=queue_store,
|
||||
staticdirector=staticdirector,
|
||||
email_sender_address=app_config.get(
|
||||
'email_sender_address', 'notice@mediagoblin.example.org'),
|
||||
email_debug_mode=asbool(app_config.get('email_debug_mode')),
|
||||
user_template_path=app_config.get('local_templates'),
|
||||
workbench_path=app_config.get('workbench_path', DEFAULT_WORKBENCH_DIR))
|
||||
mgoblin_app = MediaGoblinApp(app_config['config'])
|
||||
|
||||
return mgoblin_app
|
||||
|
||||
@ -17,86 +17,35 @@
|
||||
import os
|
||||
import sys
|
||||
|
||||
from paste.deploy.converters import asbool, asint, aslist
|
||||
|
||||
|
||||
KNOWN_CONFIG_BOOLS = [
|
||||
'CELERY_RESULT_PERSISTENT',
|
||||
'CELERY_CREATE_MISSING_QUEUES',
|
||||
'BROKER_USE_SSL', 'BROKER_CONNECTION_RETRY',
|
||||
'CELERY_ALWAYS_EAGER', 'CELERY_EAGER_PROPAGATES_EXCEPTIONS',
|
||||
'CELERY_IGNORE_RESULT', 'CELERY_TRACK_STARTED',
|
||||
'CELERY_DISABLE_RATE_LIMITS', 'CELERY_ACKS_LATE',
|
||||
'CELERY_STORE_ERRORS_EVEN_IF_IGNORED',
|
||||
'CELERY_SEND_TASK_ERROR_EMAILS',
|
||||
'CELERY_SEND_EVENTS', 'CELERY_SEND_TASK_SENT_EVENT',
|
||||
'CELERYD_LOG_COLOR', 'CELERY_REDIRECT_STDOUTS',
|
||||
]
|
||||
|
||||
KNOWN_CONFIG_INTS = [
|
||||
'CELERYD_CONCURRENCY',
|
||||
'CELERYD_PREFETCH_MULTIPLIER',
|
||||
'CELERY_AMQP_TASK_RESULT_EXPIRES',
|
||||
'CELERY_AMQP_TASK_RESULT_CONNECTION_MAX',
|
||||
'REDIS_PORT', 'REDIS_DB',
|
||||
'BROKER_PORT', 'BROKER_CONNECTION_TIMEOUT',
|
||||
'CELERY_BROKER_CONNECTION_MAX_RETRIES',
|
||||
'CELERY_TASK_RESULT_EXPIRES', 'CELERY_MAX_CACHED_RESULTS',
|
||||
'CELERY_DEFAULT_RATE_LIMIT', # ??
|
||||
'CELERYD_MAX_TASKS_PER_CHILD', 'CELERYD_TASK_TIME_LIMIT',
|
||||
'CELERYD_TASK_SOFT_TIME_LIMIT',
|
||||
'MAIL_PORT', 'CELERYBEAT_MAX_LOOP_INTERVAL',
|
||||
]
|
||||
|
||||
KNOWN_CONFIG_FLOATS = [
|
||||
'CELERYD_ETA_SCHEDULER_PRECISION',
|
||||
]
|
||||
|
||||
KNOWN_CONFIG_LISTS = [
|
||||
'CELERY_ROUTES', 'CELERY_IMPORTS',
|
||||
]
|
||||
|
||||
|
||||
## Needs special processing:
|
||||
# ADMINS, ???
|
||||
# there are a lot more; we should list here or process specially.
|
||||
|
||||
|
||||
def asfloat(obj):
|
||||
try:
|
||||
return float(obj)
|
||||
except (TypeError, ValueError), e:
|
||||
raise ValueError(
|
||||
"Bad float value: %r" % obj)
|
||||
|
||||
|
||||
MANDATORY_CELERY_IMPORTS = ['mediagoblin.process_media']
|
||||
|
||||
DEFAULT_SETTINGS_MODULE = 'mediagoblin.celery_setup.dummy_settings_module'
|
||||
|
||||
|
||||
def setup_celery_from_config(app_config, global_config,
|
||||
settings_module=DEFAULT_SETTINGS_MODULE,
|
||||
force_celery_always_eager=False,
|
||||
set_environ=True):
|
||||
"""
|
||||
Take a mediagoblin app config and the global config from a paste
|
||||
factory and try to set up a celery settings module from this.
|
||||
Take a mediagoblin app config and try to set up a celery settings
|
||||
module from this.
|
||||
|
||||
Args:
|
||||
- app_config: the application config section
|
||||
- global_config: the entire paste config, all sections
|
||||
- global_config: the entire ConfigObj loaded config, all sections
|
||||
- settings_module: the module to populate, as a string
|
||||
-
|
||||
- force_celery_always_eager: whether or not to force celery into
|
||||
always eager mode; good for development and small installs
|
||||
- set_environ: if set, this will CELERY_CONFIG_MODULE to the
|
||||
settings_module
|
||||
"""
|
||||
if asbool(app_config.get('use_celery_environment_var')) == True:
|
||||
if app_config.get('celery_setup_elsewhere') == True:
|
||||
# Don't setup celery based on our config file.
|
||||
return
|
||||
|
||||
celery_conf_section = app_config.get('celery_section', 'celery')
|
||||
if global_config.has_key(celery_conf_section):
|
||||
celery_conf = global_config[celery_conf_section]
|
||||
if global_config.has_key('celery'):
|
||||
celery_conf = global_config['celery']
|
||||
else:
|
||||
celery_conf = {}
|
||||
|
||||
@ -114,9 +63,9 @@ def setup_celery_from_config(app_config, global_config,
|
||||
if celery_settings['BROKER_BACKEND'] == 'mongodb':
|
||||
celery_settings['BROKER_HOST'] = app_config['db_host']
|
||||
if app_config.has_key('db_port'):
|
||||
celery_mongo_settings['port'] = asint(app_config['db_port'])
|
||||
celery_mongo_settings['port'] = app_config['db_port']
|
||||
if celery_settings['BROKER_BACKEND'] == 'mongodb':
|
||||
celery_settings['BROKER_PORT'] = asint(app_config['db_port'])
|
||||
celery_settings['BROKER_PORT'] = app_config['db_port']
|
||||
celery_mongo_settings['database'] = app_config.get('db_name', 'mediagoblin')
|
||||
|
||||
celery_settings['CELERY_MONGODB_BACKEND_SETTINGS'] = celery_mongo_settings
|
||||
@ -124,14 +73,6 @@ def setup_celery_from_config(app_config, global_config,
|
||||
# Add anything else
|
||||
for key, value in celery_conf.iteritems():
|
||||
key = key.upper()
|
||||
if key in KNOWN_CONFIG_BOOLS:
|
||||
value = asbool(value)
|
||||
elif key in KNOWN_CONFIG_INTS:
|
||||
value = asint(value)
|
||||
elif key in KNOWN_CONFIG_FLOATS:
|
||||
value = asfloat(value)
|
||||
elif key in KNOWN_CONFIG_LISTS:
|
||||
value = aslist(value)
|
||||
celery_settings[key] = value
|
||||
|
||||
# add mandatory celery imports
|
||||
|
||||
@ -16,14 +16,8 @@
|
||||
|
||||
import os
|
||||
|
||||
from paste.deploy.loadwsgi import NicerConfigParser
|
||||
from paste.deploy.converters import asbool
|
||||
|
||||
from mediagoblin import storage
|
||||
from mediagoblin.db.open import setup_connection_and_db_from_config
|
||||
from mediagoblin import app, mg_globals
|
||||
from mediagoblin.celery_setup import setup_celery_from_config
|
||||
from mediagoblin.mg_globals import setup_globals
|
||||
from mediagoblin.workbench import WorkbenchManager, DEFAULT_WORKBENCH_DIR
|
||||
|
||||
|
||||
OUR_MODULENAME = __name__
|
||||
@ -33,64 +27,28 @@ def setup_self():
|
||||
"""
|
||||
Transform this module into a celery config module by reading the
|
||||
mediagoblin config file. Set the environment variable
|
||||
MEDIAGOBLIN_CONFIG to specify where this config file is at and
|
||||
what section it uses.
|
||||
MEDIAGOBLIN_CONFIG to specify where this config file is.
|
||||
|
||||
By default it defaults to 'mediagoblin.ini:app:mediagoblin'.
|
||||
By default it defaults to 'mediagoblin.ini'.
|
||||
|
||||
The first colon ":" is a delimiter between the filename and the
|
||||
config section, so in this case the filename is 'mediagoblin.ini'
|
||||
and the section where mediagoblin is defined is 'app:mediagoblin'.
|
||||
|
||||
Args:
|
||||
- 'setup_globals_func': this is for testing purposes only. Don't
|
||||
set this!
|
||||
Note that if celery_setup_elsewhere is set in your config file,
|
||||
this simply won't work.
|
||||
"""
|
||||
mgoblin_conf_file, mgoblin_section = os.environ.get(
|
||||
'MEDIAGOBLIN_CONFIG', 'mediagoblin.ini:app:mediagoblin').split(':', 1)
|
||||
mgoblin_conf_file = os.path.abspath(
|
||||
os.environ.get('MEDIAGOBLIN_CONFIG', 'mediagoblin.ini'))
|
||||
if not os.path.exists(mgoblin_conf_file):
|
||||
raise IOError(
|
||||
"MEDIAGOBLIN_CONFIG not set or file does not exist")
|
||||
|
||||
parser = NicerConfigParser(mgoblin_conf_file)
|
||||
parser.read(mgoblin_conf_file)
|
||||
parser._defaults.setdefault(
|
||||
'here', os.path.dirname(os.path.abspath(mgoblin_conf_file)))
|
||||
parser._defaults.setdefault(
|
||||
'__file__', os.path.abspath(mgoblin_conf_file))
|
||||
|
||||
mgoblin_section = dict(parser.items(mgoblin_section))
|
||||
mgoblin_conf = dict(
|
||||
[(section_name, dict(parser.items(section_name)))
|
||||
for section_name in parser.sections()])
|
||||
# By setting the environment variable here we should ensure that
|
||||
# this is the module that gets set up.
|
||||
os.environ['CELERY_CONFIG_MODULE'] = OUR_MODULENAME
|
||||
app.MediaGoblinApp(mgoblin_conf_file, setup_celery=False)
|
||||
setup_celery_from_config(
|
||||
mgoblin_section, mgoblin_conf,
|
||||
mg_globals.app_config, mg_globals.global_config,
|
||||
settings_module=OUR_MODULENAME,
|
||||
set_environ=False)
|
||||
|
||||
connection, db = setup_connection_and_db_from_config(mgoblin_section)
|
||||
|
||||
# Set up the storage systems.
|
||||
public_store = storage.storage_system_from_paste_config(
|
||||
mgoblin_section, 'publicstore')
|
||||
queue_store = storage.storage_system_from_paste_config(
|
||||
mgoblin_section, 'queuestore')
|
||||
|
||||
workbench_manager = WorkbenchManager(
|
||||
mgoblin_section.get(
|
||||
'workbench_path', DEFAULT_WORKBENCH_DIR))
|
||||
|
||||
setup_globals(
|
||||
db_connection=connection,
|
||||
database=db,
|
||||
public_store=public_store,
|
||||
email_debug_mode=asbool(mgoblin_section.get('email_debug_mode')),
|
||||
email_sender_address=mgoblin_section.get(
|
||||
'email_sender_address',
|
||||
'notice@mediagoblin.example.org'),
|
||||
queue_store=queue_store,
|
||||
workbench_manager=workbench_manager)
|
||||
|
||||
|
||||
if os.environ['CELERY_CONFIG_MODULE'] == OUR_MODULENAME:
|
||||
setup_self()
|
||||
|
||||
@ -1,42 +0,0 @@
|
||||
# GNU MediaGoblin -- federated, autonomous media hosting
|
||||
# Copyright (C) 2011 Free Software Foundation, Inc
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
import os
|
||||
|
||||
from mediagoblin.tests.tools import TEST_APP_CONFIG
|
||||
from mediagoblin import util
|
||||
from mediagoblin.celery_setup import setup_celery_from_config
|
||||
|
||||
|
||||
OUR_MODULENAME = __name__
|
||||
|
||||
|
||||
def setup_self():
|
||||
"""
|
||||
Set up celery for testing's sake, which just needs to set up
|
||||
celery and celery only.
|
||||
"""
|
||||
mgoblin_conf = util.read_config_file(TEST_APP_CONFIG)
|
||||
mgoblin_section = mgoblin_conf['app:mediagoblin']
|
||||
|
||||
setup_celery_from_config(
|
||||
mgoblin_section, mgoblin_conf,
|
||||
settings_module=OUR_MODULENAME,
|
||||
set_environ=False)
|
||||
|
||||
|
||||
if os.environ.get('CELERY_CONFIG_MODULE') == OUR_MODULENAME:
|
||||
setup_self()
|
||||
122
mediagoblin/config.py
Normal file
122
mediagoblin/config.py
Normal file
@ -0,0 +1,122 @@
|
||||
# GNU MediaGoblin -- federated, autonomous media hosting
|
||||
# Copyright (C) 2011 Free Software Foundation, Inc
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
import os
|
||||
import pkg_resources
|
||||
|
||||
from configobj import ConfigObj, flatten_errors
|
||||
from validate import Validator
|
||||
|
||||
|
||||
CONFIG_SPEC_PATH = pkg_resources.resource_filename(
|
||||
'mediagoblin', 'config_spec.ini')
|
||||
|
||||
|
||||
def _setup_defaults(config, config_path):
|
||||
"""
|
||||
Setup DEFAULTS in a config object from an (absolute) config_path.
|
||||
"""
|
||||
config.setdefault('DEFAULT', {})
|
||||
config['DEFAULT']['here'] = os.path.dirname(config_path)
|
||||
config['DEFAULT']['__file__'] = config_path
|
||||
|
||||
|
||||
def read_mediagoblin_config(config_path, config_spec=CONFIG_SPEC_PATH):
|
||||
"""
|
||||
Read a config object from config_path.
|
||||
|
||||
Does automatic value transformation based on the config_spec.
|
||||
Also provides %(__file__)s and %(here)s values of this file and
|
||||
its directory respectively similar to paste deploy.
|
||||
|
||||
This function doesn't itself raise any exceptions if validation
|
||||
fails, you'll have to do something
|
||||
|
||||
Args:
|
||||
- config_path: path to the config file
|
||||
- config_spec: config file that provides defaults and value types
|
||||
for validation / conversion. Defaults to mediagoblin/config_spec.ini
|
||||
|
||||
Returns:
|
||||
A tuple like: (config, validation_result)
|
||||
... where 'conf' is the parsed config object and 'validation_result'
|
||||
is the information from the validation process.
|
||||
"""
|
||||
config_path = os.path.abspath(config_path)
|
||||
|
||||
config_spec = ConfigObj(
|
||||
config_spec,
|
||||
encoding='UTF8', list_values=False, _inspec=True)
|
||||
|
||||
_setup_defaults(config_spec, config_path)
|
||||
|
||||
config = ConfigObj(
|
||||
config_path,
|
||||
configspec=config_spec,
|
||||
interpolation='ConfigParser')
|
||||
|
||||
_setup_defaults(config, config_path)
|
||||
|
||||
# For now the validator just works with the default functions,
|
||||
# but in the future if we want to add additional validation/configuration
|
||||
# functions we'd add them to validator.functions here.
|
||||
#
|
||||
# See also:
|
||||
# http://www.voidspace.org.uk/python/validate.html#adding-functions
|
||||
validator = Validator()
|
||||
validation_result = config.validate(validator, preserve_errors=True)
|
||||
|
||||
return config, validation_result
|
||||
|
||||
|
||||
REPORT_HEADER = u"""\
|
||||
There were validation problems loading this config file:
|
||||
--------------------------------------------------------
|
||||
"""
|
||||
|
||||
|
||||
def generate_validation_report(config, validation_result):
|
||||
"""
|
||||
Generate a report if necessary of problems while validating.
|
||||
|
||||
Returns:
|
||||
Either a string describing for a user the problems validating
|
||||
this config or None if there are no problems.
|
||||
"""
|
||||
report = []
|
||||
|
||||
# Organize the report
|
||||
for entry in flatten_errors(config, validation_result):
|
||||
# each entry is a tuple
|
||||
section_list, key, error = entry
|
||||
|
||||
if key is not None:
|
||||
section_list.append(key)
|
||||
else:
|
||||
section_list.append(u'[missing section]')
|
||||
|
||||
section_string = u':'.join(section_list)
|
||||
|
||||
if error == False:
|
||||
# We don't care about missing values for now.
|
||||
continue
|
||||
|
||||
report.append(u"%s = %s" % (section_string, error))
|
||||
|
||||
if report:
|
||||
return REPORT_HEADER + u"\n".join(report)
|
||||
else:
|
||||
return None
|
||||
76
mediagoblin/config_spec.ini
Normal file
76
mediagoblin/config_spec.ini
Normal file
@ -0,0 +1,76 @@
|
||||
[mediagoblin]
|
||||
# database stuff
|
||||
db_host = string()
|
||||
db_name = string()
|
||||
db_port = integer()
|
||||
|
||||
#
|
||||
queuestore_base_dir = string(default="%(here)s/user_dev/media/queue")
|
||||
publicstore_base_dir = string(default="%(here)s/user_dev/media/public")
|
||||
|
||||
# Where temporary files used in processing and etc are kept
|
||||
workbench_path = string(default="%(here)s/user_dev/media/workbench")
|
||||
|
||||
#
|
||||
publicstore_base_url = string(default="/mgoblin_media/")
|
||||
|
||||
# Where mediagoblin-builtin static assets are kept
|
||||
direct_remote_path = string(default="/mgoblin_static/")
|
||||
|
||||
# set to false to enable sending notices
|
||||
email_debug_mode = boolean(default=True)
|
||||
email_sender_address = string(default="notice@mediagoblin.example.org")
|
||||
|
||||
# By default not set, but you might want something like:
|
||||
# "%(here)s/user_dev/templates/"
|
||||
local_templates = string()
|
||||
|
||||
# Whether or not celery is set up via an environment variable or
|
||||
# something else (and thus mediagoblin should not attempt to set it up
|
||||
# itself)
|
||||
celery_setup_elsewhere = boolean(default=False)
|
||||
|
||||
[celery]
|
||||
# known booleans
|
||||
celery_result_persistent = boolean()
|
||||
celery_create_missing_queues = boolean()
|
||||
broker_use_ssl = boolean()
|
||||
broker_connection_retry = boolean()
|
||||
celery_always_eager = boolean()
|
||||
celery_eager_propagates_exceptions = boolean()
|
||||
celery_ignore_result = boolean()
|
||||
celery_track_started = boolean()
|
||||
celery_disable_rate_limits = boolean()
|
||||
celery_acks_late = boolean()
|
||||
celery_store_errors_even_if_ignored = boolean()
|
||||
celery_send_task_error_emails = boolean()
|
||||
celery_send_events = boolean()
|
||||
celery_send_task_sent_event = boolean()
|
||||
celeryd_log_color = boolean()
|
||||
celery_redirect_stdouts = boolean()
|
||||
|
||||
# known ints
|
||||
celeryd_concurrency = integer()
|
||||
celeryd_prefetch_multiplier = integer()
|
||||
celery_amqp_task_result_expires = integer()
|
||||
celery_amqp_task_result_connection_max = integer()
|
||||
redis_port = integer()
|
||||
redis_db = integer()
|
||||
broker_port = integer()
|
||||
broker_connection_timeout = integer()
|
||||
celery_broker_connection_max_retries = integer()
|
||||
celery_task_result_expires = integer()
|
||||
celery_max_cached_results = integer()
|
||||
celery_default_rate_limit = integer()
|
||||
celeryd_max_tasks_per_child = integer()
|
||||
celeryd_task_time_limit = integer()
|
||||
celeryd_task_soft_time_limit = integer()
|
||||
mail_port = integer()
|
||||
celerybeat_max_loop_interval = integer()
|
||||
|
||||
# known floats
|
||||
celeryd_eta_scheduler_precision = float()
|
||||
|
||||
# known lists
|
||||
celery_routes = string_list()
|
||||
celery_imports = string_list()
|
||||
@ -23,9 +23,6 @@ def migrate_parser_setup(subparser):
|
||||
subparser.add_argument(
|
||||
'-cf', '--conf_file', default='mediagoblin.ini',
|
||||
help="Config file used to set up environment")
|
||||
subparser.add_argument(
|
||||
'-cs', '--app_section', default='app:mediagoblin',
|
||||
help="Section of the config file where the app config is stored.")
|
||||
|
||||
|
||||
def migrate(args):
|
||||
|
||||
@ -25,9 +25,6 @@ def shell_parser_setup(subparser):
|
||||
subparser.add_argument(
|
||||
'-cf', '--conf_file', default='mediagoblin.ini',
|
||||
help="Config file used to set up environment")
|
||||
subparser.add_argument(
|
||||
'-cs', '--app_section', default='app:mediagoblin',
|
||||
help="Section of the config file where the app config is stored.")
|
||||
|
||||
|
||||
SHELL_BANNER = """\
|
||||
|
||||
@ -15,10 +15,6 @@
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
|
||||
import os
|
||||
|
||||
from paste.deploy.loadwsgi import NicerConfigParser
|
||||
|
||||
from mediagoblin import app
|
||||
|
||||
|
||||
@ -26,20 +22,6 @@ def setup_app(args):
|
||||
"""
|
||||
Setup the application after reading the mediagoblin config files
|
||||
"""
|
||||
# Duplicated from from_celery.py, remove when we have the generic util
|
||||
parser = NicerConfigParser(args.conf_file)
|
||||
parser.read(args.conf_file)
|
||||
parser._defaults.setdefault(
|
||||
'here', os.path.dirname(os.path.abspath(args.conf_file)))
|
||||
parser._defaults.setdefault(
|
||||
'__file__', os.path.abspath(args.conf_file))
|
||||
|
||||
mgoblin_section = dict(parser.items(args.app_section))
|
||||
mgoblin_conf = dict(
|
||||
[(section_name, dict(parser.items(section_name)))
|
||||
for section_name in parser.sections()])
|
||||
|
||||
mgoblin_app = app.paste_app_factory(
|
||||
mgoblin_conf, **mgoblin_section)
|
||||
mgoblin_app = app.MediaGoblinApp(args.conf_file)
|
||||
|
||||
return mgoblin_app
|
||||
|
||||
@ -247,7 +247,7 @@ def clean_listy_filepath(listy_filepath):
|
||||
return cleaned_filepath
|
||||
|
||||
|
||||
def storage_system_from_paste_config(paste_config, storage_prefix):
|
||||
def storage_system_from_config(paste_config, storage_prefix):
|
||||
"""
|
||||
Utility for setting up a storage system from the paste app config.
|
||||
|
||||
@ -266,7 +266,7 @@ def storage_system_from_paste_config(paste_config, storage_prefix):
|
||||
An instantiated storage system.
|
||||
|
||||
Example:
|
||||
storage_system_from_paste_config(
|
||||
storage_system_from_config(
|
||||
{'publicstore_base_url': '/media/',
|
||||
'publicstore_base_dir': '/var/whatever/media/'},
|
||||
'publicstore')
|
||||
|
||||
@ -21,6 +21,7 @@ def setup_package():
|
||||
pass
|
||||
|
||||
def teardown_package():
|
||||
print "Killing db ..."
|
||||
mg_globals.db_connection.drop_database(mg_globals.database.name)
|
||||
print "... done"
|
||||
if mg_globals.db_connection:
|
||||
print "Killing db ..."
|
||||
mg_globals.db_connection.drop_database(mg_globals.database.name)
|
||||
print "... done"
|
||||
|
||||
14
mediagoblin/tests/fake_carrot_conf_bad.ini
Normal file
14
mediagoblin/tests/fake_carrot_conf_bad.ini
Normal file
@ -0,0 +1,14 @@
|
||||
[carrotapp]
|
||||
# Whether or not our carrots are going to be turned into cake.
|
||||
## These should throw errors
|
||||
carrotcake = slobber
|
||||
num_carrots = GROSS
|
||||
|
||||
# A message encouraging our users to eat their carrots.
|
||||
encouragement_phrase = 586956856856 # shouldn't throw error
|
||||
|
||||
# Something extra!
|
||||
blah_blah = "blah!" # shouldn't throw error either
|
||||
|
||||
[celery]
|
||||
eat_celery_with_carrots = pants # yeah that's def an error right there.
|
||||
0
mediagoblin/tests/fake_carrot_conf_empty.ini
Normal file
0
mediagoblin/tests/fake_carrot_conf_empty.ini
Normal file
13
mediagoblin/tests/fake_carrot_conf_good.ini
Normal file
13
mediagoblin/tests/fake_carrot_conf_good.ini
Normal file
@ -0,0 +1,13 @@
|
||||
[carrotapp]
|
||||
# Whether or not our carrots are going to be turned into cake.
|
||||
carrotcake = true
|
||||
num_carrots = 88
|
||||
|
||||
# A message encouraging our users to eat their carrots.
|
||||
encouragement_phrase = "I'd love it if you eat your carrots!"
|
||||
|
||||
# Something extra!
|
||||
blah_blah = "blah!"
|
||||
|
||||
[celery]
|
||||
eat_celery_with_carrots = False
|
||||
9
mediagoblin/tests/fake_celery_conf.ini
Normal file
9
mediagoblin/tests/fake_celery_conf.ini
Normal file
@ -0,0 +1,9 @@
|
||||
['mediagoblin']
|
||||
# I got nothin' in this file!
|
||||
|
||||
['celery']
|
||||
some_variable = floop
|
||||
mail_port = 2000
|
||||
celeryd_eta_scheduler_precision = 1.3
|
||||
celery_result_persistent = true
|
||||
celery_imports = foo.bar.baz, this.is.an.import
|
||||
14
mediagoblin/tests/fake_celery_conf_mgdb.ini
Normal file
14
mediagoblin/tests/fake_celery_conf_mgdb.ini
Normal file
@ -0,0 +1,14 @@
|
||||
['mediagoblin']
|
||||
db_host = mongodb.example.org
|
||||
db_port = 8080
|
||||
db_name = captain_lollerskates
|
||||
|
||||
['something']
|
||||
or = other
|
||||
|
||||
['celery']
|
||||
some_variable = poolf
|
||||
mail_port = 2020
|
||||
celeryd_eta_scheduler_precision = 3.1
|
||||
celery_result_persistent = false
|
||||
celery_imports = baz.bar.foo, import.is.a.this
|
||||
10
mediagoblin/tests/fake_config_spec.ini
Normal file
10
mediagoblin/tests/fake_config_spec.ini
Normal file
@ -0,0 +1,10 @@
|
||||
[carrotapp]
|
||||
# Whether or not our carrots are going to be turned into cake.
|
||||
carrotcake = boolean(default=False)
|
||||
num_carrots = integer(default=1)
|
||||
|
||||
# A message encouraging our users to eat their carrots.
|
||||
encouragement_phrase = string()
|
||||
|
||||
[celery]
|
||||
eat_celery_with_carrots = boolean(default=True)
|
||||
@ -15,6 +15,13 @@
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
from mediagoblin import celery_setup
|
||||
from mediagoblin.config import read_mediagoblin_config
|
||||
|
||||
|
||||
TEST_CELERY_CONF_NOSPECIALDB = pkg_resources.resource_filename(
|
||||
'mediagoblin.tests', 'fake_celery_conf.ini')
|
||||
TEST_CELERY_CONF_MGSPECIALDB = pkg_resources.resource_filename(
|
||||
'mediagoblin.tests', 'fake_celery_conf_mgdb.ini')
|
||||
|
||||
|
||||
def test_setup_celery_from_config():
|
||||
@ -25,14 +32,12 @@ def test_setup_celery_from_config():
|
||||
for var in vars_to_wipe:
|
||||
delattr(module, var)
|
||||
|
||||
global_config, validation_result = read_mediagoblin_config(
|
||||
TEST_CELERY_CONF_NOSPECIALDB)
|
||||
app_config = global_config['mediagoblin']
|
||||
|
||||
celery_setup.setup_celery_from_config(
|
||||
{},
|
||||
{'something': {'or': 'other'},
|
||||
'celery': {'some_variable': 'floop',
|
||||
'mail_port': '2000',
|
||||
'CELERYD_ETA_SCHEDULER_PRECISION': '1.3',
|
||||
'celery_result_persistent': 'true',
|
||||
'celery_imports': 'foo.bar.baz this.is.an.import'}},
|
||||
app_config, global_config,
|
||||
'mediagoblin.tests.fake_celery_module', set_environ=False)
|
||||
|
||||
from mediagoblin.tests import fake_celery_module
|
||||
@ -51,17 +56,12 @@ def test_setup_celery_from_config():
|
||||
|
||||
_wipe_testmodule_clean(fake_celery_module)
|
||||
|
||||
global_config, validation_result = read_mediagoblin_config(
|
||||
TEST_CELERY_CONF_MGSPECIALDB)
|
||||
app_config = global_config['mediagoblin']
|
||||
|
||||
celery_setup.setup_celery_from_config(
|
||||
{'db_host': 'mongodb.example.org',
|
||||
'db_port': '8080',
|
||||
'db_name': 'captain_lollerskates',
|
||||
'celery_section': 'vegetable'},
|
||||
{'something': {'or': 'other'},
|
||||
'vegetable': {'some_variable': 'poolf',
|
||||
'mail_port': '2020',
|
||||
'CELERYD_ETA_SCHEDULER_PRECISION': '3.1',
|
||||
'celery_result_persistent': 'false',
|
||||
'celery_imports': 'baz.bar.foo import.is.a.this'}},
|
||||
app_config, global_config,
|
||||
'mediagoblin.tests.fake_celery_module', set_environ=False)
|
||||
|
||||
from mediagoblin.tests import fake_celery_module
|
||||
|
||||
97
mediagoblin/tests/test_config.py
Normal file
97
mediagoblin/tests/test_config.py
Normal file
@ -0,0 +1,97 @@
|
||||
# GNU MediaGoblin -- federated, autonomous media hosting
|
||||
# Copyright (C) 2011 Free Software Foundation, Inc
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
import pkg_resources
|
||||
|
||||
from mediagoblin import config
|
||||
|
||||
|
||||
CARROT_CONF_GOOD = pkg_resources.resource_filename(
|
||||
'mediagoblin.tests', 'fake_carrot_conf_good.ini')
|
||||
CARROT_CONF_EMPTY = pkg_resources.resource_filename(
|
||||
'mediagoblin.tests', 'fake_carrot_conf_empty.ini')
|
||||
CARROT_CONF_BAD = pkg_resources.resource_filename(
|
||||
'mediagoblin.tests', 'fake_carrot_conf_bad.ini')
|
||||
FAKE_CONFIG_SPEC = pkg_resources.resource_filename(
|
||||
'mediagoblin.tests', 'fake_config_spec.ini')
|
||||
|
||||
|
||||
def test_read_mediagoblin_config():
|
||||
# An empty file
|
||||
this_conf, validation_results = config.read_mediagoblin_config(
|
||||
CARROT_CONF_EMPTY, FAKE_CONFIG_SPEC)
|
||||
|
||||
assert this_conf['carrotapp']['carrotcake'] == False
|
||||
assert this_conf['carrotapp']['num_carrots'] == 1
|
||||
assert not this_conf['carrotapp'].has_key('encouragement_phrase')
|
||||
assert this_conf['celery']['eat_celery_with_carrots'] == True
|
||||
|
||||
# A good file
|
||||
this_conf, validation_results = config.read_mediagoblin_config(
|
||||
CARROT_CONF_GOOD, FAKE_CONFIG_SPEC)
|
||||
|
||||
assert this_conf['carrotapp']['carrotcake'] == True
|
||||
assert this_conf['carrotapp']['num_carrots'] == 88
|
||||
assert this_conf['carrotapp']['encouragement_phrase'] == \
|
||||
"I'd love it if you eat your carrots!"
|
||||
assert this_conf['carrotapp']['blah_blah'] == "blah!"
|
||||
assert this_conf['celery']['eat_celery_with_carrots'] == False
|
||||
|
||||
# A bad file
|
||||
this_conf, validation_results = config.read_mediagoblin_config(
|
||||
CARROT_CONF_BAD, FAKE_CONFIG_SPEC)
|
||||
|
||||
# These should still open but will have errors that we'll test for
|
||||
# in test_generate_validation_report()
|
||||
assert this_conf['carrotapp']['carrotcake'] == 'slobber'
|
||||
assert this_conf['carrotapp']['num_carrots'] == 'GROSS'
|
||||
assert this_conf['carrotapp']['encouragement_phrase'] == \
|
||||
"586956856856"
|
||||
assert this_conf['carrotapp']['blah_blah'] == "blah!"
|
||||
assert this_conf['celery']['eat_celery_with_carrots'] == "pants"
|
||||
|
||||
|
||||
def test_generate_validation_report():
|
||||
# Empty
|
||||
this_conf, validation_results = config.read_mediagoblin_config(
|
||||
CARROT_CONF_EMPTY, FAKE_CONFIG_SPEC)
|
||||
report = config.generate_validation_report(this_conf, validation_results)
|
||||
assert report is None
|
||||
|
||||
# Good
|
||||
this_conf, validation_results = config.read_mediagoblin_config(
|
||||
CARROT_CONF_GOOD, FAKE_CONFIG_SPEC)
|
||||
report = config.generate_validation_report(this_conf, validation_results)
|
||||
assert report is None
|
||||
|
||||
# Bad
|
||||
this_conf, validation_results = config.read_mediagoblin_config(
|
||||
CARROT_CONF_BAD, FAKE_CONFIG_SPEC)
|
||||
report = config.generate_validation_report(this_conf, validation_results)
|
||||
|
||||
assert report.startswith("""\
|
||||
There were validation problems loading this config file:
|
||||
--------------------------------------------------------""")
|
||||
|
||||
expected_warnings = [
|
||||
'carrotapp:carrotcake = the value "slobber" is of the wrong type.',
|
||||
'carrotapp:num_carrots = the value "GROSS" is of the wrong type.',
|
||||
'celery:eat_celery_with_carrots = the value "pants" is of the wrong type.']
|
||||
warnings = report.splitlines()[2:]
|
||||
|
||||
assert len(warnings) == 3
|
||||
for warning in expected_warnings:
|
||||
assert warning in warnings
|
||||
12
mediagoblin/tests/test_mgoblin_app.ini
Normal file
12
mediagoblin/tests/test_mgoblin_app.ini
Normal file
@ -0,0 +1,12 @@
|
||||
[mediagoblin]
|
||||
queuestore_base_dir = %(here)s/test_user_dev/media/queue
|
||||
publicstore_base_dir = %(here)s/test_user_dev/media/public
|
||||
publicstore_base_url = /mgoblin_media/
|
||||
direct_remote_path = /mgoblin_static/
|
||||
email_sender_address = "notice@mediagoblin.example.org"
|
||||
email_debug_mode = true
|
||||
db_name = __mediagoblin_tests__
|
||||
|
||||
# Celery shouldn't be set up by the paste app factory as it's set up
|
||||
# elsewhere
|
||||
celery_setup_elsewhere = true
|
||||
@ -10,16 +10,7 @@ use = egg:Paste#urlmap
|
||||
[app:mediagoblin]
|
||||
use = egg:mediagoblin#app
|
||||
filter-with = beaker
|
||||
queuestore_base_dir = %(here)s/test_user_dev/media/queue
|
||||
publicstore_base_dir = %(here)s/test_user_dev/media/public
|
||||
publicstore_base_url = /mgoblin_media/
|
||||
direct_remote_path = /mgoblin_static/
|
||||
email_sender_address = "notice@mediagoblin.example.org"
|
||||
email_debug_mode = true
|
||||
db_name = __mediagoblin_tests__
|
||||
# Celery shouldn't be set up by the paste app factory as it's set up
|
||||
# elsewhere
|
||||
celery_setup_elsewhere = true
|
||||
config = %(here)s/test_mgoblin_app.ini
|
||||
|
||||
[app:publicstore_serve]
|
||||
use = egg:Paste#static
|
||||
@ -58,8 +58,8 @@ class FakeRemoteStorage(storage.BasicFileStorage):
|
||||
local_storage = False
|
||||
|
||||
|
||||
def test_storage_system_from_paste_config():
|
||||
this_storage = storage.storage_system_from_paste_config(
|
||||
def test_storage_system_from_config():
|
||||
this_storage = storage.storage_system_from_config(
|
||||
{'somestorage_base_url': 'http://example.org/moodia/',
|
||||
'somestorage_base_dir': '/tmp/',
|
||||
'somestorage_garbage_arg': 'garbage_arg',
|
||||
@ -69,7 +69,7 @@ def test_storage_system_from_paste_config():
|
||||
assert this_storage.base_dir == '/tmp/'
|
||||
assert this_storage.__class__ is storage.BasicFileStorage
|
||||
|
||||
this_storage = storage.storage_system_from_paste_config(
|
||||
this_storage = storage.storage_system_from_config(
|
||||
{'somestorage_foobie': 'eiboof',
|
||||
'somestorage_blech': 'hcelb',
|
||||
'somestorage_garbage_arg': 'garbage_arg',
|
||||
|
||||
@ -18,20 +18,25 @@
|
||||
import pkg_resources
|
||||
import os, shutil
|
||||
|
||||
from paste.deploy import appconfig, loadapp
|
||||
from paste.deploy import loadapp
|
||||
from webtest import TestApp
|
||||
|
||||
from mediagoblin import util
|
||||
from mediagoblin import util, mg_globals
|
||||
from mediagoblin.config import read_mediagoblin_config
|
||||
from mediagoblin.celery_setup import setup_celery_from_config
|
||||
from mediagoblin.decorators import _make_safe
|
||||
from mediagoblin.db.open import setup_connection_and_db_from_config
|
||||
|
||||
|
||||
MEDIAGOBLIN_TEST_DB_NAME = '__mediagoblinunittests__'
|
||||
TEST_SERVER_CONFIG = pkg_resources.resource_filename(
|
||||
'mediagoblin.tests', 'test_server.ini')
|
||||
TEST_APP_CONFIG = pkg_resources.resource_filename(
|
||||
'mediagoblin.tests', 'mgoblin_test_app.ini')
|
||||
'mediagoblin.tests', 'test_mgoblin_app.ini')
|
||||
TEST_USER_DEV = pkg_resources.resource_filename(
|
||||
'mediagoblin.tests', 'test_user_dev')
|
||||
MGOBLIN_APP = None
|
||||
CELERY_SETUP = False
|
||||
|
||||
USER_DEV_DIRECTORIES_TO_SETUP = [
|
||||
'media/public', 'media/queue',
|
||||
@ -42,12 +47,13 @@ class BadCeleryEnviron(Exception): pass
|
||||
|
||||
|
||||
def get_test_app(dump_old_app=True):
|
||||
if not os.environ.get('CELERY_CONFIG_MODULE') == \
|
||||
'mediagoblin.celery_setup.from_tests':
|
||||
if os.environ.get('CELERY_CONFIG_MODULE'):
|
||||
raise BadCeleryEnviron(
|
||||
u"Sorry, you *absolutely* must run nosetests with the\n"
|
||||
u"mediagoblin.celery_setup.from_tests module. Like so:\n"
|
||||
u"$ CELERY_CONFIG_MODULE=mediagoblin.celery_setup.from_tests ./bin/nosetests")
|
||||
u"Sorry, you *ABSOLUTELY MUST *NOT* run nosetests with the\n"
|
||||
u"CELERY_CONFIG_MODULE set to anything.")
|
||||
|
||||
global MGOBLIN_APP
|
||||
global CELERY_SETUP
|
||||
|
||||
# Just return the old app if that exists and it's okay to set up
|
||||
# and return
|
||||
@ -63,16 +69,13 @@ def get_test_app(dump_old_app=True):
|
||||
os.makedirs(full_dir)
|
||||
|
||||
# Get app config
|
||||
config = appconfig(
|
||||
'config:' + os.path.basename(TEST_APP_CONFIG),
|
||||
relative_to=os.path.dirname(TEST_APP_CONFIG),
|
||||
name='mediagoblin')
|
||||
global_config, validation_result = read_mediagoblin_config(TEST_APP_CONFIG)
|
||||
app_config = global_config['mediagoblin']
|
||||
|
||||
# Wipe database
|
||||
# @@: For now we're dropping collections, but we could also just
|
||||
# collection.remove() ?
|
||||
connection, db = setup_connection_and_db_from_config(
|
||||
config.local_conf)
|
||||
connection, db = setup_connection_and_db_from_config(app_config)
|
||||
|
||||
collections_to_wipe = [
|
||||
collection
|
||||
@ -90,9 +93,19 @@ def get_test_app(dump_old_app=True):
|
||||
|
||||
# setup app and return
|
||||
test_app = loadapp(
|
||||
'config:' + TEST_APP_CONFIG)
|
||||
'config:' + TEST_SERVER_CONFIG)
|
||||
|
||||
return TestApp(test_app)
|
||||
app = TestApp(test_app)
|
||||
MGOBLIN_APP = app
|
||||
|
||||
# setup celery
|
||||
if not CELERY_SETUP:
|
||||
setup_celery_from_config(
|
||||
mg_globals.app_config, mg_globals.global_config,
|
||||
set_environ=True)
|
||||
CELERY_SETUP = True
|
||||
|
||||
return app
|
||||
|
||||
|
||||
def setup_fresh_app(func):
|
||||
|
||||
@ -18,7 +18,6 @@ from email.MIMEText import MIMEText
|
||||
import gettext
|
||||
import pkg_resources
|
||||
import smtplib
|
||||
import os
|
||||
import sys
|
||||
import re
|
||||
import urllib
|
||||
@ -28,7 +27,6 @@ import copy
|
||||
from babel.localedata import exists
|
||||
import jinja2
|
||||
import translitcodec
|
||||
from paste.deploy.loadwsgi import NicerConfigParser
|
||||
from webob import Response, exc
|
||||
from lxml.html.clean import Cleaner
|
||||
|
||||
@ -352,28 +350,6 @@ def get_locale_from_request(request):
|
||||
return locale_to_lower_upper(target_lang)
|
||||
|
||||
|
||||
def read_config_file(conf_file):
|
||||
"""
|
||||
Read a paste deploy style config file and process it.
|
||||
"""
|
||||
if not os.path.exists(conf_file):
|
||||
raise IOError(
|
||||
"MEDIAGOBLIN_CONFIG not set or file does not exist")
|
||||
|
||||
parser = NicerConfigParser(conf_file)
|
||||
parser.read(conf_file)
|
||||
parser._defaults.setdefault(
|
||||
'here', os.path.dirname(os.path.abspath(conf_file)))
|
||||
parser._defaults.setdefault(
|
||||
'__file__', os.path.abspath(conf_file))
|
||||
|
||||
mgoblin_conf = dict(
|
||||
[(section_name, dict(parser.items(section_name)))
|
||||
for section_name in parser.sections()])
|
||||
|
||||
return mgoblin_conf
|
||||
|
||||
|
||||
# A super strict version of the lxml.html cleaner class
|
||||
HTML_CLEANER = Cleaner(
|
||||
scripts=True,
|
||||
|
||||
34
server.ini
Normal file
34
server.ini
Normal file
@ -0,0 +1,34 @@
|
||||
[DEFAULT]
|
||||
debug = true
|
||||
|
||||
[composite:main]
|
||||
use = egg:Paste#urlmap
|
||||
/ = mediagoblin
|
||||
/mgoblin_media/ = publicstore_serve
|
||||
/mgoblin_static/ = mediagoblin_static
|
||||
|
||||
[app:mediagoblin]
|
||||
use = egg:mediagoblin#app
|
||||
filter-with = beaker
|
||||
config = %(here)s/mediagoblin.ini
|
||||
|
||||
[app:publicstore_serve]
|
||||
use = egg:Paste#static
|
||||
document_root = %(here)s/user_dev/media/public/
|
||||
|
||||
[app:mediagoblin_static]
|
||||
use = egg:Paste#static
|
||||
document_root = %(here)s/mediagoblin/static/
|
||||
|
||||
[filter:beaker]
|
||||
use = egg:Beaker#beaker_session
|
||||
cache_dir = %(here)s/user_dev/beaker
|
||||
beaker.session.key = mediagoblin
|
||||
# beaker.session.secret = somesupersecret
|
||||
beaker.session.data_dir = %(here)s/user_dev/beaker/sessions/data
|
||||
beaker.session.lock_dir = %(here)s/user_dev/beaker/sessions/lock
|
||||
|
||||
[server:main]
|
||||
use = egg:Paste#http
|
||||
host = 127.0.0.1
|
||||
port = 6543
|
||||
Loading…
x
Reference in New Issue
Block a user