Changeset - f973b866fffc
[Not reviewed]
default
0 15 0
Thomas De Schampheleire - 9 years ago 2016-08-22 17:50:51
thomas.de.schampheleire@gmail.com
Turbogears2 migration: use sqlalchemy.url iso sqlalchemy.db1.url

In Turbogears2, much of the application initialization is handled by the
framework, whereas in Pylons the application was responsible for it.
Initializing SQLAlchemy is one such part of initialization which is handled
by Turbogears2.

Turbogears2 expects the configuration file to refer to the database using
'sqlalchemy.url' rather than the current 'sqlalchemy.db1.url'. While the
exact name is not really important, not following this approach means we'll
need to override the sqlalchemy initialization method.

Therefore, as a preparation to the Turbogears2 migration, already change the
database reference string under Pylons.
When upgrading to a version of Kallithea containing this commit, the .ini
file will manually need to be adapted to remove the .db1 strings.
15 files changed with 34 insertions and 34 deletions:
0 comments (0 inline, 0 general)
development.ini
Show inline comments
 
@@ -475,36 +475,36 @@ set debug = true
 
###       LOGVIEW CONFIG       ###
 
##################################
 

	
 
logview.sqlalchemy = #faa
 
logview.pylons.templating = #bfb
 
logview.pylons.util = #eee
 

	
 
#########################################################
 
### DB CONFIGS - EACH DB WILL HAVE IT'S OWN CONFIG    ###
 
#########################################################
 

	
 
# SQLITE [default]
 
sqlalchemy.db1.url = sqlite:///%(here)s/kallithea.db?timeout=60
 
sqlalchemy.url = sqlite:///%(here)s/kallithea.db?timeout=60
 

	
 
# POSTGRESQL
 
#sqlalchemy.db1.url = postgresql://user:pass@localhost/kallithea
 
#sqlalchemy.url = postgresql://user:pass@localhost/kallithea
 

	
 
# MySQL
 
#sqlalchemy.db1.url = mysql://user:pass@localhost/kallithea?charset=utf8
 
#sqlalchemy.url = mysql://user:pass@localhost/kallithea?charset=utf8
 

	
 
# see sqlalchemy docs for others
 

	
 
sqlalchemy.db1.echo = false
 
sqlalchemy.db1.pool_recycle = 3600
 
sqlalchemy.echo = false
 
sqlalchemy.pool_recycle = 3600
 

	
 
################################
 
### ALEMBIC CONFIGURATION   ####
 
################################
 

	
 
[alembic]
 
script_location = kallithea:alembic
 

	
 
################################
 
### LOGGING CONFIGURATION   ####
 
################################
 

	
docs/upgrade.rst
Show inline comments
 
@@ -45,25 +45,25 @@ make a copy of the entire ``rcextensions
 
Back up your database
 
^^^^^^^^^^^^^^^^^^^^^
 

	
 
If using SQLite, simply make a copy of the Kallithea database (``.db``)
 
file.
 

	
 
If using PostgreSQL, please consult the documentation for the ``pg_dump``
 
utility.
 

	
 
If using MySQL, please consult the documentation for the ``mysqldump``
 
utility.
 

	
 
Look for ``sqlalchemy.db1.url`` in your configuration file to determine
 
Look for ``sqlalchemy.url`` in your configuration file to determine
 
database type, settings, location, etc.
 

	
 

	
 
3. Activate the Kallithea virtual environment (if any)
 
------------------------------------------------------
 

	
 
Verify that you are using the Python environment that you originally
 
installed Kallithea in by running::
 

	
 
    pip freeze
 

	
 
This will list all packages installed in the current environment. If
kallithea/alembic/env.py
Show inline comments
 
@@ -22,25 +22,25 @@ from sqlalchemy import engine_from_confi
 

	
 
from kallithea.model import db
 

	
 

	
 
# The alembic.config.Config object, which wraps the current .ini file.
 
config = context.config
 

	
 
# Default to use the main Kallithea database string in [app:main].
 
# For advanced uses, this can be overridden by specifying an explicit
 
# [alembic] sqlalchemy.url.
 
database_url = (
 
    config.get_main_option('sqlalchemy.url') or
 
    config.get_section_option('app:main', 'sqlalchemy.db1.url')
 
    config.get_section_option('app:main', 'sqlalchemy.url')
 
)
 

	
 
# Configure default logging for Alembic. (This can be overriden by the
 
# config file, but usually isn't.)
 
logging.getLogger('alembic').setLevel(logging.INFO)
 

	
 
# Setup Python loggers based on the config file provided to the alembic
 
# command. If we're being invoked via the Alembic API (presumably for
 
# stamping during "paster setup-db"), config_file_name is not available,
 
# and loggers are assumed to already have been configured.
 
if config.config_file_name:
 
    fileConfig(config.config_file_name, disable_existing_loggers=False)
kallithea/bin/template.ini.mako
Show inline comments
 
@@ -475,39 +475,39 @@ set debug = false
 
<%text>##################################</%text>
 

	
 
logview.sqlalchemy = #faa
 
logview.pylons.templating = #bfb
 
logview.pylons.util = #eee
 

	
 
<%text>#########################################################</%text>
 
<%text>### DB CONFIGS - EACH DB WILL HAVE IT'S OWN CONFIG    ###</%text>
 
<%text>#########################################################</%text>
 

	
 
%if database_engine == 'sqlite':
 
# SQLITE [default]
 
sqlalchemy.db1.url = sqlite:///${here}/kallithea.db?timeout=60
 
sqlalchemy.url = sqlite:///${here}/kallithea.db?timeout=60
 

	
 
%elif database_engine == 'postgres':
 
# POSTGRESQL
 
sqlalchemy.db1.url = postgresql://user:pass@localhost/kallithea
 
sqlalchemy.url = postgresql://user:pass@localhost/kallithea
 

	
 
%elif database_engine == 'mysql':
 
# MySQL
 
sqlalchemy.db1.url = mysql://user:pass@localhost/kallithea?charset=utf8
 
sqlalchemy.url = mysql://user:pass@localhost/kallithea?charset=utf8
 

	
 
%endif
 
# see sqlalchemy docs for others
 

	
 
sqlalchemy.db1.echo = false
 
sqlalchemy.db1.pool_recycle = 3600
 
sqlalchemy.echo = false
 
sqlalchemy.pool_recycle = 3600
 

	
 
<%text>################################</%text>
 
<%text>### ALEMBIC CONFIGURATION   ####</%text>
 
<%text>################################</%text>
 

	
 
[alembic]
 
script_location = kallithea:alembic
 

	
 
<%text>################################</%text>
 
<%text>### LOGGING CONFIGURATION   ####</%text>
 
<%text>################################</%text>
 

	
kallithea/config/deployment.ini_tmpl
Show inline comments
 
@@ -467,36 +467,36 @@ set debug = false
 
###       LOGVIEW CONFIG       ###
 
##################################
 

	
 
logview.sqlalchemy = #faa
 
logview.pylons.templating = #bfb
 
logview.pylons.util = #eee
 

	
 
#########################################################
 
### DB CONFIGS - EACH DB WILL HAVE IT'S OWN CONFIG    ###
 
#########################################################
 

	
 
# SQLITE [default]
 
sqlalchemy.db1.url = sqlite:///%(here)s/kallithea.db?timeout=60
 
sqlalchemy.url = sqlite:///%(here)s/kallithea.db?timeout=60
 

	
 
# POSTGRESQL
 
#sqlalchemy.db1.url = postgresql://user:pass@localhost/kallithea
 
#sqlalchemy.url = postgresql://user:pass@localhost/kallithea
 

	
 
# MySQL
 
#sqlalchemy.db1.url = mysql://user:pass@localhost/kallithea?charset=utf8
 
#sqlalchemy.url = mysql://user:pass@localhost/kallithea?charset=utf8
 

	
 
# see sqlalchemy docs for others
 

	
 
sqlalchemy.db1.echo = false
 
sqlalchemy.db1.pool_recycle = 3600
 
sqlalchemy.echo = false
 
sqlalchemy.pool_recycle = 3600
 

	
 
################################
 
### ALEMBIC CONFIGURATION   ####
 
################################
 

	
 
[alembic]
 
script_location = kallithea:alembic
 

	
 
################################
 
### LOGGING CONFIGURATION   ####
 
################################
 

	
kallithea/config/environment.py
Show inline comments
 
@@ -84,40 +84,40 @@ def load_environment(global_conf, app_co
 
        imports=['from webhelpers.html import escape'])
 

	
 
    # sets the c attribute access when don't existing attribute are accessed
 
    config['pylons.strict_tmpl_context'] = True
 
    test = os.path.split(config['__file__'])[-1] == 'test.ini'
 
    if test:
 
        if test_env is None:
 
            test_env = not int(os.environ.get('KALLITHEA_NO_TMP_PATH', 0))
 
        if test_index is None:
 
            test_index = not int(os.environ.get('KALLITHEA_WHOOSH_TEST_DISABLE', 0))
 
        if os.environ.get('TEST_DB'):
 
            # swap config if we pass enviroment variable
 
            config['sqlalchemy.db1.url'] = os.environ.get('TEST_DB')
 
            config['sqlalchemy.url'] = os.environ.get('TEST_DB')
 

	
 
        from kallithea.lib.utils import create_test_env, create_test_index
 
        from kallithea.tests import TESTS_TMP_PATH
 
        #set KALLITHEA_NO_TMP_PATH=1 to disable re-creating the database and
 
        #test repos
 
        if test_env:
 
            create_test_env(TESTS_TMP_PATH, config)
 
        #set KALLITHEA_WHOOSH_TEST_DISABLE=1 to disable whoosh index during tests
 
        if test_index:
 
            create_test_index(TESTS_TMP_PATH, config, True)
 

	
 
    # MULTIPLE DB configs
 
    # Setup the SQLAlchemy database engine
 
    sa_engine_db1 = engine_from_config(config, 'sqlalchemy.db1.')
 
    init_model(sa_engine_db1)
 
    sa_engine = engine_from_config(config, 'sqlalchemy.')
 
    init_model(sa_engine)
 

	
 
    set_available_permissions(config)
 
    repos_path = make_ui('db').configitems('paths')[0][1]
 
    config['base_path'] = repos_path
 
    set_app_settings(config)
 

	
 
    instance_id = kallithea.CONFIG.get('instance_id', '*')
 
    if instance_id == '*':
 
        instance_id = '%s-%s' % (platform.uname()[1], os.getpid())
 
        kallithea.CONFIG['instance_id'] = instance_id
 

	
 
    # CONFIGURATION OPTIONS HERE (note: all config options will override
kallithea/lib/celerylib/__init__.py
Show inline comments
 
@@ -113,25 +113,25 @@ def locked_task(func):
 
            ret = func(*fargs, **fkwargs)
 
            l.release()
 
            return ret
 
        except LockHeld:
 
            log.info('LockHeld')
 
            return 'Task with key %s already running' % lockkey
 

	
 
    return decorator(__wrapper, func)
 

	
 

	
 
def get_session():
 
    if CELERY_ON:
 
        engine = engine_from_config(config, 'sqlalchemy.db1.')
 
        engine = engine_from_config(config, 'sqlalchemy.')
 
        init_model(engine)
 
    sa = meta.Session()
 
    return sa
 

	
 

	
 
def dbsession(func):
 
    def __wrapper(func, *fargs, **fkwargs):
 
        try:
 
            ret = func(*fargs, **fkwargs)
 
            return ret
 
        finally:
 
            if CELERY_ON and not CELERY_EAGER:
kallithea/lib/hooks.py
Show inline comments
 
@@ -386,25 +386,25 @@ def handle_git_receive(repo_path, revs, 
 
    from kallithea.config.environment import load_environment
 
    from kallithea.model import init_model
 
    from kallithea.model.db import Ui
 
    from kallithea.lib.utils import make_ui
 
    extras = _extract_extras(env)
 

	
 
    repo_path = safe_unicode(repo_path)
 
    path, ini_name = os.path.split(extras['config'])
 
    conf = appconfig('config:%s' % ini_name, relative_to=path)
 
    load_environment(conf.global_conf, conf.local_conf, test_env=False,
 
                     test_index=False)
 

	
 
    engine = engine_from_config(conf, 'sqlalchemy.db1.')
 
    engine = engine_from_config(conf, 'sqlalchemy.')
 
    init_model(engine)
 

	
 
    baseui = make_ui('db')
 
    # fix if it's not a bare repo
 
    if repo_path.endswith(os.sep + '.git'):
 
        repo_path = repo_path[:-5]
 

	
 
    repo = Repository.get_by_full_path(repo_path)
 
    if not repo:
 
        raise OSError('Repository %s not found in database'
 
                      % (safe_str(repo_path)))
 

	
kallithea/lib/paster_commands/common.py
Show inline comments
 
@@ -94,14 +94,14 @@ class BasePasterCommand(Command):
 
        pylonsconfig.init_app(conf.global_conf, conf.local_conf)
 

	
 
    def _init_session(self):
 
        """
 
        Inits SqlAlchemy Session
 
        """
 
        logging.config.fileConfig(self.path_to_ini_file)
 

	
 
        from pylons import config
 
        from kallithea.model import init_model
 
        from kallithea.lib.utils2 import engine_from_config
 
        setup_cache_regions(config)
 
        engine = engine_from_config(config, 'sqlalchemy.db1.')
 
        engine = engine_from_config(config, 'sqlalchemy.')
 
        init_model(engine)
kallithea/lib/utils.py
Show inline comments
 
@@ -623,25 +623,25 @@ def create_test_index(repo_location, con
 
        pass
 

	
 

	
 
def create_test_env(repos_test_path, config):
 
    """
 
    Makes a fresh database and
 
    install test repository into tmp dir
 
    """
 
    from kallithea.lib.db_manage import DbManage
 
    from kallithea.tests import HG_REPO, GIT_REPO, TESTS_TMP_PATH
 

	
 
    # PART ONE create db
 
    dbconf = config['sqlalchemy.db1.url']
 
    dbconf = config['sqlalchemy.url']
 
    log.debug('making test db %s', dbconf)
 

	
 
    # create test dir if it doesn't exist
 
    if not os.path.isdir(repos_test_path):
 
        log.debug('Creating testdir %s', repos_test_path)
 
        os.makedirs(repos_test_path)
 

	
 
    dbmanage = DbManage(log_sql=True, dbconf=dbconf, root=config['here'],
 
                        tests=True)
 
    dbmanage.create_tables(override=True)
 
    # for tests dynamically set new root paths based on generated content
 
    dbmanage.create_settings(dbmanage.config_prompt(repos_test_path))
kallithea/tests/scripts/manual_test_concurrency.py
Show inline comments
 
@@ -73,25 +73,25 @@ class Command(object):
 
        command = cmd + ' ' + ' '.join(args)
 
        log.debug('Executing %s', command)
 
        if DEBUG:
 
            print command
 
        p = Popen(command, shell=True, stdout=PIPE, stderr=PIPE, cwd=self.cwd)
 
        stdout, stderr = p.communicate()
 
        if DEBUG:
 
            print stdout, stderr
 
        return stdout, stderr
 

	
 

	
 
def get_session():
 
    engine = engine_from_config(conf, 'sqlalchemy.db1.')
 
    engine = engine_from_config(conf, 'sqlalchemy.')
 
    init_model(engine)
 
    sa = meta.Session
 
    return sa
 

	
 

	
 
def create_test_user(force=True):
 
    print 'creating test user'
 
    sa = get_session()
 

	
 
    user = sa.query(User).filter(User.username == USER).scalar()
 

	
 
    if force and user is not None:
kallithea/tests/test.ini
Show inline comments
 
@@ -479,37 +479,37 @@ set debug = false
 
###       LOGVIEW CONFIG       ###
 
##################################
 

	
 
logview.sqlalchemy = #faa
 
logview.pylons.templating = #bfb
 
logview.pylons.util = #eee
 

	
 
#########################################################
 
### DB CONFIGS - EACH DB WILL HAVE IT'S OWN CONFIG    ###
 
#########################################################
 

	
 
# SQLITE [default]
 
#sqlalchemy.db1.url = sqlite:///%(here)s/kallithea.db?timeout=60
 
sqlalchemy.db1.url = sqlite:///%(here)s/kallithea_test.sqlite
 
#sqlalchemy.url = sqlite:///%(here)s/kallithea.db?timeout=60
 
sqlalchemy.url = sqlite:///%(here)s/kallithea_test.sqlite
 

	
 
# POSTGRESQL
 
#sqlalchemy.db1.url = postgresql://user:pass@localhost/kallithea
 
#sqlalchemy.url = postgresql://user:pass@localhost/kallithea
 

	
 
# MySQL
 
#sqlalchemy.db1.url = mysql://user:pass@localhost/kallithea?charset=utf8
 
#sqlalchemy.url = mysql://user:pass@localhost/kallithea?charset=utf8
 

	
 
# see sqlalchemy docs for others
 

	
 
sqlalchemy.db1.echo = false
 
sqlalchemy.db1.pool_recycle = 3600
 
sqlalchemy.echo = false
 
sqlalchemy.pool_recycle = 3600
 

	
 
################################
 
### ALEMBIC CONFIGURATION   ####
 
################################
 

	
 
[alembic]
 
script_location = kallithea:alembic
 

	
 
################################
 
### LOGGING CONFIGURATION   ####
 
################################
 

	
kallithea/websetup.py
Show inline comments
 
@@ -28,24 +28,24 @@ Original author and date, and relevant c
 
import logging
 

	
 
from kallithea.config.environment import load_environment
 
from kallithea.lib.db_manage import DbManage
 
from kallithea.model.meta import Session
 

	
 

	
 
log = logging.getLogger(__name__)
 

	
 

	
 
def setup_app(command, conf, vars):
 
    """Place any commands to setup kallithea here"""
 
    dbconf = conf['sqlalchemy.db1.url']
 
    dbconf = conf['sqlalchemy.url']
 
    dbmanage = DbManage(log_sql=True, dbconf=dbconf, root=conf['here'],
 
                        tests=False, cli_args=command.options.__dict__)
 
    dbmanage.create_tables(override=True)
 
    opts = dbmanage.config_prompt(None)
 
    dbmanage.create_settings(opts)
 
    dbmanage.create_default_user()
 
    dbmanage.admin_prompt()
 
    dbmanage.create_permissions()
 
    dbmanage.populate_default_permissions()
 
    Session().commit()
 
    load_environment(conf.global_conf, conf.local_conf)
scripts/dbmigrate-test
Show inline comments
 
modified file chmod 100755 => 100644
 
@@ -56,25 +56,25 @@ HG() {
 
# If upgrading to "current revision", warn if working directory is dirty.
 
if [ ! "$to_rev" ] && [ "$(HG status -mard)" ]; then
 
    announce "Warning: Uncommitted changes in working directory will be ignored!"
 
fi
 

	
 
from_rev_hash=$(HG id --id --rev "${from_rev:-.}")
 
to_rev_hash=$(HG id --id --rev "${to_rev:-.}")
 
temp=$(readlink -f "$(mktemp --tmpdir -d 'dbmigrate-test.XXXXXX')")
 

	
 
cat <<EOD
 
Config file:    $config_file
 
EOD
 
sed -n -e 's/^sqlalchemy\.db1\.url *= */Database URL:   /p' "$config_file"
 
sed -n -e 's/^sqlalchemy\.url *= */Database URL:   /p' "$config_file"
 
cat <<EOD
 
Working dir:    $temp
 
Repository:     $source_repo
 
Upgrade from:   $from_rev_hash (${from_rev:-current})
 
Upgrade to:     $to_rev_hash (${to_rev:-current})
 
Extra packages: ${EXTRA:-(none)}
 
EOD
 

	
 
mkdir "$temp/repos" # empty
 

	
 
# Enable caching for old pip versions (this will cache the pip upgrade)
 
# Newer pip versions cache automatically, and don't use this variable.
scripts/generate-ini.py
Show inline comments
 
modified file chmod 100755 => 100644
 
@@ -51,25 +51,25 @@ ini_files = [
 
            },
 
            '[app:main]': {
 
                'initial_repo_scan': 'true',
 
                'app_instance_uuid': 'test',
 
                'show_revision_number': 'true',
 
                'beaker.cache.sql_cache_short.expire': '1',
 
                'beaker.session.secret': '{74e0cd75-b339-478b-b129-07dd221def1f}',
 
                'cache_dir': '%(here)s/../../data/test/cache',
 
                'index_dir': '%(here)s/../../data/test/index',
 
                'archive_cache_dir': '%(here)s/../../data/test/tarballcache',
 
                'beaker.cache.data_dir': '%(here)s/../../data/test/cache/data',
 
                'beaker.cache.lock_dir': '%(here)s/../../data/test/cache/lock',
 
                'sqlalchemy.db1.url': 'sqlite:///%(here)s/kallithea_test.sqlite',
 
                'sqlalchemy.url': 'sqlite:///%(here)s/kallithea_test.sqlite',
 
            },
 
            '[handler_console]': {
 
                'level': 'DEBUG',
 
                'formatter': 'color_formatter',
 
            },
 
            # The 'handler_console_sql' block is very similar to the one in
 
            # development.ini, but without the explicit 'level=DEBUG' setting:
 
            # it causes duplicate sqlalchemy debug logs, one through
 
            # handler_console_sql and another through another path.
 
            '[handler_console_sql]': {
 
                'formatter': 'color_formatter_sql',
 
            },
0 comments (0 inline, 0 general)