Changeset - 28a4bb11bb6f
[Not reviewed]
beta
0 9 0
Marcin Kuzminski - 15 years ago 2010-12-11 03:41:27
marcin@python-works.com
dbmigrations:
added first working upgrade script
fixed wrong versions
fixed template path
9 files changed with 50 insertions and 32 deletions:
0 comments (0 inline, 0 general)
rhodecode/__init__.py
Show inline comments
 
@@ -20,25 +20,25 @@
 
# but WITHOUT ANY WARRANTY; without even the implied warranty of
 
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 
# GNU General Public License for more details.
 
# 
 
# You should have received a copy of the GNU General Public License
 
# along with this program; if not, write to the Free Software
 
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 
# MA  02110-1301, USA.
 

	
 

	
 
VERSION = (1, 1, 0, 'beta')
 
__version__ = '.'.join((str(each) for each in VERSION[:4]))
 
__dbversion__ = 1 #defines current db version for migrations
 
__dbversion__ = 2 #defines current db version for migrations
 

	
 
from rhodecode.lib.utils import get_current_revision
 
_rev = get_current_revision()
 

	
 
if len(VERSION) > 3 and _rev:
 
    __version__ += ' [rev:%s]' % _rev[0]
 

	
 
def get_version():
 
    """Returns shorter version (digit parts only) as string."""
 

	
 
    return '.'.join((str(each) for each in VERSION[:3]))
 

	
rhodecode/lib/dbmigrate/__init__.py
Show inline comments
 
@@ -19,59 +19,68 @@
 
# but WITHOUT ANY WARRANTY; without even the implied warranty of
 
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 
# GNU General Public License for more details.
 
# 
 
# You should have received a copy of the GNU General Public License
 
# along with this program; if not, write to the Free Software
 
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 
# MA  02110-1301, USA.
 

	
 
import logging
 
from sqlalchemy import engine_from_config
 

	
 
from rhodecode import __dbversion__
 
from rhodecode.lib.dbmigrate.migrate.versioning import api
 
from rhodecode.lib.dbmigrate.migrate.exceptions import \
 
    DatabaseNotControlledError
 
from rhodecode.lib.utils import BasePasterCommand, Command, add_cache
 

	
 
log = logging.getLogger(__name__)
 

	
 
class UpgradeDb(BasePasterCommand):
 
    """Command used for paster to upgrade our database to newer version
 
    """
 

	
 
    max_args = 1
 
    min_args = 1
 

	
 
    usage = "CONFIG_FILE"
 
    summary = "Upgrades current db to newer version given configuration file"
 
    group_name = "RhodeCode"
 

	
 
    parser = Command.standard_parser(verbose=True)
 

	
 
    def command(self):
 
        from pylons import config
 

	
 
        add_cache(config)
 
        #engine = engine_from_config(config, 'sqlalchemy.db1.')
 
        #rint engine
 

	
 
        from rhodecode.lib.dbmigrate.migrate.versioning import api
 
        path = 'rhodecode/lib/dbmigrate'
 
        #engine = engine_from_config(config, 'sqlalchemy.db1.')
 

	
 
        repository_path = 'rhodecode/lib/dbmigrate'
 
        db_uri = config['sqlalchemy.db1.url']
 

	
 
        try:
 
            curr_version = api.db_version(config['sqlalchemy.db1.url'], path)
 
            curr_version = api.db_version(db_uri, repository_path)
 
            msg = ('Found current database under version'
 
                 ' control with version %s' % curr_version)
 

	
 
        except (RuntimeError, DatabaseNotControlledError), e:
 
            curr_version = 0
 
            curr_version = 1
 
            msg = ('Current database is not under version control setting'
 
                   ' as version %s' % curr_version)
 
            api.version_control(db_uri, repository_path, curr_version)
 

	
 

	
 
        print msg
 
        #now we have our dbversion we can do upgrade
 

	
 
        msg = 'attempting to do database upgrade to version %s' % __dbversion__
 
        print msg
 
        api.upgrade(db_uri, repository_path, __dbversion__)
 

	
 
    def update_parser(self):
 
        self.parser.add_option('--sql',
 
                      action='store_true',
 
                      dest='just_sql',
 
                      help="Prints upgrade sql for further investigation",
 
                      default=False)
rhodecode/lib/dbmigrate/migrate/changeset/schema.py
Show inline comments
 
@@ -110,25 +110,25 @@ def alter_column(*p, **k):
 
      reflection and schema alterations.
 
    
 
    :param alter_metadata:
 
      If `True`, which is the default, the
 
      :class:`~sqlalchemy.schema.Column` will also modified.
 
      If `False`, the :class:`~sqlalchemy.schema.Column` will be left
 
      as it was.
 
    
 
    :returns: A :class:`ColumnDelta` instance representing the change.
 

	
 
    
 
    """
 
    
 

	
 
    k.setdefault('alter_metadata', DEFAULT_ALTER_METADATA)
 

	
 
    if 'table' not in k and isinstance(p[0], sqlalchemy.Column):
 
        k['table'] = p[0].table
 
    if 'engine' not in k:
 
        k['engine'] = k['table'].bind
 

	
 
    # deprecation
 
    if len(p) >= 2 and isinstance(p[1], sqlalchemy.Column):
 
        warnings.warn(
 
            "Passing a Column object to alter_column is deprecated."
 
            " Just pass in keyword parameters instead.",
 
@@ -358,25 +358,25 @@ class ColumnDelta(DictMixin, sqlalchemy.
 
            if isinstance(column.server_default, sqlalchemy.FetchedValue):
 
                toinit.append(column.server_default)
 
            else:
 
                toinit.append(sqlalchemy.DefaultClause(column.server_default))
 
        if column.server_onupdate is not None:
 
            if isinstance(column.server_onupdate, FetchedValue):
 
                toinit.append(column.server_default)
 
            else:
 
                toinit.append(sqlalchemy.DefaultClause(column.server_onupdate,
 
                                            for_update=True))
 
        if toinit:
 
            column._init_items(*toinit)
 
            
 

	
 
        if not SQLA_06:
 
            column.args = []
 

	
 
    def _get_table(self):
 
        return getattr(self, '_table', None)
 

	
 
    def _set_table(self, table):
 
        if isinstance(table, basestring):
 
            if self.alter_metadata:
 
                if not self.meta:
 
                    raise ValueError("metadata must be specified for table"
 
                        " reflection when using alter_metadata")
 
@@ -564,54 +564,54 @@ populated with defaults
 
        visitorcallable = get_engine_visitor(engine, 'columndropper')
 
        engine._run_visitor(visitorcallable, self, connection, **kwargs)
 
        if self.alter_metadata:
 
            self.table = None
 
        return self
 

	
 
    def add_to_table(self, table):
 
        if table is not None  and self.table is None:
 
            self._set_parent(table)
 

	
 
    def _col_name_in_constraint(self, cons, name):
 
        return False
 
    
 

	
 
    def remove_from_table(self, table, unset_table=True):
 
        # TODO: remove primary keys, constraints, etc
 
        if unset_table:
 
            self.table = None
 
            
 

	
 
        to_drop = set()
 
        for index in table.indexes:
 
            columns = []
 
            for col in index.columns:
 
                if col.name != self.name:
 
                    columns.append(col)
 
            if columns:
 
                index.columns = columns
 
            else:
 
                to_drop.add(index)
 
        table.indexes = table.indexes - to_drop
 
        
 

	
 
        to_drop = set()
 
        for cons in table.constraints:
 
            # TODO: deal with other types of constraint
 
            if isinstance(cons, (ForeignKeyConstraint,
 
                                UniqueConstraint)):
 
                for col_name in cons.columns:
 
                    if not isinstance(col_name, basestring):
 
                        col_name = col_name.name
 
                    if self.name == col_name:
 
                        to_drop.add(cons)
 
        table.constraints = table.constraints - to_drop
 
        
 

	
 
        if table.c.contains_column(self):
 
            table.c.remove(self)
 

	
 
    # TODO: this is fixed in 0.6
 
    def copy_fixed(self, **kw):
 
        """Create a copy of this ``Column``, with all attributes."""
 
        return sqlalchemy.Column(self.name, self.type, self.default,
 
            key=self.key,
 
            primary_key=self.primary_key,
 
            nullable=self.nullable,
 
            quote=self.quote,
 
            index=self.index,
rhodecode/lib/dbmigrate/migrate/versioning/api.py
Show inline comments
 
@@ -21,26 +21,26 @@
 
# please do not comment this module using sphinx syntax because its
 
# docstrings are presented as user help and most users cannot
 
# interpret sphinx annotated ReStructuredText.
 
#
 
# Thanks,
 
# Jan Dittberner
 

	
 
import sys
 
import inspect
 
import logging
 

	
 
from rhodecode.lib.dbmigrate.migrate import exceptions
 
from rhodecode.lib.dbmigrate.migrate.versioning import (repository, schema, version,
 
    script as script_) # command name conflict
 
from rhodecode.lib.dbmigrate.migrate.versioning import repository, schema, version, \
 
    script as script_ # command name conflict
 
from rhodecode.lib.dbmigrate.migrate.versioning.util import catch_known_errors, with_engine
 

	
 

	
 
log = logging.getLogger(__name__)
 
command_desc = {
 
    'help': 'displays help on a given command',
 
    'create': 'create an empty repository at the specified path',
 
    'script': 'create an empty change Python script',
 
    'script_sql': 'create empty change SQL scripts for given database',
 
    'version': 'display the latest version available in a repository',
 
    'db_version': 'show the current version of the repository under version control',
 
    'source': 'display the Python code for a particular version in this repository',
rhodecode/lib/dbmigrate/migrate/versioning/script/sql.py
Show inline comments
 
@@ -9,25 +9,24 @@ from rhodecode.lib.dbmigrate.migrate.ver
 

	
 
log = logging.getLogger(__name__)
 

	
 
class SqlScript(base.BaseScript):
 
    """A file containing plain SQL statements."""
 

	
 
    @classmethod
 
    def create(cls, path, **opts):
 
        """Create an empty migration script at specified path
 
        
 
        :returns: :class:`SqlScript instance <migrate.versioning.script.sql.SqlScript>`"""
 
        cls.require_notfound(path)
 

	
 
        src = Template(opts.pop('templates_path', None)).get_sql_script(theme=opts.pop('templates_theme', None))
 
        shutil.copy(src, path)
 
        return cls(path)
 

	
 
    # TODO: why is step parameter even here?
 
    def run(self, engine, step=None, executemany=True):
 
        """Runs SQL script through raw dbapi execute call"""
 
        text = self.source()
 
        # Don't rely on SA's autocommit here
 
        # (SA uses .startswith to check if a commit is needed. What if script
 
        # starts with a comment?)
 
        conn = engine.connect()
rhodecode/lib/dbmigrate/migrate/versioning/template.py
Show inline comments
 
@@ -28,25 +28,25 @@ class ScriptCollection(Collection):
 
class ManageCollection(Collection):
 
    _mask = '%s.py_tmpl'
 

	
 
class SQLScriptCollection(Collection):
 
    _mask = '%s.py_tmpl'
 

	
 
class Template(pathed.Pathed):
 
    """Finds the paths/packages of various Migrate templates.
 
    
 
    :param path: Templates are loaded from rhodecode.lib.dbmigrate.migrate package
 
    if `path` is not provided.
 
    """
 
    pkg = 'migrate.versioning.templates'
 
    pkg = 'rhodecode.lib.dbmigrate.migrate.versioning.templates'
 
    _manage = 'manage.py_tmpl'
 

	
 
    def __new__(cls, path=None):
 
        if path is None:
 
            path = cls._find_path(cls.pkg)
 
        return super(Template, cls).__new__(cls, path)
 

	
 
    def __init__(self, path=None):
 
        if path is None:
 
            path = Template._find_path(self.pkg)
 
        super(Template, self).__init__(path)
 
        self.repository = RepositoryCollection(os.path.join(path, 'repository'))
 
@@ -71,24 +71,24 @@ class Template(pathed.Pathed):
 
        :param type_: type of subfolder in collection (defaults to "_default")
 
        :returns: (package, source)
 
        :rtype: str, str
 
        """
 
        item = getattr(self, collection)
 
        theme_mask = getattr(item, '_mask')
 
        theme = theme_mask % (theme or 'default')
 
        return item.get_path(theme)
 

	
 
    def get_repository(self, *a, **kw):
 
        """Calls self._get_item('repository', *a, **kw)"""
 
        return self._get_item('repository', *a, **kw)
 
    
 

	
 
    def get_script(self, *a, **kw):
 
        """Calls self._get_item('script', *a, **kw)"""
 
        return self._get_item('script', *a, **kw)
 

	
 
    def get_sql_script(self, *a, **kw):
 
        """Calls self._get_item('sql_script', *a, **kw)"""
 
        return self._get_item('sql_script', *a, **kw)
 

	
 
    def get_manage(self, *a, **kw):
 
        """Calls self._get_item('manage', *a, **kw)"""
 
        return self._get_item('manage', *a, **kw)
rhodecode/lib/dbmigrate/versions/001_initial_release.py
Show inline comments
 
from migrate import *
 

	
 
#==============================================================================
 
# DB INITIAL MODEL
 
#==============================================================================
 
import logging
 
import datetime
 

	
 
from sqlalchemy import *
 
from sqlalchemy.exc import DatabaseError
 
from sqlalchemy.orm import relation, backref, class_mapper
 
from sqlalchemy.orm.session import Session
 
from rhodecode.model.meta import Base
 

	
 
from rhodecode.model.meta import Base
 
from rhodecode.lib.dbmigrate.migrate import *
 

	
 
log = logging.getLogger(__name__)
 

	
 
class BaseModel(object):
 

	
 
    @classmethod
 
    def _get_keys(cls):
 
        """return column names for this model """
 
        return class_mapper(cls).c.keys()
 

	
 
    def get_dict(self):
 
        """return dict with keys and values corresponding 
rhodecode/lib/dbmigrate/versions/002_version_1_1_0.py
Show inline comments
 
from sqlalchemy import *
 
from sqlalchemy.orm import relation
 
import logging
 
import datetime
 

	
 
from migrate import *
 
from migrate.changeset import *
 
from rhodecode.model.meta import Base, BaseModel
 
from sqlalchemy import *
 
from sqlalchemy.exc import DatabaseError
 
from sqlalchemy.orm import relation, backref, class_mapper
 
from sqlalchemy.orm.session import Session
 
from rhodecode.model.meta import Base
 
from rhodecode.model.db import BaseModel
 

	
 
from rhodecode.lib.dbmigrate.migrate import *
 

	
 
log = logging.getLogger(__name__)
 

	
 
def upgrade(migrate_engine):
 
    """ Upgrade operations go here. 
 
    Don't create your own engine; bind migrate_engine to your metadata
 
    """
 

	
 
    #==========================================================================
 
    # Upgrade of `users` table
 
    #==========================================================================
 
    tblname = 'users'
 
    tbl = Table(tblname, MetaData(bind=migrate_engine), autoload=True,
 
                    autoload_with=migrate_engine)
 

	
 
    #ADD is_ldap column
 
    is_ldap = Column("is_ldap", Boolean(), nullable=False,
 
    is_ldap = Column("is_ldap", Boolean(), nullable=True,
 
                     unique=None, default=False)
 
    is_ldap.create(tbl)
 

	
 
    is_ldap.create(tbl, populate_default=True)
 
    is_ldap.alter(nullable=False)
 

	
 
    #==========================================================================
 
    # Upgrade of `user_logs` table
 
    #==========================================================================    
 

	
 
    tblname = 'users'
 
    tbl = Table(tblname, MetaData(bind=migrate_engine), autoload=True,
 
                    autoload_with=migrate_engine)
 

	
 
    #ADD revision column
 
    revision = Column('revision', TEXT(length=None, convert_unicode=False,
 
                                       assert_unicode=None),
 
@@ -40,60 +47,64 @@ def upgrade(migrate_engine):
 

	
 

	
 
    #==========================================================================
 
    # Upgrade of `repositories` table
 
    #==========================================================================    
 
    tblname = 'users'
 
    tbl = Table(tblname, MetaData(bind=migrate_engine), autoload=True,
 
                    autoload_with=migrate_engine)
 

	
 
    #ADD repo_type column
 
    repo_type = Column("repo_type", String(length=None, convert_unicode=False,
 
                                           assert_unicode=None),
 
                       nullable=False, unique=False, default=None)
 
    repo_type.create(tbl)
 
                       nullable=True, unique=False, default='hg')
 

	
 
    repo_type.create(tbl, populate_default=True)
 
    repo_type.alter(nullable=False)
 

	
 
    #ADD statistics column
 
    enable_statistics = Column("statistics", Boolean(), nullable=True,
 
                               unique=None, default=True)
 
    enable_statistics.create(tbl)
 

	
 

	
 

	
 
    #==========================================================================
 
    # Add table `user_followings`
 
    #==========================================================================
 
    tblname = 'user_followings'
 

	
 
    class UserFollowing(Base, BaseModel):
 
        __tablename__ = 'user_followings'
 
        __table_args__ = (UniqueConstraint('user_id', 'follows_repository_id'),
 
                          UniqueConstraint('user_id', 'follows_user_id')
 
                          , {'useexisting':True})
 

	
 
        user_following_id = Column("user_following_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
 
        user_id = Column("user_id", Integer(), ForeignKey(u'users.user_id'), nullable=False, unique=None, default=None)
 
        follows_repo_id = Column("follows_repository_id", Integer(), ForeignKey(u'repositories.repo_id'), nullable=True, unique=None, default=None)
 
        follows_user_id = Column("follows_user_id", Integer(), ForeignKey(u'users.user_id'), nullable=True, unique=None, default=None)
 

	
 
        user = relation('User', primaryjoin='User.user_id==UserFollowing.user_id')
 

	
 
        follows_user = relation('User', primaryjoin='User.user_id==UserFollowing.follows_user_id')
 
        follows_repository = relation('Repository')
 

	
 
    Base.metadata.tables[tblname].create(migrate_engine)
 

	
 
    #==========================================================================
 
    # Add table `cache_invalidation`
 
    #==========================================================================
 
    tblname = 'cache_invalidation'
 

	
 
    class CacheInvalidation(Base, BaseModel):
 
        __tablename__ = 'cache_invalidation'
 
        __table_args__ = (UniqueConstraint('cache_key'), {'useexisting':True})
 
        cache_id = Column("cache_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
 
        cache_key = Column("cache_key", String(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
 
        cache_args = Column("cache_args", String(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
 
        cache_active = Column("cache_active", Boolean(), nullable=True, unique=None, default=False)
 

	
 

	
 
        def __init__(self, cache_key, cache_args=''):
 
            self.cache_key = cache_key
 
            self.cache_args = cache_args
rhodecode/model/db.py
Show inline comments
 
@@ -143,25 +143,25 @@ class UserLog(Base, BaseModel):
 
    user_ip = Column("user_ip", String(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
 
    action = Column("action", String(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
 
    action_date = Column("action_date", DateTime(timezone=False), nullable=True, unique=None, default=None)
 

	
 
    user = relation('User')
 
    repository = relation('Repository')
 

	
 
class Repository(Base, BaseModel):
 
    __tablename__ = 'repositories'
 
    __table_args__ = (UniqueConstraint('repo_name'), {'useexisting':True},)
 
    repo_id = Column("repo_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
 
    repo_name = Column("repo_name", String(length=None, convert_unicode=False, assert_unicode=None), nullable=False, unique=True, default=None)
 
    repo_type = Column("repo_type", String(length=None, convert_unicode=False, assert_unicode=None), nullable=False, unique=False, default=None)
 
    repo_type = Column("repo_type", String(length=None, convert_unicode=False, assert_unicode=None), nullable=False, unique=False, default='hg')
 
    user_id = Column("user_id", Integer(), ForeignKey(u'users.user_id'), nullable=False, unique=False, default=None)
 
    private = Column("private", Boolean(), nullable=True, unique=None, default=None)
 
    enable_statistics = Column("statistics", Boolean(), nullable=True, unique=None, default=True)
 
    description = Column("description", String(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
 
    fork_id = Column("fork_id", Integer(), ForeignKey(u'repositories.repo_id'), nullable=True, unique=False, default=None)
 

	
 
    user = relation('User')
 
    fork = relation('Repository', remote_side=repo_id)
 
    repo_to_perm = relation('RepoToPerm', cascade='all')
 
    stats = relation('Statistics', cascade='all', uselist=False)
 

	
 
    repo_followers = relation('UserFollowing', primaryjoin='UserFollowing.follows_repo_id==Repository.repo_id', cascade='all')
0 comments (0 inline, 0 general)