kallithea Changeset - 7f31de1584c6

Changeset - 7f31de1584c6

Parent rev.

Child rev.

[Not reviewed]

beta

0 12 0

Marcin Kuzminski - 14 years ago 2011-08-14 22:51:21
marcin@python-works.com

update migrations for 1.2

12 files changed with 216 insertions and 108 deletions:

rhodecode/lib/dbmigrate/migrate/__init__.py

rhodecode/lib/dbmigrate/migrate/changeset/__init__.py

rhodecode/lib/dbmigrate/migrate/changeset/schema.py

rhodecode/lib/dbmigrate/migrate/exceptions.py

rhodecode/lib/dbmigrate/migrate/versioning/api.py

rhodecode/lib/dbmigrate/migrate/versioning/genmodel.py

105

rhodecode/lib/dbmigrate/migrate/versioning/repository.py

rhodecode/lib/dbmigrate/migrate/versioning/schema.py

rhodecode/lib/dbmigrate/migrate/versioning/script/py.py

rhodecode/lib/dbmigrate/migrate/versioning/templates/repository/default/migrate.cfg

rhodecode/lib/dbmigrate/migrate/versioning/version.py

rhodecode/lib/dbmigrate/versions/003_version_1_2_0.py

0 comments (0 inline, 0 general)

rhodecode/lib/dbmigrate/migrate/__init__.py

➞

Show inline comments

 """
    SQLAlchemy migrate provides two APIs :mod:`migrate.versioning` for
    database schema version and repository management and
    :mod:`migrate.changeset` that allows to define database schema changes
    using Python.
 """
 from rhodecode.lib.dbmigrate.migrate.versioning import *
 from rhodecode.lib.dbmigrate.migrate.changeset import *
 __version__ = '0.7.2.dev'
@@ \ No newline at end of file @@

rhodecode/lib/dbmigrate/migrate/changeset/__init__.py

➞

Show inline comments

 """
    This module extends SQLAlchemy and provides additional DDL [#]_
    support.
    .. [#] SQL Data Definition Language
 """
 import re
 import warnings
 import sqlalchemy
 from sqlalchemy import __version__ as _sa_version
 warnings.simplefilter('always', DeprecationWarning)
 _sa_version = tuple(int(re.match("\d+", x).group(0))
+_sa_version = tuple(int(re.match("\d+", x).group(0))
                     for x in _sa_version.split("."))
 SQLA_06 = _sa_version >= (0, 6)
 SQLA_07 = _sa_version >= (0, 7)
 del re
 del _sa_version
 from rhodecode.lib.dbmigrate.migrate.changeset.schema import *
 from rhodecode.lib.dbmigrate.migrate.changeset.constraint import *
 sqlalchemy.schema.Table.__bases__ += (ChangesetTable,)
 sqlalchemy.schema.Column.__bases__ += (ChangesetColumn,)
 sqlalchemy.schema.Index.__bases__ += (ChangesetIndex,)
 sqlalchemy.schema.DefaultClause.__bases__ += (ChangesetDefaultClause,)

rhodecode/lib/dbmigrate/migrate/changeset/schema.py

➞

Show inline comments

 """
    Schema module providing common schema operations.
 """
 import warnings
 from UserDict import DictMixin
 import sqlalchemy
 from sqlalchemy.schema import ForeignKeyConstraint
 from sqlalchemy.schema import UniqueConstraint
 from rhodecode.lib.dbmigrate.migrate.exceptions import *
 from rhodecode.lib.dbmigrate.migrate.changeset import SQLA_06
+from rhodecode.lib.dbmigrate.migrate.changeset import SQLA_06, SQLA_07
 from rhodecode.lib.dbmigrate.migrate.changeset.databases.visitor import (get_engine_visitor,
                                                  run_single_visitor)
+                                                                         run_single_visitor)
 __all__ = [
     'create_column',
     'drop_column',
     'alter_column',
     'rename_table',
     'rename_index',
     'ChangesetTable',
     'ChangesetColumn',
     'ChangesetIndex',
     'ChangesetDefaultClause',
     'ColumnDelta',
+]
 def create_column(column, table=None, *p, **kw):
     """Create a column, given the table.
     API to :meth:`ChangesetColumn.create`.
     """
     if table is not None:
         return table.create_column(column, *p, **kw)
     return column.create(*p, **kw)
@@ @@ -534,84 +534,90 @@ populated with defaults @@
             stmt = table.update().values({self: engine._execute_default(self.default)})
             engine.execute(stmt)
         return self
     def drop(self, table=None, connection=None, **kwargs):
         """Drop this column from the database, leaving its table intact.
         ``ALTER TABLE DROP COLUMN``, for most databases.
         :param connection: reuse connection istead of creating new one.
         :type connection: :class:`sqlalchemy.engine.base.Connection` instance
         """
         if table is not None:
             self.table = table
         engine = self.table.bind
         visitorcallable = get_engine_visitor(engine, 'columndropper')
         engine._run_visitor(visitorcallable, self, connection, **kwargs)
         self.remove_from_table(self.table, unset_table=False)
         self.table = None
         return self
     def add_to_table(self, table):
         if table is not None  and self.table is None:
             self._set_parent(table)
             if SQLA_07:
                 table.append_column(self)
             else:
                 self._set_parent(table)
     def _col_name_in_constraint(self,cons,name):
         return False
     def remove_from_table(self, table, unset_table=True):
         # TODO: remove primary keys, constraints, etc
         if unset_table:
             self.table = None
         to_drop = set()
         for index in table.indexes:
             columns = []
             for col in index.columns:
                 if col.name!=self.name:
                     columns.append(col)
             if columns:
                 index.columns=columns
             else:
                 to_drop.add(index)
         table.indexes = table.indexes - to_drop
         to_drop = set()
         for cons in table.constraints:
             # TODO: deal with other types of constraint
             if isinstance(cons,(ForeignKeyConstraint,
                                 UniqueConstraint)):
                 for col_name in cons.columns:
                     if not isinstance(col_name,basestring):
                         col_name = col_name.name
                     if self.name==col_name:
                         to_drop.add(cons)
         table.constraints = table.constraints - to_drop
         if table.c.contains_column(self):
             table.c.remove(self)
             if SQLA_07:
                 table._columns.remove(self)
             else:
                 table.c.remove(self)
     # TODO: this is fixed in 0.6
     def copy_fixed(self, **kw):
         """Create a copy of this ``Column``, with all attributes."""
         return sqlalchemy.Column(self.name, self.type, self.default,
             key=self.key,
             primary_key=self.primary_key,
             nullable=self.nullable,
             quote=self.quote,
             index=self.index,
             unique=self.unique,
             onupdate=self.onupdate,
             autoincrement=self.autoincrement,
             server_default=self.server_default,
             server_onupdate=self.server_onupdate,
             *[c.copy(**kw) for c in self.constraints])
     def _check_sanity_constraints(self, name):
         """Check if constraints names are correct"""
         obj = getattr(self, name)
         if (getattr(self, name[:-5]) and not obj):
             raise InvalidConstraintError("Column.create() accepts index_name,"
             " primary_key_name and unique_name to generate constraints")
         if not isinstance(obj, basestring) and obj is not None:

rhodecode/lib/dbmigrate/migrate/exceptions.py

➞

Show inline comments

@@ @@ -50,34 +50,39 @@ class PathError(Error): @@
 class PathNotFoundError(PathError):
     """A path with no file was required; found a file."""
 class PathFoundError(PathError):
     """A path with a file was required; found no file."""
 class RepositoryError(Error):
     """Base class for repository errors."""
 class InvalidRepositoryError(RepositoryError):
     """Invalid repository error."""
 class ScriptError(Error):
     """Base class for script errors."""
 class InvalidScriptError(ScriptError):
     """Invalid script error."""
 class InvalidVersionError(Error):
     """Invalid version error."""
 # migrate.changeset
 class NotSupportedError(Error):
     """Not supported error"""
 class InvalidConstraintError(Error):
     """Invalid constraint error"""
 class MigrateDeprecationWarning(DeprecationWarning):
     """Warning for deprecated features in Migrate"""

rhodecode/lib/dbmigrate/migrate/versioning/api.py

➞

Show inline comments

@@ @@ -89,61 +89,61 @@ def create(repository, name, **opts): @@
     Create an empty repository at the specified path.
     You can specify the version_table to be used; by default, it is
     'migrate_version'.  This table is created in all version-controlled
     databases.
     """
     repo_path = Repository.create(repository, name, **opts)
 @catch_known_errors
 def script(description, repository, **opts):
     """%prog script DESCRIPTION REPOSITORY_PATH
     Create an empty change script using the next unused version number
     appended with the given description.
     For instance, manage.py script "Add initial tables" creates:
     repository/versions/001_Add_initial_tables.py
     """
     repo = Repository(repository)
     repo.create_script(description, **opts)
 @catch_known_errors
 def script_sql(database, repository, **opts):
     """%prog script_sql DATABASE REPOSITORY_PATH
 def script_sql(database, description, repository, **opts):
     """%prog script_sql DATABASE DESCRIPTION REPOSITORY_PATH
     Create empty change SQL scripts for given DATABASE, where DATABASE
-    is either specific ('postgres', 'mysql', 'oracle', 'sqlite', etc.)
+    is either specific ('postgresql', 'mysql', 'oracle', 'sqlite', etc.)
     or generic ('default').
     For instance, manage.py script_sql postgres creates:
     repository/versions/001_postgres_upgrade.sql and
     repository/versions/001_postgres_postgres.sql
     For instance, manage.py script_sql postgresql description creates:
     repository/versions/001_description_postgresql_upgrade.sql and
     repository/versions/001_description_postgresql_postgres.sql
     """
     repo = Repository(repository)
     repo.create_script_sql(database, **opts)
+    repo.create_script_sql(database, description, **opts)
 def version(repository, **opts):
     """%prog version REPOSITORY_PATH
     Display the latest version available in a repository.
     """
     repo = Repository(repository)
     return repo.latest
 @with_engine
 def db_version(url, repository, **opts):
     """%prog db_version URL REPOSITORY_PATH
     Show the current version of the repository with the given
     connection string, under version control of the specified
     repository.
     The url should be any valid SQLAlchemy connection string.
     """
     engine = opts.pop('engine')
     schema = ControlledSchema(engine, repository)
     return schema.version

rhodecode/lib/dbmigrate/migrate/versioning/genmodel.py

➞

Show inline comments

 """
    Code to generate a Python model from a database or differences
    between a model and database.
 Code to generate a Python model from a database or differences
 between a model and database.
    Some of this is borrowed heavily from the AutoCode project at:
    http://code.google.com/p/sqlautocode/
 Some of this is borrowed heavily from the AutoCode project at:
 http://code.google.com/p/sqlautocode/
 """
 import sys
 import logging
 import sqlalchemy
 from rhodecode.lib.dbmigrate import migrate
 from rhodecode.lib.dbmigrate.migrate import changeset
 log = logging.getLogger(__name__)
 HEADER = """
 ## File autogenerated by genmodel.py
 from sqlalchemy import *
 meta = MetaData()
 """
 DECLARATIVE_HEADER = """
 ## File autogenerated by genmodel.py
 from sqlalchemy import *
 from sqlalchemy.ext import declarative
 Base = declarative.declarative_base()
 """
 class ModelGenerator(object):
     """Various transformations from an A, B diff.
     In the implementation, A tends to be called the model and B
     the database (although this is not true of all diffs).
     The diff is directionless, but transformations apply the diff
     in a particular direction, described in the method name.
     """
     def __init__(self, diff, engine, declarative=False):
         self.diff = diff
         self.engine = engine
         self.declarative = declarative
     def column_repr(self, col):
         kwarg = []
         if col.key != col.name:
             kwarg.append('key')
         if col.primary_key:
             col.primary_key = True  # otherwise it dumps it as 1
             kwarg.append('primary_key')
         if not col.nullable:
             kwarg.append('nullable')
         if col.onupdate:
             kwarg.append('onupdate')
         if col.default:
             if col.primary_key:
                 # I found that PostgreSQL automatically creates a
                 # default value for the sequence, but let's not show
                 # that.
                 pass
             else:
                 kwarg.append('default')
-        ks = ', '.join('%s=%r' % (k, getattr(col, k)) for k in kwarg)
+        args = ['%s=%r' % (k, getattr(col, k)) for k in kwarg]
         # crs: not sure if this is good idea, but it gets rid of extra
         # u''
         name = col.name.encode('utf8')
         type_ = col.type
         for cls in col.type.__class__.__mro__:
             if cls.__module__ == 'sqlalchemy.types' and \
                 not cls.__name__.isupper():
                 if cls is not type_.__class__:
                     type_ = cls()
                 break
         type_repr = repr(type_)
         if type_repr.endswith('()'):
             type_repr = type_repr[:-2]
         constraints = [repr(cn) for cn in col.constraints]
         data = {
             'name': name,
             'type': type_,
             'constraints': ', '.join([repr(cn) for cn in col.constraints]),
             'args': ks and ks or ''}
             'commonStuff': ', '.join([type_repr] + constraints + args),
+        }
         if data['constraints']:
             if data['args']:
                 data['args'] = ',' + data['args']
         if data['constraints'] or data['args']:
             data['maybeComma'] = ','
         if self.declarative:
             return """%(name)s = Column(%(commonStuff)s)""" % data
         else:
-            data['maybeComma'] = ''
+            return """Column(%(name)r, %(commonStuff)s)""" % data
         commonStuff = """ %(maybeComma)s %(constraints)s %(args)s)""" % data
         commonStuff = commonStuff.strip()
         data['commonStuff'] = commonStuff
         if self.declarative:
             return """%(name)s = Column(%(type)r%(commonStuff)s""" % data
         else:
             return """Column(%(name)r, %(type)r%(commonStuff)s""" % data
     def getTableDefn(self, table):
     def _getTableDefn(self, table, metaName='meta'):
         out = []
         tableName = table.name
         if self.declarative:
             out.append("class %(table)s(Base):" % {'table': tableName})
             out.append("  __tablename__ = '%(table)s'" % {'table': tableName})
             out.append("    __tablename__ = '%(table)s'\n" %
                             {'table': tableName})
             for col in table.columns:
                 out.append("  %s" % self.column_repr(col))
                 out.append("    %s" % self.column_repr(col))
             out.append('\n')
         else:
             out.append("%(table)s = Table('%(table)s', meta," % \
                            {'table': tableName})
             out.append("%(table)s = Table('%(table)s', %(meta)s," %
                        {'table': tableName, 'meta': metaName})
             for col in table.columns:
                 out.append("  %s," % self.column_repr(col))
             out.append(")")
                 out.append("    %s," % self.column_repr(col))
             out.append(")\n")
         return out
     def _get_tables(self,missingA=False,missingB=False,modified=False):
         to_process = []
         for bool_,names,metadata in (
             (missingA,self.diff.tables_missing_from_A,self.diff.metadataB),
             (missingB,self.diff.tables_missing_from_B,self.diff.metadataA),
             (modified,self.diff.tables_different,self.diff.metadataA),
                 ):
             if bool_:
                 for name in names:
                     yield metadata.tables.get(name)
     def toPython(self):
         """Assume database is current and model is empty."""
     def genBDefinition(self):
         """Generates the source code for a definition of B.
         Assumes a diff where A is empty.
         Was: toPython. Assume database (B) is current and model (A) is empty.
         """
         out = []
         if self.declarative:
             out.append(DECLARATIVE_HEADER)
         else:
             out.append(HEADER)
         out.append("")
         for table in self._get_tables(missingA=True):
             out.extend(self.getTableDefn(table))
             out.append("")
             out.extend(self._getTableDefn(table))
         return '\n'.join(out)
     def toUpgradeDowngradePython(self, indent='    '):
         ''' Assume model is most current and database is out-of-date. '''
         decls = ['from rhodecode.lib.dbmigrate.migrate.changeset import schema',
                  'meta = MetaData()']
         for table in self._get_tables(
             missingA=True,missingB=True,modified=True
             ):
             decls.extend(self.getTableDefn(table))
     def genB2AMigration(self, indent='    '):
         '''Generate a migration from B to A.
         Was: toUpgradeDowngradePython
         Assume model (A) is most current and database (B) is out-of-date.
         '''
         decls = ['from migrate.changeset import schema',
                  'pre_meta = MetaData()',
                  'post_meta = MetaData()',
+                ]
         upgradeCommands = ['pre_meta.bind = migrate_engine',
                            'post_meta.bind = migrate_engine']
         downgradeCommands = list(upgradeCommands)
         for tn in self.diff.tables_missing_from_A:
             pre_table = self.diff.metadataB.tables[tn]
             decls.extend(self._getTableDefn(pre_table, metaName='pre_meta'))
             upgradeCommands.append(
                 "pre_meta.tables[%(table)r].drop()" % {'table': tn})
             downgradeCommands.append(
                 "pre_meta.tables[%(table)r].create()" % {'table': tn})
         upgradeCommands, downgradeCommands = [], []
         for tableName in self.diff.tables_missing_from_A:
             upgradeCommands.append("%(table)s.drop()" % {'table': tableName})
             downgradeCommands.append("%(table)s.create()" % \
                                          {'table': tableName})
         for tableName in self.diff.tables_missing_from_B:
             upgradeCommands.append("%(table)s.create()" % {'table': tableName})
             downgradeCommands.append("%(table)s.drop()" % {'table': tableName})
         for tn in self.diff.tables_missing_from_B:
             post_table = self.diff.metadataA.tables[tn]
             decls.extend(self._getTableDefn(post_table, metaName='post_meta'))
             upgradeCommands.append(
                 "post_meta.tables[%(table)r].create()" % {'table': tn})
             downgradeCommands.append(
                 "post_meta.tables[%(table)r].drop()" % {'table': tn})
         for tableName in self.diff.tables_different:
             dbTable = self.diff.metadataB.tables[tableName]
             missingInDatabase, missingInModel, diffDecl = \
                 self.diff.colDiffs[tableName]
             for col in missingInDatabase:
                 upgradeCommands.append('%s.columns[%r].create()' % (
                         modelTable, col.name))
                 downgradeCommands.append('%s.columns[%r].drop()' % (
                         modelTable, col.name))
             for col in missingInModel:
                 upgradeCommands.append('%s.columns[%r].drop()' % (
                         modelTable, col.name))
                 downgradeCommands.append('%s.columns[%r].create()' % (
                         modelTable, col.name))
             for modelCol, databaseCol, modelDecl, databaseDecl in diffDecl:
         for (tn, td) in self.diff.tables_different.iteritems():
             if td.columns_missing_from_A or td.columns_different:
                 pre_table = self.diff.metadataB.tables[tn]
                 decls.extend(self._getTableDefn(
                     pre_table, metaName='pre_meta'))
             if td.columns_missing_from_B or td.columns_different:
                 post_table = self.diff.metadataA.tables[tn]
                 decls.extend(self._getTableDefn(
                     post_table, metaName='post_meta'))
             for col in td.columns_missing_from_A:
                 upgradeCommands.append(
                     'pre_meta.tables[%r].columns[%r].drop()' % (tn, col))
                 downgradeCommands.append(
                     'pre_meta.tables[%r].columns[%r].create()' % (tn, col))
             for col in td.columns_missing_from_B:
                 upgradeCommands.append(
                     'post_meta.tables[%r].columns[%r].create()' % (tn, col))
                 downgradeCommands.append(
                     'post_meta.tables[%r].columns[%r].drop()' % (tn, col))
             for modelCol, databaseCol, modelDecl, databaseDecl in td.columns_different:
                 upgradeCommands.append(
                     'assert False, "Can\'t alter columns: %s:%s=>%s"' % (
-                    modelTable, modelCol.name, databaseCol.name))
+                    tn, modelCol.name, databaseCol.name))
                 downgradeCommands.append(
                     'assert False, "Can\'t alter columns: %s:%s=>%s"' % (
                     modelTable, modelCol.name, databaseCol.name))
         pre_command = '    meta.bind = migrate_engine'
                     tn, modelCol.name, databaseCol.name))
         return (
             '\n'.join(decls),
             '\n'.join([pre_command] + ['%s%s' % (indent, line) for line in upgradeCommands]),
             '\n'.join([pre_command] + ['%s%s' % (indent, line) for line in downgradeCommands]))
             '\n'.join('%s%s' % (indent, line) for line in upgradeCommands),
             '\n'.join('%s%s' % (indent, line) for line in downgradeCommands))
     def _db_can_handle_this_change(self,td):
         """Check if the database can handle going from B to A."""
         if (td.columns_missing_from_B
             and not td.columns_missing_from_A
             and not td.columns_different):
-            # Even sqlite can handle this.
+            # Even sqlite can handle column additions.
             return True
         else:
             return not self.engine.url.drivername.startswith('sqlite')
     def applyModel(self):
         """Apply model to current database."""
     def runB2A(self):
         """Goes from B to A.
         Was: applyModel. Apply model (A) to current database (B).
         """
         meta = sqlalchemy.MetaData(self.engine)
         for table in self._get_tables(missingA=True):
             table = table.tometadata(meta)
             table.drop()
         for table in self._get_tables(missingB=True):
             table = table.tometadata(meta)
             table.create()
         for modelTable in self._get_tables(modified=True):
             tableName = modelTable.name
             modelTable = modelTable.tometadata(meta)
             dbTable = self.diff.metadataB.tables[tableName]
             td = self.diff.tables_different[tableName]
             if self._db_can_handle_this_change(td):
                 for col in td.columns_missing_from_B:
                     modelTable.columns[col].create()
                 for col in td.columns_missing_from_A:
                     dbTable.columns[col].drop()
                 # XXX handle column changes here.
             else:
@@ @@ -230,24 +261,25 @@ class ModelGenerator(object): @@
                         if modelCol.name in dbTable.columns:
                             commonCols.append(modelCol.name)
                     commonColsStr = ', '.join(commonCols)
                     return 'INSERT INTO %s (%s) SELECT %s FROM %s' % \
                         (tableName, commonColsStr, commonColsStr, tempName)
                 # Move the data in one transaction, so that we don't
                 # leave the database in a nasty state.
                 connection = self.engine.connect()
                 trans = connection.begin()
                 try:
                     connection.execute(
                         'CREATE TEMPORARY TABLE %s as SELECT * from %s' % \
                             (tempName, modelTable.name))
                     # make sure the drop takes place inside our
                     # transaction with the bind parameter
                     modelTable.drop(bind=connection)
                     modelTable.create(bind=connection)
                     connection.execute(getCopyStatement())
                     connection.execute('DROP TABLE %s' % tempName)
                     trans.commit()
                 except:
                     trans.rollback()
                     raise

rhodecode/lib/dbmigrate/migrate/versioning/repository.py

➞

Show inline comments

@@ @@ -94,106 +94,117 @@ class Repository(pathed.Pathed): @@
             cls.require_found(path)
             cls.require_found(os.path.join(path, cls._config))
             cls.require_found(os.path.join(path, cls._versions))
         except exceptions.PathNotFoundError, e:
             raise exceptions.InvalidRepositoryError(path)
     @classmethod
     def prepare_config(cls, tmpl_dir, name, options=None):
         """
         Prepare a project configuration file for a new project.
         :param tmpl_dir: Path to Repository template
         :param config_file: Name of the config file in Repository template
         :param name: Repository name
         :type tmpl_dir: string
         :type config_file: string
         :type name: string
         :returns: Populated config file
         """
         if options is None:
             options = {}
         options.setdefault('version_table', 'migrate_version')
         options.setdefault('repository_id', name)
         options.setdefault('required_dbs', [])
         options.setdefault('use_timestamp_numbering', '0')
         tmpl = open(os.path.join(tmpl_dir, cls._config)).read()
         ret = TempitaTemplate(tmpl).substitute(options)
         # cleanup
         del options['__template_name__']
         return ret
     @classmethod
     def create(cls, path, name, **opts):
         """Create a repository at a specified path"""
         cls.require_notfound(path)
         theme = opts.pop('templates_theme', None)
         t_path = opts.pop('templates_path', None)
         # Create repository
         tmpl_dir = Template(t_path).get_repository(theme=theme)
         shutil.copytree(tmpl_dir, path)
         # Edit config defaults
         config_text = cls.prepare_config(tmpl_dir, name, options=opts)
         fd = open(os.path.join(path, cls._config), 'w')
         fd.write(config_text)
         fd.close()
         opts['repository_name'] = name
         # Create a management script
         manager = os.path.join(path, 'manage.py')
         Repository.create_manage_file(manager, templates_theme=theme,
             templates_path=t_path, **opts)
         return cls(path)
     def create_script(self, description, **k):
         """API to :meth:`migrate.versioning.version.Collection.create_new_python_version`"""
         k['use_timestamp_numbering'] = self.use_timestamp_numbering
         self.versions.create_new_python_version(description, **k)
     def create_script_sql(self, database, **k):
+    def create_script_sql(self, database, description, **k):
         """API to :meth:`migrate.versioning.version.Collection.create_new_sql_version`"""
         self.versions.create_new_sql_version(database, **k)
         k['use_timestamp_numbering'] = self.use_timestamp_numbering
         self.versions.create_new_sql_version(database, description, **k)
     @property
     def latest(self):
         """API to :attr:`migrate.versioning.version.Collection.latest`"""
         return self.versions.latest
     @property
     def version_table(self):
         """Returns version_table name specified in config"""
         return self.config.get('db_settings', 'version_table')
     @property
     def id(self):
         """Returns repository id specified in config"""
         return self.config.get('db_settings', 'repository_id')
     @property
     def use_timestamp_numbering(self):
         """Returns use_timestamp_numbering specified in config"""
         ts_numbering = self.config.get('db_settings', 'use_timestamp_numbering', raw=True)
         return ts_numbering
     def version(self, *p, **k):
         """API to :attr:`migrate.versioning.version.Collection.version`"""
         return self.versions.version(*p, **k)
     @classmethod
     def clear(cls):
         # TODO: deletes repo
         super(Repository, cls).clear()
         version.Collection.clear()
     def changeset(self, database, start, end=None):
         """Create a changeset to migrate this database from ver. start to end/latest.
         :param database: name of database to generate changeset
         :param start: version to start at
         :param end: version to end at (latest if None given)
         :type database: string
         :type start: int
         :type end: int
         :returns: :class:`Changeset instance <migration.versioning.repository.Changeset>`
         """
         start = version.VerNum(start)
         if end is None:

rhodecode/lib/dbmigrate/migrate/versioning/schema.py

➞

Show inline comments

 """
    Database schema version management.
 """
 import sys
 import logging
 from sqlalchemy import (Table, Column, MetaData, String, Text, Integer,
     create_engine)
 from sqlalchemy.sql import and_
 from sqlalchemy import exceptions as sa_exceptions
 from sqlalchemy.sql import bindparam
 from rhodecode.lib.dbmigrate.migrate import exceptions
 from rhodecode.lib.dbmigrate.migrate.changeset import SQLA_07
 from rhodecode.lib.dbmigrate.migrate.versioning import genmodel, schemadiff
 from rhodecode.lib.dbmigrate.migrate.versioning.repository import Repository
 from rhodecode.lib.dbmigrate.migrate.versioning.util import load_model
 from rhodecode.lib.dbmigrate.migrate.versioning.version import VerNum
 log = logging.getLogger(__name__)
 class ControlledSchema(object):
     """A database under version control"""
     def __init__(self, engine, repository):
         if isinstance(repository, basestring):
             repository = Repository(repository)
         self.engine = engine
         self.repository = repository
         self.meta = MetaData(engine)
         self.load()
     def __eq__(self, other):
         """Compare two schemas by repositories and versions"""
         return (self.repository is other.repository \
             and self.version == other.version)
     def load(self):
         """Load controlled schema version info from DB"""
         tname = self.repository.version_table
         try:
             if not hasattr(self, 'table') or self.table is None:
                     self.table = Table(tname, self.meta, autoload=True)
             result = self.engine.execute(self.table.select(
                 self.table.c.repository_id == str(self.repository.id)))
             data = list(result)[0]
         except:
             cls, exc, tb = sys.exc_info()
             raise exceptions.DatabaseNotControlledError, exc.__str__(), tb
         self.version = data['version']
         return data
     def drop(self):
         """
         Remove version control from a database.
         """
         try:
             self.table.drop()
         except (sa_exceptions.SQLError):
             raise exceptions.DatabaseNotControlledError(str(self.table))
         if SQLA_07:
             try:
                 self.table.drop()
             except sa_exceptions.DatabaseError:
                 raise exceptions.DatabaseNotControlledError(str(self.table))
         else:
             try:
                 self.table.drop()
             except (sa_exceptions.SQLError):
                 raise exceptions.DatabaseNotControlledError(str(self.table))
     def changeset(self, version=None):
         """API to Changeset creation.
         Uses self.version for start version and engine.name
         to get database name.
         """
         database = self.engine.name
         start_ver = self.version
         changeset = self.repository.changeset(database, start_ver, version)
         return changeset
     def runchange(self, ver, change, step):
         startver = ver
         endver = ver + step
         # Current database version must be correct! Don't run if corrupt!
         if self.version != startver:
             raise exceptions.InvalidVersionError("%s is not %s" % \
                                                      (self.version, startver))
         # Run the change
         change.run(self.engine, step)
         # Update/refresh database version
         self.update_repository_table(startver, endver)
@@ @@ -89,49 +96,49 @@ class ControlledSchema(object): @@
     def update_repository_table(self, startver, endver):
         """Update version_table with new information"""
         update = self.table.update(and_(self.table.c.version == int(startver),
              self.table.c.repository_id == str(self.repository.id)))
         self.engine.execute(update, version=int(endver))
     def upgrade(self, version=None):
         """
         Upgrade (or downgrade) to a specified version, or latest version.
         """
         changeset = self.changeset(version)
         for ver, change in changeset:
             self.runchange(ver, change, changeset.step)
     def update_db_from_model(self, model):
         """
         Modify the database to match the structure of the current Python model.
         """
         model = load_model(model)
         diff = schemadiff.getDiffOfModelAgainstDatabase(
             model, self.engine, excludeTables=[self.repository.version_table]
+            )
-        genmodel.ModelGenerator(diff,self.engine).applyModel()
+        genmodel.ModelGenerator(diff,self.engine).runB2A()
         self.update_repository_table(self.version, int(self.repository.latest))
         self.load()
     @classmethod
     def create(cls, engine, repository, version=None):
         """
         Declare a database to be under a repository's version control.
         :raises: :exc:`DatabaseAlreadyControlledError`
         :returns: :class:`ControlledSchema`
         """
         # Confirm that the version # is valid: positive, integer,
         # exists in repos
         if isinstance(repository, basestring):
             repository = Repository(repository)
         version = cls._validate_version(repository, version)
         table = cls._create_table_version(engine, repository, version)
         # TODO: history table
         # Load repository information and return
         return cls(engine, repository)
     @classmethod
@@ @@ -189,25 +196,25 @@ class ControlledSchema(object): @@
     @classmethod
     def compare_model_to_db(cls, engine, model, repository):
         """
         Compare the current model against the current database.
         """
         if isinstance(repository, basestring):
             repository = Repository(repository)
         model = load_model(model)
         diff = schemadiff.getDiffOfModelAgainstDatabase(
             model, engine, excludeTables=[repository.version_table])
         return diff
     @classmethod
     def create_model(cls, engine, repository, declarative=False):
         """
         Dump the current database as a Python model.
         """
         if isinstance(repository, basestring):
             repository = Repository(repository)
         diff = schemadiff.getDiffOfModelAgainstDatabase(
             MetaData(), engine, excludeTables=[repository.version_table]
+            )
-        return genmodel.ModelGenerator(diff, engine, declarative).toPython()
+        return genmodel.ModelGenerator(diff, engine, declarative).genBDefinition()

rhodecode/lib/dbmigrate/migrate/versioning/script/py.py

➞

Show inline comments

@@ @@ -40,54 +40,54 @@ class PythonScript(base.BaseScript): @@
         """Create a migration script based on difference between two SA models.
         :param repository: path to migrate repository
         :param oldmodel: dotted.module.name:SAClass or SAClass object
         :param model: dotted.module.name:SAClass or SAClass object
         :param engine: SQLAlchemy engine
         :type repository: string or :class:`Repository instance <migrate.versioning.repository.Repository>`
         :type oldmodel: string or Class
         :type model: string or Class
         :type engine: Engine instance
         :returns: Upgrade / Downgrade script
         :rtype: string
         """
         if isinstance(repository, basestring):
             # oh dear, an import cycle!
             from rhodecode.lib.dbmigrate.migrate.versioning.repository import Repository
             repository = Repository(repository)
         oldmodel = load_model(oldmodel)
         model = load_model(model)
         # Compute differences.
         diff = schemadiff.getDiffOfModelAgainstModel(
             model,
             oldmodel,
             model,
             excludeTables=[repository.version_table])
         # TODO: diff can be False (there is no difference?)
         decls, upgradeCommands, downgradeCommands = \
-            genmodel.ModelGenerator(diff,engine).toUpgradeDowngradePython()
+            genmodel.ModelGenerator(diff,engine).genB2AMigration()
         # Store differences into file.
         src = Template(opts.pop('templates_path', None)).get_script(opts.pop('templates_theme', None))
         f = open(src)
         contents = f.read()
         f.close()
         # generate source
         search = 'def upgrade(migrate_engine):'
         contents = contents.replace(search, '\n\n'.join((decls, search)), 1)
         if upgradeCommands:
             contents = contents.replace('    pass', upgradeCommands, 1)
         if downgradeCommands:
             contents = contents.replace('    pass', downgradeCommands, 1)
         return contents
     @classmethod
     def verify_module(cls, path):
         """Ensure path is a valid script
         :param path: Script location
         :type path: string
         :raises: :exc:`InvalidScriptError <migrate.exceptions.InvalidScriptError>`
         :returns: Python module

rhodecode/lib/dbmigrate/migrate/versioning/templates/repository/default/migrate.cfg

➞

Show inline comments

 [db_settings]
 # Used to identify which repository this database is versioned under.
 # You can use the name of your project.
 repository_id={{ locals().pop('repository_id') }}
 # The name of the database table used to track the schema version.
 # This name shouldn't already be used by your project.
 # If this is changed once a database is under version control, you'll need to
 # change the table name in each database too.
 version_table={{ locals().pop('version_table') }}
 # When committing a change script, Migrate will attempt to generate the
 # sql for all supported databases; normally, if one of them fails - probably
 # because you don't have that database installed - it is ignored and the
 # commit continues, perhaps ending successfully.
 # Databases in this list MUST compile successfully during a commit, or the
 # entire commit will fail. List the databases your application will actually
 # be using to ensure your updates to that database work properly.
 # This must be a list; example: ['postgres','sqlite']
 required_dbs={{ locals().pop('required_dbs') }}
 # When creating new change scripts, Migrate will stamp the new script with
 # a version number. By default this is latest_version + 1. You can set this
 # to 'true' to tell Migrate to use the UTC timestamp instead.
 use_timestamp_numbering='false'
@@ \ No newline at end of file @@

rhodecode/lib/dbmigrate/migrate/versioning/version.py

➞

Show inline comments

 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 import os
 import re
 import shutil
 import logging
 from rhodecode.lib.dbmigrate.migrate import exceptions
 from rhodecode.lib.dbmigrate.migrate.versioning import pathed, script
 from datetime import datetime
 log = logging.getLogger(__name__)
 class VerNum(object):
     """A version number that behaves like a string and int at the same time"""
     _instances = dict()
     def __new__(cls, value):
         val = str(value)
         if val not in cls._instances:
             cls._instances[val] = super(VerNum, cls).__new__(cls)
         ret = cls._instances[val]
         return ret
     def __init__(self,value):
         self.value = str(int(value))
         if self < 0:
             raise ValueError("Version number cannot be negative")
     def __add__(self, value):
         ret = int(self) + int(value)
         return VerNum(ret)
@@ @@ -38,177 +39,201 @@ class VerNum(object): @@
     def __cmp__(self, value):
         return int(self) - int(value)
     def __repr__(self):
         return "<VerNum(%s)>" % self.value
     def __str__(self):
         return str(self.value)
     def __int__(self):
         return int(self.value)
 class Collection(pathed.Pathed):
     """A collection of versioning scripts in a repository"""
     FILENAME_WITH_VERSION = re.compile(r'^(\d{3,}).*')
     def __init__(self, path):
         """Collect current version scripts in repository
         and store them in self.versions
         """
         super(Collection, self).__init__(path)
         # Create temporary list of files, allowing skipped version numbers.
         files = os.listdir(path)
         if '1' in files:
             # deprecation
             raise Exception('It looks like you have a repository in the old '
                 'format (with directories for each version). '
                 'Please convert repository before proceeding.')
         tempVersions = dict()
         for filename in files:
             match = self.FILENAME_WITH_VERSION.match(filename)
             if match:
                 num = int(match.group(1))
                 tempVersions.setdefault(num, []).append(filename)
             else:
                 pass  # Must be a helper file or something, let's ignore it.
         # Create the versions member where the keys
         # are VerNum's and the values are Version's.
         self.versions = dict()
         for num, files in tempVersions.items():
             self.versions[VerNum(num)] = Version(num, path, files)
     @property
     def latest(self):
         """:returns: Latest version in Collection"""
         return max([VerNum(0)] + self.versions.keys())
     def _next_ver_num(self, use_timestamp_numbering):
         print use_timestamp_numbering
         if use_timestamp_numbering == True:
             print "Creating new timestamp version!"
             return VerNum(int(datetime.utcnow().strftime('%Y%m%d%H%M%S')))
         else:
             return self.latest + 1
     def create_new_python_version(self, description, **k):
         """Create Python files for new version"""
-        ver = self.latest + 1
+        ver = self._next_ver_num(k.pop('use_timestamp_numbering', False))
         extra = str_to_filename(description)
         if extra:
             if extra == '_':
                 extra = ''
             elif not extra.startswith('_'):
                 extra = '_%s' % extra
         filename = '%03d%s.py' % (ver, extra)
         filepath = self._version_path(filename)
         script.PythonScript.create(filepath, **k)
         self.versions[ver] = Version(ver, self.path, [filename])
     def create_new_sql_version(self, database, **k):
     def create_new_sql_version(self, database, description, **k):
         """Create SQL files for new version"""
-        ver = self.latest + 1
+        ver = self._next_ver_num(k.pop('use_timestamp_numbering', False))
         self.versions[ver] = Version(ver, self.path, [])
         extra = str_to_filename(description)
         if extra:
             if extra == '_':
                 extra = ''
             elif not extra.startswith('_'):
                 extra = '_%s' % extra
         # Create new files.
         for op in ('upgrade', 'downgrade'):
-            filename = '%03d_%s_%s.sql' % (ver, database, op)
+            filename = '%03d%s_%s_%s.sql' % (ver, extra, database, op)
             filepath = self._version_path(filename)
             script.SqlScript.create(filepath, **k)
             self.versions[ver].add_script(filepath)
     def version(self, vernum=None):
         """Returns latest Version if vernum is not given.
         Otherwise, returns wanted version"""
         if vernum is None:
             vernum = self.latest
         return self.versions[VerNum(vernum)]
     @classmethod
     def clear(cls):
         super(Collection, cls).clear()
     def _version_path(self, ver):
         """Returns path of file in versions repository"""
         return os.path.join(self.path, str(ver))
 class Version(object):
     """A single version in a collection
     :param vernum: Version Number
+    :param vernum: Version Number
     :param path: Path to script files
     :param filelist: List of scripts
     :type vernum: int, VerNum
     :type path: string
     :type filelist: list
     """
     def __init__(self, vernum, path, filelist):
         self.version = VerNum(vernum)
         # Collect scripts in this folder
         self.sql = dict()
         self.python = None
         for script in filelist:
             self.add_script(os.path.join(path, script))
     def script(self, database=None, operation=None):
         """Returns SQL or Python Script"""
         for db in (database, 'default'):
             # Try to return a .sql script first
             try:
                 return self.sql[db][operation]
             except KeyError:
                 continue  # No .sql script exists
         # TODO: maybe add force Python parameter?
         ret = self.python
         assert ret is not None, \
             "There is no script for %d version" % self.version
         return ret
     def add_script(self, path):
         """Add script to Collection/Version"""
         if path.endswith(Extensions.py):
             self._add_script_py(path)
         elif path.endswith(Extensions.sql):
             self._add_script_sql(path)
-    SQL_FILENAME = re.compile(r'^(\d+)_([^_]+)_([^_]+).sql')
+    SQL_FILENAME = re.compile(r'^.*\.sql')
     def _add_script_sql(self, path):
         basename = os.path.basename(path)
         match = self.SQL_FILENAME.match(basename)
         if match:
             version, dbms, op = match.group(1), match.group(2), match.group(3)
             basename = basename.replace('.sql', '')
             parts = basename.split('_')
             if len(parts) < 3:
                 raise exceptions.ScriptError(
                     "Invalid SQL script name %s " % basename + \
                     "(needs to be ###_description_database_operation.sql)")
             version = parts[0]
             op = parts[-1]
             dbms = parts[-2]
         else:
             raise exceptions.ScriptError(
                 "Invalid SQL script name %s " % basename + \
-                "(needs to be ###_database_operation.sql)")
+                "(needs to be ###_description_database_operation.sql)")
         # File the script into a dictionary
         self.sql.setdefault(dbms, {})[op] = script.SqlScript(path)
     def _add_script_py(self, path):
         if self.python is not None:
             raise exceptions.ScriptError('You can only have one Python script '
                 'per version, but you have: %s and %s' % (self.python, path))
         self.python = script.PythonScript(path)
 class Extensions:
     """A namespace for file extensions"""
     py = 'py'
     sql = 'sql'
 def str_to_filename(s):
     """Replaces spaces, (double and single) quotes
     and double underscores to underscores
     """
     s = s.replace(' ', '_').replace('"', '_').replace("'", '_').replace(".", "_")
     while '__' in s:
         s = s.replace('__', '_')

rhodecode/lib/dbmigrate/versions/003_version_1_2_0.py

➞

Show inline comments

@@ @@ -59,44 +59,58 @@ def upgrade(migrate_engine): @@
     #==========================================================================
     from rhodecode.model.db import User
     #add column
     ldap_dn = Column("ldap_dn", String(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     ldap_dn.create(User().__table__)
     api_key = Column("api_key", String(length=255, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     api_key.create(User().__table__)
     #remove old column
     is_ldap = Column("is_ldap", Boolean(), nullable=False, unique=None, default=False)
     is_ldap.drop(User().__table__)
     #==========================================================================
     # Upgrade of `repositories` table
     #==========================================================================
     from rhodecode.model.db import Repository
     #ADD downloads column#
     enable_downloads = Column("downloads", Boolean(), nullable=True, unique=None, default=True)
     enable_downloads.create(Repository().__table__)
     #ADD column created_on
     created_on = Column('created_on', DateTime(timezone=False), nullable=True,
                         unique=None, default=datetime.datetime.now)
     created_on.create(Repository().__table__)
     #ADD group_id column#
     group_id = Column("group_id", Integer(), ForeignKey('groups.group_id'),
                   nullable=True, unique=False, default=None)
     group_id.create(Repository().__table__)
     #ADD clone_uri column#
     clone_uri = Column("clone_uri", String(length=255, convert_unicode=False,
                                            assert_unicode=None),
                         nullable=True, unique=False, default=None)
     clone_uri.create(Repository().__table__)
     #==========================================================================
     # Upgrade of `user_followings` table
     #==========================================================================
     follows_from = Column('follows_from', DateTime(timezone=False), nullable=True, unique=None, default=datetime.datetime.now)
     follows_from.create(Repository().__table__)
     return
 def downgrade(migrate_engine):
     meta = MetaData()
     meta.bind = migrate_engine

0 comments (0 inline, 0 general)