kallithea Changeset - 7f31de1584c6

Changeset - 7f31de1584c6

Parent rev.

Child rev.

[Not reviewed]

beta

0 12 0

Marcin Kuzminski - 14 years ago 2011-08-14 22:51:21
marcin@python-works.com

update migrations for 1.2

12 files changed with 196 insertions and 88 deletions:

rhodecode/lib/dbmigrate/migrate/__init__.py

rhodecode/lib/dbmigrate/migrate/changeset/__init__.py

rhodecode/lib/dbmigrate/migrate/changeset/schema.py

rhodecode/lib/dbmigrate/migrate/exceptions.py

rhodecode/lib/dbmigrate/migrate/versioning/api.py

rhodecode/lib/dbmigrate/migrate/versioning/genmodel.py

rhodecode/lib/dbmigrate/migrate/versioning/repository.py

rhodecode/lib/dbmigrate/migrate/versioning/schema.py

rhodecode/lib/dbmigrate/migrate/versioning/script/py.py

rhodecode/lib/dbmigrate/migrate/versioning/templates/repository/default/migrate.cfg

rhodecode/lib/dbmigrate/migrate/versioning/version.py

rhodecode/lib/dbmigrate/versions/003_version_1_2_0.py

0 comments (0 inline, 0 general)

rhodecode/lib/dbmigrate/migrate/__init__.py

➞

Show inline comments

@@ @@ -7,3 +7,5 @@ @@
 from rhodecode.lib.dbmigrate.migrate.versioning import *
 from rhodecode.lib.dbmigrate.migrate.changeset import *
 __version__ = '0.7.2.dev'
@@ \ No newline at end of file @@

rhodecode/lib/dbmigrate/migrate/changeset/__init__.py

➞

Show inline comments

@@ @@ -15,6 +15,7 @@ warnings.simplefilter('always', Deprecat @@
 _sa_version = tuple(int(re.match("\d+", x).group(0))
                     for x in _sa_version.split("."))
 SQLA_06 = _sa_version >= (0, 6)
 SQLA_07 = _sa_version >= (0, 7)
 del re
 del _sa_version

rhodecode/lib/dbmigrate/migrate/changeset/schema.py

➞

Show inline comments

@@ @@ -11,7 +11,7 @@ from sqlalchemy.schema import ForeignKey @@
 from sqlalchemy.schema import UniqueConstraint
 from rhodecode.lib.dbmigrate.migrate.exceptions import *
 from rhodecode.lib.dbmigrate.migrate.changeset import SQLA_06
+from rhodecode.lib.dbmigrate.migrate.changeset import SQLA_06, SQLA_07
 from rhodecode.lib.dbmigrate.migrate.changeset.databases.visitor import (get_engine_visitor,
                                                  run_single_visitor)
@@ @@ -555,6 +555,9 @@ populated with defaults @@
     def add_to_table(self, table):
         if table is not None  and self.table is None:
             if SQLA_07:
                 table.append_column(self)
             else:
             self._set_parent(table)
     def _col_name_in_constraint(self,cons,name):
@@ @@ -590,6 +593,9 @@ populated with defaults @@
         table.constraints = table.constraints - to_drop
         if table.c.contains_column(self):
             if SQLA_07:
                 table._columns.remove(self)
             else:
             table.c.remove(self)
     # TODO: this is fixed in 0.6

rhodecode/lib/dbmigrate/migrate/exceptions.py

➞

Show inline comments

@@ @@ -71,6 +71,11 @@ class InvalidScriptError(ScriptError): @@
     """Invalid script error."""
 class InvalidVersionError(Error):
     """Invalid version error."""
 # migrate.changeset
 class NotSupportedError(Error):
     """Not supported error"""

rhodecode/lib/dbmigrate/migrate/versioning/api.py

➞

Show inline comments

@@ @@ -110,19 +110,19 @@ def script(description, repository, **op @@
 @catch_known_errors
 def script_sql(database, repository, **opts):
     """%prog script_sql DATABASE REPOSITORY_PATH
 def script_sql(database, description, repository, **opts):
     """%prog script_sql DATABASE DESCRIPTION REPOSITORY_PATH
     Create empty change SQL scripts for given DATABASE, where DATABASE
-    is either specific ('postgres', 'mysql', 'oracle', 'sqlite', etc.)
+    is either specific ('postgresql', 'mysql', 'oracle', 'sqlite', etc.)
     or generic ('default').
     For instance, manage.py script_sql postgres creates:
     repository/versions/001_postgres_upgrade.sql and
     repository/versions/001_postgres_postgres.sql
     For instance, manage.py script_sql postgresql description creates:
     repository/versions/001_description_postgresql_upgrade.sql and
     repository/versions/001_description_postgresql_postgres.sql
     """
     repo = Repository(repository)
     repo.create_script_sql(database, **opts)
+    repo.create_script_sql(database, description, **opts)
 def version(repository, **opts):

rhodecode/lib/dbmigrate/migrate/versioning/genmodel.py

➞

Show inline comments

@@ @@ -14,6 +14,7 @@ import sqlalchemy @@
 from rhodecode.lib.dbmigrate import migrate
 from rhodecode.lib.dbmigrate.migrate import changeset
 log = logging.getLogger(__name__)
 HEADER = """
 ## File autogenerated by genmodel.py
@@ @@ -33,6 +34,13 @@ Base = declarative.declarative_base() @@
 class ModelGenerator(object):
     """Various transformations from an A, B diff.
     In the implementation, A tends to be called the model and B
     the database (although this is not true of all diffs).
     The diff is directionless, but transformations apply the diff
     in a particular direction, described in the method name.
     """
     def __init__(self, diff, engine, declarative=False):
         self.diff = diff
@@ @@ -58,7 +66,7 @@ class ModelGenerator(object): @@
                 pass
             else:
                 kwarg.append('default')
-        ks = ', '.join('%s=%r' % (k, getattr(col, k)) for k in kwarg)
+        args = ['%s=%r' % (k, getattr(col, k)) for k in kwarg]
         # crs: not sure if this is good idea, but it gets rid of extra
         # u''
@@ @@ -72,43 +80,38 @@ class ModelGenerator(object): @@
                     type_ = cls()
                 break
         type_repr = repr(type_)
         if type_repr.endswith('()'):
             type_repr = type_repr[:-2]
         constraints = [repr(cn) for cn in col.constraints]
         data = {
             'name': name,
             'type': type_,
             'constraints': ', '.join([repr(cn) for cn in col.constraints]),
             'args': ks and ks or ''}
             'commonStuff': ', '.join([type_repr] + constraints + args),
+        }
         if data['constraints']:
             if data['args']:
                 data['args'] = ',' + data['args']
         if data['constraints'] or data['args']:
             data['maybeComma'] = ','
         if self.declarative:
             return """%(name)s = Column(%(commonStuff)s)""" % data
         else:
-            data['maybeComma'] = ''
+            return """Column(%(name)r, %(commonStuff)s)""" % data
         commonStuff = """ %(maybeComma)s %(constraints)s %(args)s)""" % data
         commonStuff = commonStuff.strip()
         data['commonStuff'] = commonStuff
         if self.declarative:
             return """%(name)s = Column(%(type)r%(commonStuff)s""" % data
         else:
             return """Column(%(name)r, %(type)r%(commonStuff)s""" % data
     def getTableDefn(self, table):
     def _getTableDefn(self, table, metaName='meta'):
         out = []
         tableName = table.name
         if self.declarative:
             out.append("class %(table)s(Base):" % {'table': tableName})
             out.append("  __tablename__ = '%(table)s'" % {'table': tableName})
             out.append("    __tablename__ = '%(table)s'\n" %
                             {'table': tableName})
             for col in table.columns:
                 out.append("  %s" % self.column_repr(col))
             out.append('\n')
         else:
             out.append("%(table)s = Table('%(table)s', meta," % \
                            {'table': tableName})
             out.append("%(table)s = Table('%(table)s', %(meta)s," %
                        {'table': tableName, 'meta': metaName})
             for col in table.columns:
                 out.append("  %s," % self.column_repr(col))
             out.append(")")
+            out.append(")\n")
         return out
     def _get_tables(self,missingA=False,missingB=False,modified=False):
@@ @@ -122,8 +125,14 @@ class ModelGenerator(object): @@
                 for name in names:
                     yield metadata.tables.get(name)
     def toPython(self):
         """Assume database is current and model is empty."""
     def genBDefinition(self):
         """Generates the source code for a definition of B.
         Assumes a diff where A is empty.
         Was: toPython. Assume database (B) is current and model (A) is empty.
         """
         out = []
         if self.declarative:
             out.append(DECLARATIVE_HEADER)
@@ @@ -131,67 +140,89 @@ class ModelGenerator(object): @@
             out.append(HEADER)
         out.append("")
         for table in self._get_tables(missingA=True):
             out.extend(self.getTableDefn(table))
             out.append("")
             out.extend(self._getTableDefn(table))
         return '\n'.join(out)
     def toUpgradeDowngradePython(self, indent='    '):
         ''' Assume model is most current and database is out-of-date. '''
         decls = ['from rhodecode.lib.dbmigrate.migrate.changeset import schema',
                  'meta = MetaData()']
         for table in self._get_tables(
             missingA=True,missingB=True,modified=True
             ):
             decls.extend(self.getTableDefn(table))
     def genB2AMigration(self, indent='    '):
         '''Generate a migration from B to A.
         Was: toUpgradeDowngradePython
         Assume model (A) is most current and database (B) is out-of-date.
         '''
         decls = ['from migrate.changeset import schema',
                  'pre_meta = MetaData()',
                  'post_meta = MetaData()',
+                ]
         upgradeCommands = ['pre_meta.bind = migrate_engine',
                            'post_meta.bind = migrate_engine']
         downgradeCommands = list(upgradeCommands)
         for tn in self.diff.tables_missing_from_A:
             pre_table = self.diff.metadataB.tables[tn]
             decls.extend(self._getTableDefn(pre_table, metaName='pre_meta'))
             upgradeCommands.append(
                 "pre_meta.tables[%(table)r].drop()" % {'table': tn})
             downgradeCommands.append(
                 "pre_meta.tables[%(table)r].create()" % {'table': tn})
         upgradeCommands, downgradeCommands = [], []
         for tableName in self.diff.tables_missing_from_A:
             upgradeCommands.append("%(table)s.drop()" % {'table': tableName})
             downgradeCommands.append("%(table)s.create()" % \
                                          {'table': tableName})
         for tableName in self.diff.tables_missing_from_B:
             upgradeCommands.append("%(table)s.create()" % {'table': tableName})
             downgradeCommands.append("%(table)s.drop()" % {'table': tableName})
         for tn in self.diff.tables_missing_from_B:
             post_table = self.diff.metadataA.tables[tn]
             decls.extend(self._getTableDefn(post_table, metaName='post_meta'))
             upgradeCommands.append(
                 "post_meta.tables[%(table)r].create()" % {'table': tn})
             downgradeCommands.append(
                 "post_meta.tables[%(table)r].drop()" % {'table': tn})
         for tableName in self.diff.tables_different:
             dbTable = self.diff.metadataB.tables[tableName]
             missingInDatabase, missingInModel, diffDecl = \
                 self.diff.colDiffs[tableName]
             for col in missingInDatabase:
                 upgradeCommands.append('%s.columns[%r].create()' % (
                         modelTable, col.name))
                 downgradeCommands.append('%s.columns[%r].drop()' % (
                         modelTable, col.name))
             for col in missingInModel:
                 upgradeCommands.append('%s.columns[%r].drop()' % (
                         modelTable, col.name))
                 downgradeCommands.append('%s.columns[%r].create()' % (
                         modelTable, col.name))
             for modelCol, databaseCol, modelDecl, databaseDecl in diffDecl:
         for (tn, td) in self.diff.tables_different.iteritems():
             if td.columns_missing_from_A or td.columns_different:
                 pre_table = self.diff.metadataB.tables[tn]
                 decls.extend(self._getTableDefn(
                     pre_table, metaName='pre_meta'))
             if td.columns_missing_from_B or td.columns_different:
                 post_table = self.diff.metadataA.tables[tn]
                 decls.extend(self._getTableDefn(
                     post_table, metaName='post_meta'))
             for col in td.columns_missing_from_A:
                 upgradeCommands.append(
                     'pre_meta.tables[%r].columns[%r].drop()' % (tn, col))
                 downgradeCommands.append(
                     'pre_meta.tables[%r].columns[%r].create()' % (tn, col))
             for col in td.columns_missing_from_B:
                 upgradeCommands.append(
                     'post_meta.tables[%r].columns[%r].create()' % (tn, col))
                 downgradeCommands.append(
                     'post_meta.tables[%r].columns[%r].drop()' % (tn, col))
             for modelCol, databaseCol, modelDecl, databaseDecl in td.columns_different:
                 upgradeCommands.append(
                     'assert False, "Can\'t alter columns: %s:%s=>%s"' % (
-                    modelTable, modelCol.name, databaseCol.name))
+                    tn, modelCol.name, databaseCol.name))
                 downgradeCommands.append(
                     'assert False, "Can\'t alter columns: %s:%s=>%s"' % (
                     modelTable, modelCol.name, databaseCol.name))
         pre_command = '    meta.bind = migrate_engine'
                     tn, modelCol.name, databaseCol.name))
         return (
             '\n'.join(decls),
             '\n'.join([pre_command] + ['%s%s' % (indent, line) for line in upgradeCommands]),
             '\n'.join([pre_command] + ['%s%s' % (indent, line) for line in downgradeCommands]))
             '\n'.join('%s%s' % (indent, line) for line in upgradeCommands),
             '\n'.join('%s%s' % (indent, line) for line in downgradeCommands))
     def _db_can_handle_this_change(self,td):
         """Check if the database can handle going from B to A."""
         if (td.columns_missing_from_B
             and not td.columns_missing_from_A
             and not td.columns_different):
-            # Even sqlite can handle this.
+            # Even sqlite can handle column additions.
             return True
         else:
             return not self.engine.url.drivername.startswith('sqlite')
     def applyModel(self):
         """Apply model to current database."""
     def runB2A(self):
         """Goes from B to A.
         Was: applyModel. Apply model (A) to current database (B).
         """
         meta = sqlalchemy.MetaData(self.engine)
@@ @@ -251,3 +282,4 @@ class ModelGenerator(object): @@
                 except:
                     trans.rollback()
                     raise

rhodecode/lib/dbmigrate/migrate/versioning/repository.py

➞

Show inline comments

@@ @@ -115,6 +115,7 @@ class Repository(pathed.Pathed): @@
         options.setdefault('version_table', 'migrate_version')
         options.setdefault('repository_id', name)
         options.setdefault('required_dbs', [])
         options.setdefault('use_timestamp_numbering', '0')
         tmpl = open(os.path.join(tmpl_dir, cls._config)).read()
         ret = TempitaTemplate(tmpl).substitute(options)
@@ @@ -152,11 +153,14 @@ class Repository(pathed.Pathed): @@
     def create_script(self, description, **k):
         """API to :meth:`migrate.versioning.version.Collection.create_new_python_version`"""
         k['use_timestamp_numbering'] = self.use_timestamp_numbering
         self.versions.create_new_python_version(description, **k)
     def create_script_sql(self, database, **k):
+    def create_script_sql(self, database, description, **k):
         """API to :meth:`migrate.versioning.version.Collection.create_new_sql_version`"""
         self.versions.create_new_sql_version(database, **k)
         k['use_timestamp_numbering'] = self.use_timestamp_numbering
         self.versions.create_new_sql_version(database, description, **k)
     @property
     def latest(self):
@@ @@ -173,6 +177,13 @@ class Repository(pathed.Pathed): @@
         """Returns repository id specified in config"""
         return self.config.get('db_settings', 'repository_id')
     @property
     def use_timestamp_numbering(self):
         """Returns use_timestamp_numbering specified in config"""
         ts_numbering = self.config.get('db_settings', 'use_timestamp_numbering', raw=True)
         return ts_numbering
     def version(self, *p, **k):
         """API to :attr:`migrate.versioning.version.Collection.version`"""
         return self.versions.version(*p, **k)

rhodecode/lib/dbmigrate/migrate/versioning/schema.py

➞

Show inline comments

@@ @@ -11,6 +11,7 @@ from sqlalchemy import exceptions as sa_ @@
 from sqlalchemy.sql import bindparam
 from rhodecode.lib.dbmigrate.migrate import exceptions
 from rhodecode.lib.dbmigrate.migrate.changeset import SQLA_07
 from rhodecode.lib.dbmigrate.migrate.versioning import genmodel, schemadiff
 from rhodecode.lib.dbmigrate.migrate.versioning.repository import Repository
 from rhodecode.lib.dbmigrate.migrate.versioning.util import load_model
@@ @@ -57,6 +58,12 @@ class ControlledSchema(object): @@
         """
         Remove version control from a database.
         """
         if SQLA_07:
             try:
                 self.table.drop()
             except sa_exceptions.DatabaseError:
                 raise exceptions.DatabaseNotControlledError(str(self.table))
         else:
         try:
             self.table.drop()
         except (sa_exceptions.SQLError):
@@ @@ -110,7 +117,7 @@ class ControlledSchema(object): @@
         diff = schemadiff.getDiffOfModelAgainstDatabase(
             model, self.engine, excludeTables=[self.repository.version_table]
+            )
-        genmodel.ModelGenerator(diff,self.engine).applyModel()
+        genmodel.ModelGenerator(diff,self.engine).runB2A()
         self.update_repository_table(self.version, int(self.repository.latest))
@@ @@ -210,4 +217,4 @@ class ControlledSchema(object): @@
         diff = schemadiff.getDiffOfModelAgainstDatabase(
             MetaData(), engine, excludeTables=[repository.version_table]
+            )
-        return genmodel.ModelGenerator(diff, engine, declarative).toPython()
+        return genmodel.ModelGenerator(diff, engine, declarative).genBDefinition()

rhodecode/lib/dbmigrate/migrate/versioning/script/py.py

➞

Show inline comments

@@ @@ -61,12 +61,12 @@ class PythonScript(base.BaseScript): @@
         # Compute differences.
         diff = schemadiff.getDiffOfModelAgainstModel(
             model,
             oldmodel,
             model,
             excludeTables=[repository.version_table])
         # TODO: diff can be False (there is no difference?)
         decls, upgradeCommands, downgradeCommands = \
-            genmodel.ModelGenerator(diff,engine).toUpgradeDowngradePython()
+            genmodel.ModelGenerator(diff,engine).genB2AMigration()
         # Store differences into file.
         src = Template(opts.pop('templates_path', None)).get_script(opts.pop('templates_theme', None))

rhodecode/lib/dbmigrate/migrate/versioning/templates/repository/default/migrate.cfg

➞

Show inline comments

@@ @@ -18,3 +18,8 @@ version_table={{ locals().pop('version_t @@
 # be using to ensure your updates to that database work properly.
 # This must be a list; example: ['postgres','sqlite']
 required_dbs={{ locals().pop('required_dbs') }}
 # When creating new change scripts, Migrate will stamp the new script with
 # a version number. By default this is latest_version + 1. You can set this
 # to 'true' to tell Migrate to use the UTC timestamp instead.
 use_timestamp_numbering='false'
@@ \ No newline at end of file @@

rhodecode/lib/dbmigrate/migrate/versioning/version.py

➞

Show inline comments

@@ @@ -8,6 +8,7 @@ import logging @@
 from rhodecode.lib.dbmigrate.migrate import exceptions
 from rhodecode.lib.dbmigrate.migrate.versioning import pathed, script
 from datetime import datetime
 log = logging.getLogger(__name__)
@@ @@ -88,9 +89,17 @@ class Collection(pathed.Pathed): @@
         """:returns: Latest version in Collection"""
         return max([VerNum(0)] + self.versions.keys())
     def _next_ver_num(self, use_timestamp_numbering):
         print use_timestamp_numbering
         if use_timestamp_numbering == True:
             print "Creating new timestamp version!"
             return VerNum(int(datetime.utcnow().strftime('%Y%m%d%H%M%S')))
         else:
             return self.latest + 1
     def create_new_python_version(self, description, **k):
         """Create Python files for new version"""
-        ver = self.latest + 1
+        ver = self._next_ver_num(k.pop('use_timestamp_numbering', False))
         extra = str_to_filename(description)
         if extra:
@@ @@ -105,14 +114,22 @@ class Collection(pathed.Pathed): @@
         script.PythonScript.create(filepath, **k)
         self.versions[ver] = Version(ver, self.path, [filename])
     def create_new_sql_version(self, database, **k):
+    def create_new_sql_version(self, database, description, **k):
         """Create SQL files for new version"""
-        ver = self.latest + 1
+        ver = self._next_ver_num(k.pop('use_timestamp_numbering', False))
         self.versions[ver] = Version(ver, self.path, [])
         extra = str_to_filename(description)
         if extra:
             if extra == '_':
                 extra = ''
             elif not extra.startswith('_'):
                 extra = '_%s' % extra
         # Create new files.
         for op in ('upgrade', 'downgrade'):
-            filename = '%03d_%s_%s.sql' % (ver, database, op)
+            filename = '%03d%s_%s_%s.sql' % (ver, extra, database, op)
             filepath = self._version_path(filename)
             script.SqlScript.create(filepath, **k)
             self.versions[ver].add_script(filepath)
@@ @@ -176,18 +193,26 @@ class Version(object): @@
         elif path.endswith(Extensions.sql):
             self._add_script_sql(path)
-    SQL_FILENAME = re.compile(r'^(\d+)_([^_]+)_([^_]+).sql')
+    SQL_FILENAME = re.compile(r'^.*\.sql')
     def _add_script_sql(self, path):
         basename = os.path.basename(path)
         match = self.SQL_FILENAME.match(basename)
         if match:
             version, dbms, op = match.group(1), match.group(2), match.group(3)
             basename = basename.replace('.sql', '')
             parts = basename.split('_')
             if len(parts) < 3:
                 raise exceptions.ScriptError(
                     "Invalid SQL script name %s " % basename + \
                     "(needs to be ###_description_database_operation.sql)")
             version = parts[0]
             op = parts[-1]
             dbms = parts[-2]
         else:
             raise exceptions.ScriptError(
                 "Invalid SQL script name %s " % basename + \
-                "(needs to be ###_database_operation.sql)")
+                "(needs to be ###_description_database_operation.sql)")
         # File the script into a dictionary
         self.sql.setdefault(dbms, {})[op] = script.SqlScript(path)

rhodecode/lib/dbmigrate/versions/003_version_1_2_0.py

➞

Show inline comments

@@ @@ -80,6 +80,11 @@ def upgrade(migrate_engine): @@
     enable_downloads = Column("downloads", Boolean(), nullable=True, unique=None, default=True)
     enable_downloads.create(Repository().__table__)
     #ADD column created_on
     created_on = Column('created_on', DateTime(timezone=False), nullable=True,
                         unique=None, default=datetime.datetime.now)
     created_on.create(Repository().__table__)
     #ADD group_id column#
     group_id = Column("group_id", Integer(), ForeignKey('groups.group_id'),
                   nullable=True, unique=False, default=None)
@@ @@ -94,6 +99,15 @@ def upgrade(migrate_engine): @@
                         nullable=True, unique=False, default=None)
     clone_uri.create(Repository().__table__)
     #==========================================================================
     # Upgrade of `user_followings` table
     #==========================================================================
     follows_from = Column('follows_from', DateTime(timezone=False), nullable=True, unique=None, default=datetime.datetime.now)
     follows_from.create(Repository().__table__)
     return

0 comments (0 inline, 0 general)