kallithea Changeset - 5b2cf21b1947

Changeset - 5b2cf21b1947

Parent rev.

Child rev.

[Not reviewed]

beta

0 10 2

Marcin Kuzminski - 14 years ago 2011-11-02 02:04:54
marcin@python-works.com

Synced with latest sqlalchemy-migrate, added new upcomming migration for 1.3

12 files changed with 77 insertions and 16 deletions:

rhodecode/lib/dbmigrate/migrate/__init__.py

rhodecode/lib/dbmigrate/migrate/changeset/databases/sqlite.py

rhodecode/lib/dbmigrate/migrate/exceptions.py

rhodecode/lib/dbmigrate/migrate/versioning/api.py

rhodecode/lib/dbmigrate/migrate/versioning/repository.py

rhodecode/lib/dbmigrate/migrate/versioning/schemadiff.py

rhodecode/lib/dbmigrate/migrate/versioning/templates/manage/default.py_tmpl

rhodecode/lib/dbmigrate/migrate/versioning/templates/manage/pylons.py_tmpl

rhodecode/lib/dbmigrate/migrate/versioning/version.py

rhodecode/lib/dbmigrate/schema/db_1_3_0.py

rhodecode/lib/dbmigrate/versions/003_version_1_2_0.py

rhodecode/lib/dbmigrate/versions/004_version_1_3_0.py

0 comments (0 inline, 0 general)

rhodecode/lib/dbmigrate/migrate/__init__.py

➞

Show inline comments

 """
    SQLAlchemy migrate provides two APIs :mod:`migrate.versioning` for
    database schema version and repository management and
    :mod:`migrate.changeset` that allows to define database schema changes
    using Python.
 """
 from rhodecode.lib.dbmigrate.migrate.versioning import *
 from rhodecode.lib.dbmigrate.migrate.changeset import *
 __version__ = '0.7.2.dev'
@@ \ No newline at end of file @@
 __version__ = '0.7.3.dev'
@@ \ No newline at end of file @@

rhodecode/lib/dbmigrate/migrate/changeset/databases/sqlite.py

➞

Show inline comments

 """
    `SQLite`_ database specific implementations of changeset classes.
    .. _`SQLite`: http://www.sqlite.org/
 """
 from UserDict import DictMixin
 from copy import copy
 from sqlalchemy.databases import sqlite as sa_base
 from rhodecode.lib.dbmigrate.migrate import exceptions
 from rhodecode.lib.dbmigrate.migrate.changeset import ansisql, SQLA_06
 if not SQLA_06:
     SQLiteSchemaGenerator = sa_base.SQLiteSchemaGenerator
 else:
     SQLiteSchemaGenerator = sa_base.SQLiteDDLCompiler
 class SQLiteCommon(object):
     def _not_supported(self, op):
         raise exceptions.NotSupportedError("SQLite does not support "
             "%s; see http://www.sqlite.org/lang_altertable.html" % op)
 class SQLiteHelper(SQLiteCommon):
     def recreate_table(self,table,column=None,delta=None):
         table_name = self.preparer.format_table(table)
         # we remove all indexes so as not to have
         # problems during copy and re-create
         for index in table.indexes:
             index.drop()
         self.append('ALTER TABLE %s RENAME TO migration_tmp' % table_name)
         self.execute()
         insertion_string = self._modify_table(table, column, delta)
         table.create()
+        table.create(bind=self.connection)
         self.append(insertion_string % {'table_name': table_name})
         self.execute()
         self.append('DROP TABLE migration_tmp')
         self.execute()
     def visit_column(self, delta):
         if isinstance(delta, DictMixin):
             column = delta.result_column
             table = self._to_table(delta.table)
         else:
             column = delta
             table = self._to_table(column.table)
         self.recreate_table(table,column,delta)
 class SQLiteColumnGenerator(SQLiteSchemaGenerator,
                             ansisql.ANSIColumnGenerator,
                             # at the end so we get the normal
                             # visit_column by default
                             SQLiteHelper,
                             SQLiteCommon
                             ):
     """SQLite ColumnGenerator"""
     def _modify_table(self, table, column, delta):
         columns = ' ,'.join(map(
                 self.preparer.format_column,
                 [c for c in table.columns if c.name!=column.name]))
         return ('INSERT INTO %%(table_name)s (%(cols)s) '
                 'SELECT %(cols)s from migration_tmp')%{'cols':columns}
     def visit_column(self,column):
         if column.foreign_keys:
             SQLiteHelper.visit_column(self,column)
         else:
             super(SQLiteColumnGenerator,self).visit_column(column)
 class SQLiteColumnDropper(SQLiteHelper, ansisql.ANSIColumnDropper):
     """SQLite ColumnDropper"""
     def _modify_table(self, table, column, delta):
         columns = ' ,'.join(map(self.preparer.format_column, table.columns))
         return 'INSERT INTO %(table_name)s SELECT ' + columns + \
             ' from migration_tmp'
     def visit_column(self,column):
         # For SQLite, we *have* to remove the column here so the table
         # is re-created properly.
         column.remove_from_table(column.table,unset_table=False)
         super(SQLiteColumnDropper,self).visit_column(column)
 class SQLiteSchemaChanger(SQLiteHelper, ansisql.ANSISchemaChanger):
     """SQLite SchemaChanger"""
     def _modify_table(self, table, column, delta):
         return 'INSERT INTO %(table_name)s SELECT * from migration_tmp'
     def visit_index(self, index):
         """Does not support ALTER INDEX"""
         self._not_supported('ALTER INDEX')
 class SQLiteConstraintGenerator(ansisql.ANSIConstraintGenerator, SQLiteHelper, SQLiteCommon):
     def visit_migrate_primary_key_constraint(self, constraint):
         tmpl = "CREATE UNIQUE INDEX %s ON %s ( %s )"
         cols = ', '.join(map(self.preparer.format_column, constraint.columns))
         tname = self.preparer.format_table(constraint.table)
         name = self.get_constraint_name(constraint)
         msg = tmpl % (name, tname, cols)
         self.append(msg)
         self.execute()
     def _modify_table(self, table, column, delta):
         return 'INSERT INTO %(table_name)s SELECT * from migration_tmp'
     def visit_migrate_foreign_key_constraint(self, *p, **k):
         self.recreate_table(p[0].table)
     def visit_migrate_unique_constraint(self, *p, **k):
         self.recreate_table(p[0].table)
 class SQLiteConstraintDropper(ansisql.ANSIColumnDropper,
                               SQLiteCommon,
                               ansisql.ANSIConstraintCommon):
     def visit_migrate_primary_key_constraint(self, constraint):
         tmpl = "DROP INDEX %s "
         name = self.get_constraint_name(constraint)
         msg = tmpl % (name)
         self.append(msg)
         self.execute()
     def visit_migrate_foreign_key_constraint(self, *p, **k):
         self._not_supported('ALTER TABLE DROP CONSTRAINT')
     def visit_migrate_check_constraint(self, *p, **k):
         self._not_supported('ALTER TABLE DROP CONSTRAINT')
     def visit_migrate_unique_constraint(self, *p, **k):
         self._not_supported('ALTER TABLE DROP CONSTRAINT')
 # TODO: technically primary key is a NOT NULL + UNIQUE constraint, should add NOT NULL to index
 class SQLiteDialect(ansisql.ANSIDialect):
     columngenerator = SQLiteColumnGenerator
     columndropper = SQLiteColumnDropper
     schemachanger = SQLiteSchemaChanger
     constraintgenerator = SQLiteConstraintGenerator
     constraintdropper = SQLiteConstraintDropper

rhodecode/lib/dbmigrate/migrate/exceptions.py

➞

Show inline comments

 """
    Provide exception classes for :mod:`migrate`
 """
 class Error(Exception):
     """Error base class."""
 class ApiError(Error):
     """Base class for API errors."""
 class KnownError(ApiError):
     """A known error condition."""
 class UsageError(ApiError):
     """A known error condition where help should be displayed."""
 class ControlledSchemaError(Error):
     """Base class for controlled schema errors."""
 class InvalidVersionError(ControlledSchemaError):
     """Invalid version number."""
 class DatabaseNotControlledError(ControlledSchemaError):
     """Database should be under version control, but it's not."""
 class DatabaseAlreadyControlledError(ControlledSchemaError):
     """Database shouldn't be under version control, but it is"""
 class WrongRepositoryError(ControlledSchemaError):
     """This database is under version control by another repository."""
 class NoSuchTableError(ControlledSchemaError):
     """The table does not exist."""
 class PathError(Error):
     """Base class for path errors."""
 class PathNotFoundError(PathError):
     """A path with no file was required; found a file."""
 class PathFoundError(PathError):
     """A path with a file was required; found no file."""
 class RepositoryError(Error):
     """Base class for repository errors."""
 class InvalidRepositoryError(RepositoryError):
     """Invalid repository error."""
 class ScriptError(Error):
     """Base class for script errors."""
 class InvalidScriptError(ScriptError):
     """Invalid script error."""
 class InvalidVersionError(Error):
     """Invalid version error."""
 # migrate.changeset
 class NotSupportedError(Error):
     """Not supported error"""
 class InvalidConstraintError(Error):
     """Invalid constraint error"""
 class MigrateDeprecationWarning(DeprecationWarning):
     """Warning for deprecated features in Migrate"""

rhodecode/lib/dbmigrate/migrate/versioning/api.py

➞

Show inline comments

 """
    This module provides an external API to the versioning system.
    .. versionchanged:: 0.6.0
     :func:`migrate.versioning.api.test` and schema diff functions
     changed order of positional arguments so all accept `url` and `repository`
     as first arguments.
    .. versionchanged:: 0.5.4
     ``--preview_sql`` displays source file when using SQL scripts.
     If Python script is used, it runs the action with mocked engine and
     returns captured SQL statements.
    .. versionchanged:: 0.5.4
     Deprecated ``--echo`` parameter in favour of new
     :func:`migrate.versioning.util.construct_engine` behavior.
 """
 # Dear migrate developers,
+#
 # please do not comment this module using sphinx syntax because its
 # docstrings are presented as user help and most users cannot
 # interpret sphinx annotated ReStructuredText.
+#
 # Thanks,
 # Jan Dittberner
 import sys
 import inspect
 import logging
 from rhodecode.lib.dbmigrate.migrate import exceptions
 from rhodecode.lib.dbmigrate.migrate.versioning import repository, schema, version, \
     script as script_ # command name conflict
 from rhodecode.lib.dbmigrate.migrate.versioning.util import catch_known_errors, with_engine
 log = logging.getLogger(__name__)
 command_desc = {
     'help': 'displays help on a given command',
     'create': 'create an empty repository at the specified path',
     'script': 'create an empty change Python script',
     'script_sql': 'create empty change SQL scripts for given database',
     'version': 'display the latest version available in a repository',
     'db_version': 'show the current version of the repository under version control',
     'source': 'display the Python code for a particular version in this repository',
     'version_control': 'mark a database as under this repository\'s version control',
     'upgrade': 'upgrade a database to a later version',
     'downgrade': 'downgrade a database to an earlier version',
     'drop_version_control': 'removes version control from a database',
     'manage': 'creates a Python script that runs Migrate with a set of default values',
     'test': 'performs the upgrade and downgrade command on the given database',
     'compare_model_to_db': 'compare MetaData against the current database state',
     'create_model': 'dump the current database as a Python model to stdout',
     'make_update_script_for_model': 'create a script changing the old MetaData to the new (current) MetaData',
     'update_db_from_model': 'modify the database to match the structure of the current MetaData',
+}
 __all__ = command_desc.keys()
 Repository = repository.Repository
 ControlledSchema = schema.ControlledSchema
 VerNum = version.VerNum
 PythonScript = script_.PythonScript
 SqlScript = script_.SqlScript
 # deprecated
 def help(cmd=None, **opts):
     """%prog help COMMAND
     Displays help on a given command.
     """
     if cmd is None:
         raise exceptions.UsageError(None)
     try:
         func = globals()[cmd]
     except:
         raise exceptions.UsageError(
             "'%s' isn't a valid command. Try 'help COMMAND'" % cmd)
     ret = func.__doc__
     if sys.argv[0]:
         ret = ret.replace('%prog', sys.argv[0])
     return ret
 @catch_known_errors
 def create(repository, name, **opts):
     """%prog create REPOSITORY_PATH NAME [--table=TABLE]
     Create an empty repository at the specified path.
     You can specify the version_table to be used; by default, it is
     'migrate_version'.  This table is created in all version-controlled
     databases.
     """
     repo_path = Repository.create(repository, name, **opts)
 @catch_known_errors
 def script(description, repository, **opts):
     """%prog script DESCRIPTION REPOSITORY_PATH
     Create an empty change script using the next unused version number
     appended with the given description.
     For instance, manage.py script "Add initial tables" creates:
     repository/versions/001_Add_initial_tables.py
     """
     repo = Repository(repository)
     repo.create_script(description, **opts)
 @catch_known_errors
 def script_sql(database, description, repository, **opts):
     """%prog script_sql DATABASE DESCRIPTION REPOSITORY_PATH
     Create empty change SQL scripts for given DATABASE, where DATABASE
     is either specific ('postgresql', 'mysql', 'oracle', 'sqlite', etc.)
     or generic ('default').
     For instance, manage.py script_sql postgresql description creates:
     repository/versions/001_description_postgresql_upgrade.sql and
-    repository/versions/001_description_postgresql_postgres.sql
+    repository/versions/001_description_postgresql_downgrade.sql
     """
     repo = Repository(repository)
     repo.create_script_sql(database, description, **opts)
 def version(repository, **opts):
     """%prog version REPOSITORY_PATH
     Display the latest version available in a repository.
     """
     repo = Repository(repository)
     return repo.latest
 @with_engine
 def db_version(url, repository, **opts):
     """%prog db_version URL REPOSITORY_PATH
     Show the current version of the repository with the given
     connection string, under version control of the specified
     repository.
     The url should be any valid SQLAlchemy connection string.
     """
     engine = opts.pop('engine')
     schema = ControlledSchema(engine, repository)
     return schema.version
 def source(version, dest=None, repository=None, **opts):
     """%prog source VERSION [DESTINATION] --repository=REPOSITORY_PATH
     Display the Python code for a particular version in this
     repository.  Save it to the file at DESTINATION or, if omitted,
     send to stdout.
     """
     if repository is None:
         raise exceptions.UsageError("A repository must be specified")
     repo = Repository(repository)
     ret = repo.version(version).script().source()
     if dest is not None:
         dest = open(dest, 'w')
         dest.write(ret)
         dest.close()
         ret = None
     return ret
 def upgrade(url, repository, version=None, **opts):
     """%prog upgrade URL REPOSITORY_PATH [VERSION] [--preview_py|--preview_sql]
     Upgrade a database to a later version.
     This runs the upgrade() function defined in your change scripts.
     By default, the database is updated to the latest available
     version. You may specify a version instead, if you wish.
     You may preview the Python or SQL code to be executed, rather than
     actually executing it, using the appropriate 'preview' option.
     """
     err = "Cannot upgrade a database of version %s to version %s. "\
         "Try 'downgrade' instead."
     return _migrate(url, repository, version, upgrade=True, err=err, **opts)
 def downgrade(url, repository, version, **opts):
     """%prog downgrade URL REPOSITORY_PATH VERSION [--preview_py|--preview_sql]
     Downgrade a database to an earlier version.
     This is the reverse of upgrade; this runs the downgrade() function
     defined in your change scripts.
     You may preview the Python or SQL code to be executed, rather than
     actually executing it, using the appropriate 'preview' option.
     """
     err = "Cannot downgrade a database of version %s to version %s. "\
         "Try 'upgrade' instead."
     return _migrate(url, repository, version, upgrade=False, err=err, **opts)
 @with_engine
 def test(url, repository, **opts):
     """%prog test URL REPOSITORY_PATH [VERSION]
     Performs the upgrade and downgrade option on the given
     database. This is not a real test and may leave the database in a
     bad state. You should therefore better run the test on a copy of
     your database.
     """
     engine = opts.pop('engine')
     repos = Repository(repository)
     script = repos.version(None).script()
     # Upgrade
     log.info("Upgrading...")
     script = repos.version(None).script(engine.name, 'upgrade')
     script.run(engine, 1)
     log.info("done")
     log.info("Downgrading...")
     script = repos.version(None).script(engine.name, 'downgrade')
     script.run(engine, -1)
     log.info("done")
     log.info("Success")
 @with_engine
 def version_control(url, repository, version=None, **opts):
     """%prog version_control URL REPOSITORY_PATH [VERSION]
     Mark a database as under this repository's version control.
     Once a database is under version control, schema changes should
     only be done via change scripts in this repository.
     This creates the table version_table in the database.
     The url should be any valid SQLAlchemy connection string.
     By default, the database begins at version 0 and is assumed to be
     empty.  If the database is not empty, you may specify a version at
     which to begin instead. No attempt is made to verify this
     version's correctness - the database schema is expected to be
     identical to what it would be if the database were created from
     scratch.
     """
     engine = opts.pop('engine')
     ControlledSchema.create(engine, repository, version)
 @with_engine
 def drop_version_control(url, repository, **opts):
     """%prog drop_version_control URL REPOSITORY_PATH
     Removes version control from a database.
     """
     engine = opts.pop('engine')
     schema = ControlledSchema(engine, repository)
     schema.drop()
 def manage(file, **opts):
     """%prog manage FILENAME [VARIABLES...]
     Creates a script that runs Migrate with a set of default values.
     For example::
         %prog manage manage.py --repository=/path/to/repository \
 --url=sqlite:///project.db
     would create the script manage.py. The following two commands
     would then have exactly the same results::
         python manage.py version
         %prog version --repository=/path/to/repository
     """
     Repository.create_manage_file(file, **opts)
 @with_engine
 def compare_model_to_db(url, repository, model, **opts):
     """%prog compare_model_to_db URL REPOSITORY_PATH MODEL
     Compare the current model (assumed to be a module level variable
     of type sqlalchemy.MetaData) against the current database.
     NOTE: This is EXPERIMENTAL.
     """  # TODO: get rid of EXPERIMENTAL label
     engine = opts.pop('engine')
     return ControlledSchema.compare_model_to_db(engine, model, repository)
 @with_engine
 def create_model(url, repository, **opts):
     """%prog create_model URL REPOSITORY_PATH [DECLERATIVE=True]
     Dump the current database as a Python model to stdout.
     NOTE: This is EXPERIMENTAL.
     """  # TODO: get rid of EXPERIMENTAL label
     engine = opts.pop('engine')
     declarative = opts.get('declarative', False)
     return ControlledSchema.create_model(engine, repository, declarative)
 @catch_known_errors
 @with_engine
 def make_update_script_for_model(url, repository, oldmodel, model, **opts):
     """%prog make_update_script_for_model URL OLDMODEL MODEL REPOSITORY_PATH
     Create a script changing the old Python model to the new (current)
     Python model, sending to stdout.
     NOTE: This is EXPERIMENTAL.
     """  # TODO: get rid of EXPERIMENTAL label
     engine = opts.pop('engine')
     return PythonScript.make_update_script_for_model(
         engine, oldmodel, model, repository, **opts)
 @with_engine
 def update_db_from_model(url, repository, model, **opts):
     """%prog update_db_from_model URL REPOSITORY_PATH MODEL
     Modify the database to match the structure of the current Python
     model. This also sets the db_version number to the latest in the
     repository.
     NOTE: This is EXPERIMENTAL.
     """  # TODO: get rid of EXPERIMENTAL label
     engine = opts.pop('engine')
     schema = ControlledSchema(engine, repository)
     schema.update_db_from_model(model)
 @with_engine
 def _migrate(url, repository, version, upgrade, err, **opts):
     engine = opts.pop('engine')
     url = str(engine.url)
     schema = ControlledSchema(engine, repository)
     version = _migrate_version(schema, version, upgrade, err)
     changeset = schema.changeset(version)
     for ver, change in changeset:
         nextver = ver + changeset.step
         log.info('%s -> %s... ', ver, nextver)
         if opts.get('preview_sql'):
             if isinstance(change, PythonScript):
                 log.info(change.preview_sql(url, changeset.step, **opts))
             elif isinstance(change, SqlScript):
                 log.info(change.source())
         elif opts.get('preview_py'):
             if not isinstance(change, PythonScript):
                 raise exceptions.UsageError("Python source can be only displayed"
                     " for python migration files")
             source_ver = max(ver, nextver)
             module = schema.repository.version(source_ver).script().module
             funcname = upgrade and "upgrade" or "downgrade"
             func = getattr(module, funcname)
             log.info(inspect.getsource(func))
         else:
             schema.runchange(ver, change, changeset.step)
             log.info('done')
 def _migrate_version(schema, version, upgrade, err):
     if version is None:
         return version
     # Version is specified: ensure we're upgrading in the right direction
     # (current version < target version for upgrading; reverse for down)
     version = VerNum(version)
     cur = schema.version
     if upgrade is not None:
         if upgrade:
             direction = cur <= version
         else:
             direction = cur >= version
         if not direction:
             raise exceptions.KnownError(err % (cur, version))
     return version

rhodecode/lib/dbmigrate/migrate/versioning/repository.py

➞

Show inline comments

 """
    SQLAlchemy migrate repository management.
 """
 import os
 import shutil
 import string
 import logging
 from pkg_resources import resource_filename
 from tempita import Template as TempitaTemplate
 from rhodecode.lib.dbmigrate.migrate import exceptions
 from rhodecode.lib.dbmigrate.migrate.versioning import version, pathed, cfgparse
 from rhodecode.lib.dbmigrate.migrate.versioning.template import Template
 from rhodecode.lib.dbmigrate.migrate.versioning.config import *
 log = logging.getLogger(__name__)
 class Changeset(dict):
     """A collection of changes to be applied to a database.
     Changesets are bound to a repository and manage a set of
     scripts from that repository.
     Behaves like a dict, for the most part. Keys are ordered based on step value.
     """
     def __init__(self, start, *changes, **k):
         """
         Give a start version; step must be explicitly stated.
         """
         self.step = k.pop('step', 1)
         self.start = version.VerNum(start)
         self.end = self.start
         for change in changes:
             self.add(change)
     def __iter__(self):
         return iter(self.items())
     def keys(self):
         """
         In a series of upgrades x -> y, keys are version x. Sorted.
         """
         ret = super(Changeset, self).keys()
         # Reverse order if downgrading
         ret.sort(reverse=(self.step < 1))
         return ret
     def values(self):
         return [self[k] for k in self.keys()]
     def items(self):
         return zip(self.keys(), self.values())
     def add(self, change):
         """Add new change to changeset"""
         key = self.end
         self.end += self.step
         self[key] = change
     def run(self, *p, **k):
         """Run the changeset scripts"""
         for version, script in self:
             script.run(*p, **k)
 class Repository(pathed.Pathed):
     """A project's change script repository"""
     _config = 'migrate.cfg'
     _versions = 'versions'
     def __init__(self, path):
         log.debug('Loading repository %s...' % path)
         self.verify(path)
         super(Repository, self).__init__(path)
         self.config = cfgparse.Config(os.path.join(self.path, self._config))
         self.versions = version.Collection(os.path.join(self.path,
                                                       self._versions))
         log.debug('Repository %s loaded successfully' % path)
         log.debug('Config: %r' % self.config.to_dict())
     @classmethod
     def verify(cls, path):
         """
         Ensure the target path is a valid repository.
         :raises: :exc:`InvalidRepositoryError <migrate.exceptions.InvalidRepositoryError>`
         """
         # Ensure the existence of required files
         try:
             cls.require_found(path)
             cls.require_found(os.path.join(path, cls._config))
             cls.require_found(os.path.join(path, cls._versions))
         except exceptions.PathNotFoundError, e:
             raise exceptions.InvalidRepositoryError(path)
     @classmethod
     def prepare_config(cls, tmpl_dir, name, options=None):
         """
         Prepare a project configuration file for a new project.
         :param tmpl_dir: Path to Repository template
         :param config_file: Name of the config file in Repository template
         :param name: Repository name
         :type tmpl_dir: string
         :type config_file: string
         :type name: string
         :returns: Populated config file
         """
         if options is None:
             options = {}
         options.setdefault('version_table', 'migrate_version')
         options.setdefault('repository_id', name)
         options.setdefault('required_dbs', [])
-        options.setdefault('use_timestamp_numbering', '0')
+        options.setdefault('use_timestamp_numbering', False)
         tmpl = open(os.path.join(tmpl_dir, cls._config)).read()
         ret = TempitaTemplate(tmpl).substitute(options)
         # cleanup
         del options['__template_name__']
         return ret
     @classmethod
     def create(cls, path, name, **opts):
         """Create a repository at a specified path"""
         cls.require_notfound(path)
         theme = opts.pop('templates_theme', None)
         t_path = opts.pop('templates_path', None)
         # Create repository
         tmpl_dir = Template(t_path).get_repository(theme=theme)
         shutil.copytree(tmpl_dir, path)
         # Edit config defaults
         config_text = cls.prepare_config(tmpl_dir, name, options=opts)
         fd = open(os.path.join(path, cls._config), 'w')
         fd.write(config_text)
         fd.close()
         opts['repository_name'] = name
         # Create a management script
         manager = os.path.join(path, 'manage.py')
         Repository.create_manage_file(manager, templates_theme=theme,
             templates_path=t_path, **opts)
         return cls(path)
     def create_script(self, description, **k):
         """API to :meth:`migrate.versioning.version.Collection.create_new_python_version`"""
         k['use_timestamp_numbering'] = self.use_timestamp_numbering
         self.versions.create_new_python_version(description, **k)
     def create_script_sql(self, database, description, **k):
         """API to :meth:`migrate.versioning.version.Collection.create_new_sql_version`"""
         k['use_timestamp_numbering'] = self.use_timestamp_numbering
         self.versions.create_new_sql_version(database, description, **k)
     @property
     def latest(self):
         """API to :attr:`migrate.versioning.version.Collection.latest`"""
         return self.versions.latest
     @property
     def version_table(self):
         """Returns version_table name specified in config"""
         return self.config.get('db_settings', 'version_table')
     @property
     def id(self):
         """Returns repository id specified in config"""
         return self.config.get('db_settings', 'repository_id')
     @property
     def use_timestamp_numbering(self):
         """Returns use_timestamp_numbering specified in config"""
         ts_numbering = self.config.get('db_settings', 'use_timestamp_numbering', raw=True)
         return ts_numbering
         if self.config.has_option('db_settings', 'use_timestamp_numbering'):
             return self.config.getboolean('db_settings', 'use_timestamp_numbering')
         return False
     def version(self, *p, **k):
         """API to :attr:`migrate.versioning.version.Collection.version`"""
         return self.versions.version(*p, **k)
     @classmethod
     def clear(cls):
         # TODO: deletes repo
         super(Repository, cls).clear()
         version.Collection.clear()
     def changeset(self, database, start, end=None):
         """Create a changeset to migrate this database from ver. start to end/latest.
         :param database: name of database to generate changeset
         :param start: version to start at
         :param end: version to end at (latest if None given)
         :type database: string
         :type start: int
         :type end: int
         :returns: :class:`Changeset instance <migration.versioning.repository.Changeset>`
         """
         start = version.VerNum(start)
         if end is None:
             end = self.latest
         else:
             end = version.VerNum(end)
         if start <= end:
             step = 1
             range_mod = 1
             op = 'upgrade'
         else:
             step = -1
             range_mod = 0
             op = 'downgrade'
         versions = range(start + range_mod, end + range_mod, step)
         changes = [self.version(v).script(database, op) for v in versions]
         ret = Changeset(start, step=step, *changes)
         return ret
     @classmethod
     def create_manage_file(cls, file_, **opts):
         """Create a project management script (manage.py)
         :param file_: Destination file to be written
         :param opts: Options that are passed to :func:`migrate.versioning.shell.main`
         """
         mng_file = Template(opts.pop('templates_path', None))\
             .get_manage(theme=opts.pop('templates_theme', None))
         tmpl = open(mng_file).read()
         fd = open(file_, 'w')
         fd.write(TempitaTemplate(tmpl).substitute(opts))
         fd.close()

rhodecode/lib/dbmigrate/migrate/versioning/schemadiff.py

➞

Show inline comments

 """
    Schema differencing support.
 """
 import logging
 import sqlalchemy
 from rhodecode.lib.dbmigrate.migrate.changeset import SQLA_06
 from sqlalchemy.types import Float
 log = logging.getLogger(__name__)
 def getDiffOfModelAgainstDatabase(metadata, engine, excludeTables=None):
     """
     Return differences of model against database.
     :return: object which will evaluate to :keyword:`True` if there \
       are differences else :keyword:`False`.
     """
     return SchemaDiff(metadata,
                       sqlalchemy.MetaData(engine, reflect=True),
     db_metadata = sqlalchemy.MetaData(engine, reflect=True)
     # sqlite will include a dynamically generated 'sqlite_sequence' table if
     # there are autoincrement sequences in the database; this should not be
     # compared.
     if engine.dialect.name == 'sqlite':
         if 'sqlite_sequence' in db_metadata.tables:
             db_metadata.remove(db_metadata.tables['sqlite_sequence'])
     return SchemaDiff(metadata, db_metadata,
                       labelA='model',
                       labelB='database',
                       excludeTables=excludeTables)
 def getDiffOfModelAgainstModel(metadataA, metadataB, excludeTables=None):
     """
     Return differences of model against another model.
     :return: object which will evaluate to :keyword:`True` if there \
       are differences else :keyword:`False`.
     """
     return SchemaDiff(metadataA, metadataB, excludeTables)
 class ColDiff(object):
     """
     Container for differences in one :class:`~sqlalchemy.schema.Column`
     between two :class:`~sqlalchemy.schema.Table` instances, ``A``
     and ``B``.
     .. attribute:: col_A
       The :class:`~sqlalchemy.schema.Column` object for A.
     .. attribute:: col_B
       The :class:`~sqlalchemy.schema.Column` object for B.
     .. attribute:: type_A
       The most generic type of the :class:`~sqlalchemy.schema.Column`
       object in A.
     .. attribute:: type_B
       The most generic type of the :class:`~sqlalchemy.schema.Column`
       object in A.
     """
     diff = False
     def __init__(self,col_A,col_B):
         self.col_A = col_A
         self.col_B = col_B
         self.type_A = col_A.type
         self.type_B = col_B.type
         self.affinity_A = self.type_A._type_affinity
         self.affinity_B = self.type_B._type_affinity
         if self.affinity_A is not self.affinity_B:
             self.diff = True
             return
         if isinstance(self.type_A,Float) or isinstance(self.type_B,Float):
             if not (isinstance(self.type_A,Float) and isinstance(self.type_B,Float)):
                 self.diff=True
                 return
         for attr in ('precision','scale','length'):
             A = getattr(self.type_A,attr,None)
             B = getattr(self.type_B,attr,None)
             if not (A is None or B is None) and A!=B:
                 self.diff=True
                 return
     def __nonzero__(self):
         return self.diff
 class TableDiff(object):
     """
     Container for differences in one :class:`~sqlalchemy.schema.Table`
     between two :class:`~sqlalchemy.schema.MetaData` instances, ``A``
     and ``B``.
     .. attribute:: columns_missing_from_A
       A sequence of column names that were found in B but weren't in
       A.
     .. attribute:: columns_missing_from_B
       A sequence of column names that were found in A but weren't in
       B.
     .. attribute:: columns_different
       A dictionary containing information about columns that were
       found to be different.
       It maps column names to a :class:`ColDiff` objects describing the
       differences found.
     """
     __slots__ = (
         'columns_missing_from_A',
         'columns_missing_from_B',
         'columns_different',
+        )
     def __nonzero__(self):
         return bool(
             self.columns_missing_from_A or
             self.columns_missing_from_B or
             self.columns_different
+            )
 class SchemaDiff(object):
     """
     Compute the difference between two :class:`~sqlalchemy.schema.MetaData`
     objects.
     The string representation of a :class:`SchemaDiff` will summarise
     the changes found between the two
     :class:`~sqlalchemy.schema.MetaData` objects.
     The length of a :class:`SchemaDiff` will give the number of
     changes found, enabling it to be used much like a boolean in
     expressions.
     :param metadataA:
       First :class:`~sqlalchemy.schema.MetaData` to compare.
     :param metadataB:
       Second :class:`~sqlalchemy.schema.MetaData` to compare.
     :param labelA:
       The label to use in messages about the first
       :class:`~sqlalchemy.schema.MetaData`.
     :param labelB:
       The label to use in messages about the second
       :class:`~sqlalchemy.schema.MetaData`.
     :param excludeTables:
       A sequence of table names to exclude.
     .. attribute:: tables_missing_from_A
       A sequence of table names that were found in B but weren't in
       A.
     .. attribute:: tables_missing_from_B
       A sequence of table names that were found in A but weren't in
       B.
     .. attribute:: tables_different
       A dictionary containing information about tables that were found
       to be different.
       It maps table names to a :class:`TableDiff` objects describing the
       differences found.
     """
     def __init__(self,
                  metadataA, metadataB,
                  labelA='metadataA',
                  labelB='metadataB',
                  excludeTables=None):
         self.metadataA, self.metadataB = metadataA, metadataB
         self.labelA, self.labelB = labelA, labelB
         self.label_width = max(len(labelA),len(labelB))
         excludeTables = set(excludeTables or [])
         A_table_names = set(metadataA.tables.keys())
         B_table_names = set(metadataB.tables.keys())
         self.tables_missing_from_A = sorted(
             B_table_names - A_table_names - excludeTables
+            )
         self.tables_missing_from_B = sorted(
             A_table_names - B_table_names - excludeTables
+            )
         self.tables_different = {}
         for table_name in A_table_names.intersection(B_table_names):
             td = TableDiff()
             A_table = metadataA.tables[table_name]
             B_table = metadataB.tables[table_name]
             A_column_names = set(A_table.columns.keys())
             B_column_names = set(B_table.columns.keys())
             td.columns_missing_from_A = sorted(
                 B_column_names - A_column_names
+                )

rhodecode/lib/dbmigrate/migrate/versioning/templates/manage/default.py_tmpl

➞

Show inline comments

 #!/usr/bin/env python
 from migrate.versioning.shell import main
 {{py:
 _vars = locals().copy()
 del _vars['__template_name__']
 _vars.pop('repository_name', None)
 defaults = ", ".join(["%s='%s'" % var for var in _vars.iteritems()])
 }}
 main({{ defaults }})
 if __name__ == '__main__':
     main({{ defaults }})

rhodecode/lib/dbmigrate/migrate/versioning/templates/manage/pylons.py_tmpl

➞

Show inline comments

 #!/usr/bin/python
 # -*- coding: utf-8 -*-
 import sys
 from sqlalchemy import engine_from_config
 from paste.deploy.loadwsgi import ConfigLoader
 from migrate.versioning.shell import main
 from {{ locals().pop('repository_name') }}.model import migrations
 if '-c' in sys.argv:
     pos = sys.argv.index('-c')
     conf_path = sys.argv[pos + 1]
     del sys.argv[pos:pos + 2]
 else:
     conf_path = 'development.ini'
 {{py:
 _vars = locals().copy()
 del _vars['__template_name__']
 defaults = ", ".join(["%s='%s'" % var for var in _vars.iteritems()])
 }}
 conf_dict = ConfigLoader(conf_path).parser._sections['app:main']
 # migrate supports passing url as an existing Engine instance (since 0.6.0)
 # usage: migrate -c path/to/config.ini COMMANDS
 main(url=engine_from_config(conf_dict), repository=migrations.__path__[0],{{ defaults }})
 if __name__ == '__main__':
     main(url=engine_from_config(conf_dict), repository=migrations.__path__[0],{{ defaults }})

rhodecode/lib/dbmigrate/migrate/versioning/version.py

➞

Show inline comments

 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 import os
 import re
 import shutil
 import logging
 from rhodecode.lib.dbmigrate.migrate import exceptions
 from rhodecode.lib.dbmigrate.migrate.versioning import pathed, script
 from datetime import datetime
 log = logging.getLogger(__name__)
 class VerNum(object):
     """A version number that behaves like a string and int at the same time"""
     _instances = dict()
     def __new__(cls, value):
         val = str(value)
         if val not in cls._instances:
             cls._instances[val] = super(VerNum, cls).__new__(cls)
         ret = cls._instances[val]
         return ret
     def __init__(self,value):
         self.value = str(int(value))
         if self < 0:
             raise ValueError("Version number cannot be negative")
     def __add__(self, value):
         ret = int(self) + int(value)
         return VerNum(ret)
     def __sub__(self, value):
         return self + (int(value) * -1)
     def __cmp__(self, value):
         return int(self) - int(value)
     def __repr__(self):
         return "<VerNum(%s)>" % self.value
     def __str__(self):
         return str(self.value)
     def __int__(self):
         return int(self.value)
 class Collection(pathed.Pathed):
     """A collection of versioning scripts in a repository"""
     FILENAME_WITH_VERSION = re.compile(r'^(\d{3,}).*')
     def __init__(self, path):
         """Collect current version scripts in repository
         and store them in self.versions
         """
         super(Collection, self).__init__(path)
         # Create temporary list of files, allowing skipped version numbers.
         files = os.listdir(path)
         if '1' in files:
             # deprecation
             raise Exception('It looks like you have a repository in the old '
                 'format (with directories for each version). '
                 'Please convert repository before proceeding.')
         tempVersions = dict()
         for filename in files:
             match = self.FILENAME_WITH_VERSION.match(filename)
             if match:
                 num = int(match.group(1))
                 tempVersions.setdefault(num, []).append(filename)
             else:
                 pass  # Must be a helper file or something, let's ignore it.
         # Create the versions member where the keys
         # are VerNum's and the values are Version's.
         self.versions = dict()
         for num, files in tempVersions.items():
             self.versions[VerNum(num)] = Version(num, path, files)
     @property
     def latest(self):
         """:returns: Latest version in Collection"""
         return max([VerNum(0)] + self.versions.keys())
     def _next_ver_num(self, use_timestamp_numbering):
         print use_timestamp_numbering
         if use_timestamp_numbering == True:
             print "Creating new timestamp version!"
             return VerNum(int(datetime.utcnow().strftime('%Y%m%d%H%M%S')))
         else:
             return self.latest + 1
     def create_new_python_version(self, description, **k):
         """Create Python files for new version"""
         ver = self._next_ver_num(k.pop('use_timestamp_numbering', False))
         extra = str_to_filename(description)
         if extra:
             if extra == '_':
                 extra = ''
             elif not extra.startswith('_'):
                 extra = '_%s' % extra
         filename = '%03d%s.py' % (ver, extra)
         filepath = self._version_path(filename)
         script.PythonScript.create(filepath, **k)
         self.versions[ver] = Version(ver, self.path, [filename])
     def create_new_sql_version(self, database, description, **k):
         """Create SQL files for new version"""
         ver = self._next_ver_num(k.pop('use_timestamp_numbering', False))
         self.versions[ver] = Version(ver, self.path, [])
         extra = str_to_filename(description)
         if extra:
             if extra == '_':
                 extra = ''
             elif not extra.startswith('_'):
                 extra = '_%s' % extra
         # Create new files.
         for op in ('upgrade', 'downgrade'):
             filename = '%03d%s_%s_%s.sql' % (ver, extra, database, op)
             filepath = self._version_path(filename)
             script.SqlScript.create(filepath, **k)
             self.versions[ver].add_script(filepath)
     def version(self, vernum=None):
         """Returns latest Version if vernum is not given.
         Otherwise, returns wanted version"""
         if vernum is None:
             vernum = self.latest
         return self.versions[VerNum(vernum)]
     @classmethod
     def clear(cls):
         super(Collection, cls).clear()
     def _version_path(self, ver):
         """Returns path of file in versions repository"""
         return os.path.join(self.path, str(ver))
 class Version(object):
     """A single version in a collection
     :param vernum: Version Number
     :param path: Path to script files
     :param filelist: List of scripts
     :type vernum: int, VerNum
     :type path: string
     :type filelist: list
     """
     def __init__(self, vernum, path, filelist):
         self.version = VerNum(vernum)
         # Collect scripts in this folder
         self.sql = dict()
         self.python = None
         for script in filelist:
             self.add_script(os.path.join(path, script))
     def script(self, database=None, operation=None):
         """Returns SQL or Python Script"""
         for db in (database, 'default'):
             # Try to return a .sql script first
             try:
                 return self.sql[db][operation]
             except KeyError:
                 continue  # No .sql script exists
         # TODO: maybe add force Python parameter?
         ret = self.python
         assert ret is not None, \
             "There is no script for %d version" % self.version
         return ret
     def add_script(self, path):
         """Add script to Collection/Version"""
         if path.endswith(Extensions.py):
             self._add_script_py(path)
         elif path.endswith(Extensions.sql):
             self._add_script_sql(path)
     SQL_FILENAME = re.compile(r'^.*\.sql')
     def _add_script_sql(self, path):
         basename = os.path.basename(path)
         match = self.SQL_FILENAME.match(basename)
         if match:
             basename = basename.replace('.sql', '')
             parts = basename.split('_')
             if len(parts) < 3:
                 raise exceptions.ScriptError(
                     "Invalid SQL script name %s " % basename + \
                     "(needs to be ###_description_database_operation.sql)")
             version = parts[0]
             op = parts[-1]
             dbms = parts[-2]
         else:
             raise exceptions.ScriptError(
                 "Invalid SQL script name %s " % basename + \
                 "(needs to be ###_description_database_operation.sql)")
         # File the script into a dictionary
         self.sql.setdefault(dbms, {})[op] = script.SqlScript(path)
     def _add_script_py(self, path):
         if self.python is not None:
             raise exceptions.ScriptError('You can only have one Python script '
                 'per version, but you have: %s and %s' % (self.python, path))
         self.python = script.PythonScript(path)
 class Extensions:
     """A namespace for file extensions"""
     py = 'py'
     sql = 'sql'
 def str_to_filename(s):
     """Replaces spaces, (double and single) quotes
     and double underscores to underscores
     """
     s = s.replace(' ', '_').replace('"', '_').replace("'", '_').replace(".", "_")
     while '__' in s:
         s = s.replace('__', '_')
     return s

rhodecode/lib/dbmigrate/schema/db_1_3_0.py

➞

Show inline comments

@@ new file 100644 @@
 # -*- coding: utf-8 -*-
 """
     rhodecode.model.db
     ~~~~~~~~~~~~~~~~~~
     Database Models for RhodeCode
     :created_on: Apr 08, 2010
     :author: marcink
     :copyright: (C) 2009-2011 Marcin Kuzminski <marcin@python-works.com>
     :license: GPLv3, see COPYING for more details.
 """
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
+#
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
@@ \ No newline at end of file @@

rhodecode/lib/dbmigrate/versions/003_version_1_2_0.py

➞

Show inline comments

 import logging
 import datetime
 from sqlalchemy import *
 from sqlalchemy.exc import DatabaseError
 from sqlalchemy.orm import relation, backref, class_mapper
 from sqlalchemy.orm.session import Session
 from rhodecode.lib.dbmigrate.migrate import *
 from rhodecode.lib.dbmigrate.migrate.changeset import *
 from rhodecode.model.meta import Base
 log = logging.getLogger(__name__)
 def upgrade(migrate_engine):
     """ Upgrade operations go here.
     Don't create your own engine; bind migrate_engine to your metadata
     """
     #==========================================================================
     # Add table `groups``
     #==========================================================================
     from rhodecode.lib.dbmigrate.schema.db_1_2_0 import Group
+    from rhodecode.lib.dbmigrate.schema.db_1_2_0 import RepoGroup as Group
     Group().__table__.create()
     #==========================================================================
     # Add table `group_to_perm`
     #==========================================================================
     from rhodecode.lib.dbmigrate.schema.db_1_2_0 import UserRepoGroupToPerm
     UserRepoGroupToPerm().__table__.create()
     #==========================================================================
     # Add table `users_groups`
     #==========================================================================
     from rhodecode.lib.dbmigrate.schema.db_1_2_0 import UsersGroup
     UsersGroup().__table__.create()
     #==========================================================================
     # Add table `users_groups_members`
     #==========================================================================
     from rhodecode.lib.dbmigrate.schema.db_1_2_0 import UsersGroupMember
     UsersGroupMember().__table__.create()
     #==========================================================================
     # Add table `users_group_repo_to_perm`
     #==========================================================================
     from rhodecode.lib.dbmigrate.schema.db_1_2_0 import UsersGroupRepoToPerm
     UsersGroupRepoToPerm().__table__.create()
     #==========================================================================
     # Add table `users_group_to_perm`
     #==========================================================================
     from rhodecode.lib.dbmigrate.schema.db_1_2_0 import UsersGroupToPerm
     UsersGroupToPerm().__table__.create()
     #==========================================================================
     # Upgrade of `users` table
     #==========================================================================
     from rhodecode.lib.dbmigrate.schema.db_1_2_0 import User
     #add column
     ldap_dn = Column("ldap_dn", String(length=None, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     ldap_dn.create(User().__table__)
     api_key = Column("api_key", String(length=255, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
     api_key.create(User().__table__)
     #remove old column
     is_ldap = Column("is_ldap", Boolean(), nullable=False, unique=None, default=False)
     is_ldap.drop(User().__table__)
     #==========================================================================
     # Upgrade of `repositories` table
     #==========================================================================
     from rhodecode.lib.dbmigrate.schema.db_1_2_0 import Repository
     #ADD clone_uri column#
     clone_uri = Column("clone_uri", String(length=255, convert_unicode=False,
                                            assert_unicode=None),
                         nullable=True, unique=False, default=None)
     clone_uri.create(Repository().__table__)
     #ADD downloads column#
     enable_downloads = Column("downloads", Boolean(), nullable=True, unique=None, default=True)
     enable_downloads.create(Repository().__table__)
     #ADD column created_on
     created_on = Column('created_on', DateTime(timezone=False), nullable=True,
                         unique=None, default=datetime.datetime.now)
     created_on.create(Repository().__table__)
     #ADD group_id column#
     group_id = Column("group_id", Integer(), ForeignKey('groups.group_id'),
                   nullable=True, unique=False, default=None)
     group_id.create(Repository().__table__)
     #==========================================================================
     # Upgrade of `user_followings` table
     #==========================================================================
     from rhodecode.lib.dbmigrate.schema.db_1_2_0 import UserFollowing
     follows_from = Column('follows_from', DateTime(timezone=False),
                           nullable=True, unique=None,
                           default=datetime.datetime.now)
     follows_from.create(UserFollowing().__table__)
     return
 def downgrade(migrate_engine):
     meta = MetaData()
     meta.bind = migrate_engine

rhodecode/lib/dbmigrate/versions/004_version_1_3_0.py

➞

Show inline comments

@@ new file 100644 @@
 import logging
 import datetime
 from sqlalchemy import *
 from sqlalchemy.exc import DatabaseError
 from sqlalchemy.orm import relation, backref, class_mapper
 from sqlalchemy.orm.session import Session
 from rhodecode.lib.dbmigrate.migrate import *
 from rhodecode.lib.dbmigrate.migrate.changeset import *
 from rhodecode.model.meta import Base
 log = logging.getLogger(__name__)
 def upgrade(migrate_engine):
     """ Upgrade operations go here.
     Don't create your own engine; bind migrate_engine to your metadata
     """
     return
 def downgrade(migrate_engine):
     meta = MetaData()
     meta.bind = migrate_engine

0 comments (0 inline, 0 general)