Changeset - 83f7b5449860
[Not reviewed]
default
0 5 0
Mads Kiilerich - 10 years ago 2016-01-05 16:30:12
madski@unity3d.com
db: stop messing with convert_unicode in .ini and db scheme

convert_unicode=False is the default for String fields ... except when the .ini file contains sets it to True.

String fields are however no longer used for unicode and we can drop convert_unicode everywhere.

See
http://docs.sqlalchemy.org/en/latest/core/type_basics.html#sqlalchemy.types.String.params.convert_unicode
for the advice of never setting convert_unicode.
5 files changed with 25 insertions and 29 deletions:
0 comments (0 inline, 0 general)
development.ini
Show inline comments
 
@@ -490,25 +490,24 @@ logview.pylons.util = #eee
 
sqlalchemy.db1.url = sqlite:///%(here)s/kallithea.db?timeout=60
 

	
 
# POSTGRESQL
 
#sqlalchemy.db1.url = postgresql://user:pass@localhost/kallithea
 

	
 
# MySQL
 
#sqlalchemy.db1.url = mysql://user:pass@localhost/kallithea?charset=utf8
 

	
 
# see sqlalchemy docs for others
 

	
 
sqlalchemy.db1.echo = false
 
sqlalchemy.db1.pool_recycle = 3600
 
sqlalchemy.db1.convert_unicode = true
 

	
 
################################
 
### LOGGING CONFIGURATION   ####
 
################################
 

	
 
[loggers]
 
keys = root, routes, kallithea, sqlalchemy, beaker, templates, whoosh_indexer
 

	
 
[handlers]
 
keys = console, console_sql
 

	
 
[formatters]
kallithea/bin/template.ini.mako
Show inline comments
 
@@ -493,25 +493,24 @@ sqlalchemy.db1.url = sqlite:///${here}/k
 
# POSTGRESQL
 
sqlalchemy.db1.url = postgresql://user:pass@localhost/kallithea
 

	
 
%elif database_engine == 'mysql':
 
# MySQL
 
sqlalchemy.db1.url = mysql://user:pass@localhost/kallithea?charset=utf8
 

	
 
%endif
 
# see sqlalchemy docs for others
 

	
 
sqlalchemy.db1.echo = false
 
sqlalchemy.db1.pool_recycle = 3600
 
sqlalchemy.db1.convert_unicode = true
 

	
 
<%text>################################</%text>
 
<%text>### LOGGING CONFIGURATION   ####</%text>
 
<%text>################################</%text>
 

	
 
[loggers]
 
keys = root, routes, kallithea, sqlalchemy, beaker, templates, whoosh_indexer
 

	
 
[handlers]
 
keys = console, console_sql
 

	
 
[formatters]
kallithea/config/deployment.ini_tmpl
Show inline comments
 
@@ -482,25 +482,24 @@ logview.pylons.util = #eee
 
sqlalchemy.db1.url = sqlite:///%(here)s/kallithea.db?timeout=60
 

	
 
# POSTGRESQL
 
#sqlalchemy.db1.url = postgresql://user:pass@localhost/kallithea
 

	
 
# MySQL
 
#sqlalchemy.db1.url = mysql://user:pass@localhost/kallithea?charset=utf8
 

	
 
# see sqlalchemy docs for others
 

	
 
sqlalchemy.db1.echo = false
 
sqlalchemy.db1.pool_recycle = 3600
 
sqlalchemy.db1.convert_unicode = true
 

	
 
################################
 
### LOGGING CONFIGURATION   ####
 
################################
 

	
 
[loggers]
 
keys = root, routes, kallithea, sqlalchemy, beaker, templates, whoosh_indexer
 

	
 
[handlers]
 
keys = console, console_sql
 

	
 
[formatters]
kallithea/model/db.py
Show inline comments
 
@@ -172,27 +172,27 @@ class Setting(Base, BaseModel):
 
    )
 

	
 
    SETTINGS_TYPES = {
 
        'str': safe_str,
 
        'int': safe_int,
 
        'unicode': safe_unicode,
 
        'bool': str2bool,
 
        'list': functools.partial(aslist, sep=',')
 
    }
 
    DEFAULT_UPDATE_URL = ''
 

	
 
    app_settings_id = Column(Integer(), nullable=False, unique=True, primary_key=True)
 
    app_settings_name = Column(String(255, convert_unicode=False), nullable=True, unique=None, default=None)
 
    app_settings_name = Column(String(255), nullable=True, unique=None, default=None)
 
    _app_settings_value = Column("app_settings_value", Unicode(4096), nullable=True, unique=None, default=None)
 
    _app_settings_type = Column("app_settings_type", String(255, convert_unicode=False), nullable=True, unique=None, default=None)
 
    _app_settings_type = Column("app_settings_type", String(255), nullable=True, unique=None, default=None)
 

	
 
    def __init__(self, key='', val='', type='unicode'):
 
        self.app_settings_name = key
 
        self.app_settings_value = val
 
        self.app_settings_type = type
 

	
 
    @validates('_app_settings_value')
 
    def validate_settings_value(self, key, val):
 
        assert type(val) == unicode
 
        return val
 

	
 
    @hybrid_property
 
@@ -334,27 +334,27 @@ class Ui(Base, BaseModel):
 
        UniqueConstraint('ui_key'),
 
        _table_args_default_dict,
 
    )
 

	
 
    HOOK_UPDATE = 'changegroup.update'
 
    HOOK_REPO_SIZE = 'changegroup.repo_size'
 
    HOOK_PUSH = 'changegroup.push_logger'
 
    HOOK_PRE_PUSH = 'prechangegroup.pre_push'
 
    HOOK_PULL = 'outgoing.pull_logger'
 
    HOOK_PRE_PULL = 'preoutgoing.pre_pull'
 

	
 
    ui_id = Column(Integer(), nullable=False, unique=True, primary_key=True)
 
    ui_section = Column(String(255, convert_unicode=False), nullable=True, unique=None, default=None)
 
    ui_key = Column(String(255, convert_unicode=False), nullable=True, unique=None, default=None)
 
    ui_value = Column(String(255, convert_unicode=False), nullable=True, unique=None, default=None)
 
    ui_section = Column(String(255), nullable=True, unique=None, default=None)
 
    ui_key = Column(String(255), nullable=True, unique=None, default=None)
 
    ui_value = Column(String(255), nullable=True, unique=None, default=None)
 
    ui_active = Column(Boolean(), nullable=True, unique=None, default=True)
 

	
 
    # def __init__(self, section='', key='', value=''):
 
    #     self.ui_section = section
 
    #     self.ui_key = key
 
    #     self.ui_value = value
 

	
 
    @classmethod
 
    def get_by_key(cls, key):
 
        return cls.query().filter(cls.ui_key == key).scalar()
 

	
 
    @classmethod
 
@@ -398,35 +398,35 @@ class User(Base, BaseModel):
 
    __table_args__ = (
 
        UniqueConstraint('username'),
 
        UniqueConstraint('email'),
 
        Index('u_username_idx', 'username'),
 
        Index('u_email_idx', 'email'),
 
        _table_args_default_dict,
 
    )
 

	
 
    DEFAULT_USER = 'default'
 
    DEFAULT_GRAVATAR_URL = 'https://secure.gravatar.com/avatar/{md5email}?d=identicon&s={size}'
 

	
 
    user_id = Column(Integer(), nullable=False, unique=True, primary_key=True)
 
    username = Column(String(255, convert_unicode=False), nullable=True, unique=None, default=None)
 
    password = Column(String(255, convert_unicode=False), nullable=True, unique=None, default=None)
 
    username = Column(String(255), nullable=True, unique=None, default=None)
 
    password = Column(String(255), nullable=True, unique=None, default=None)
 
    active = Column(Boolean(), nullable=True, unique=None, default=True)
 
    admin = Column(Boolean(), nullable=True, unique=None, default=False)
 
    name = Column("firstname", Unicode(255), nullable=True, unique=None, default=None)
 
    lastname = Column(Unicode(255), nullable=True, unique=None, default=None)
 
    _email = Column("email", String(255, convert_unicode=False), nullable=True, unique=None, default=None)
 
    _email = Column("email", String(255), nullable=True, unique=None, default=None)
 
    last_login = Column(DateTime(timezone=False), nullable=True, unique=None, default=None)
 
    extern_type = Column(String(255, convert_unicode=False), nullable=True, unique=None, default=None)
 
    extern_name = Column(String(255, convert_unicode=False), nullable=True, unique=None, default=None)
 
    api_key = Column(String(255, convert_unicode=False), nullable=True, unique=None, default=None)
 
    extern_type = Column(String(255), nullable=True, unique=None, default=None)
 
    extern_name = Column(String(255), nullable=True, unique=None, default=None)
 
    api_key = Column(String(255), nullable=True, unique=None, default=None)
 
    inherit_default_permissions = Column(Boolean(), nullable=False, unique=None, default=True)
 
    created_on = Column(DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
 
    _user_data = Column("user_data", LargeBinary(), nullable=True)  # JSON data
 

	
 
    user_log = relationship('UserLog')
 
    user_perms = relationship('UserToPerm', primaryjoin="User.user_id==UserToPerm.user_id", cascade='all')
 

	
 
    repositories = relationship('Repository')
 
    repo_groups = relationship('RepoGroup')
 
    user_groups = relationship('UserGroup')
 
    user_followers = relationship('UserFollowing', primaryjoin='UserFollowing.follows_user_id==User.user_id', cascade='all')
 
    followings = relationship('UserFollowing', primaryjoin='UserFollowing.user_id==User.user_id', cascade='all')
 
@@ -694,50 +694,50 @@ class User(Base, BaseModel):
 

	
 
class UserApiKeys(Base, BaseModel):
 
    __tablename__ = 'user_api_keys'
 
    __table_args__ = (
 
        Index('uak_api_key_idx', 'api_key'),
 
        Index('uak_api_key_expires_idx', 'api_key', 'expires'),
 
        _table_args_default_dict,
 
    )
 
    __mapper_args__ = {}
 

	
 
    user_api_key_id = Column(Integer(), nullable=False, unique=True, primary_key=True)
 
    user_id = Column(Integer(), ForeignKey('users.user_id'), nullable=True, unique=None, default=None)
 
    api_key = Column(String(255, convert_unicode=False), nullable=False, unique=True)
 
    api_key = Column(String(255), nullable=False, unique=True)
 
    description = Column(UnicodeText(1024))
 
    expires = Column(Float(53), nullable=False)
 
    created_on = Column(DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
 

	
 
    user = relationship('User')
 

	
 
    @property
 
    def expired(self):
 
        if self.expires == -1:
 
            return False
 
        return time.time() > self.expires
 

	
 

	
 
class UserEmailMap(Base, BaseModel):
 
    __tablename__ = 'user_email_map'
 
    __table_args__ = (
 
        Index('uem_email_idx', 'email'),
 
        UniqueConstraint('email'),
 
        _table_args_default_dict,
 
    )
 
    __mapper_args__ = {}
 

	
 
    email_id = Column(Integer(), nullable=False, unique=True, primary_key=True)
 
    user_id = Column(Integer(), ForeignKey('users.user_id'), nullable=True, unique=None, default=None)
 
    _email = Column("email", String(255, convert_unicode=False), nullable=True, unique=False, default=None)
 
    _email = Column("email", String(255), nullable=True, unique=False, default=None)
 
    user = relationship('User')
 

	
 
    @validates('_email')
 
    def validate_email(self, key, email):
 
        # check if this email is not main one
 
        main_email = Session().query(User).filter(User.email == email).scalar()
 
        if main_email is not None:
 
            raise AttributeError('email %s is present is user table' % email)
 
        return email
 

	
 
    @hybrid_property
 
    def email(self):
 
@@ -749,25 +749,25 @@ class UserEmailMap(Base, BaseModel):
 

	
 

	
 
class UserIpMap(Base, BaseModel):
 
    __tablename__ = 'user_ip_map'
 
    __table_args__ = (
 
        UniqueConstraint('user_id', 'ip_addr'),
 
        _table_args_default_dict,
 
    )
 
    __mapper_args__ = {}
 

	
 
    ip_id = Column(Integer(), nullable=False, unique=True, primary_key=True)
 
    user_id = Column(Integer(), ForeignKey('users.user_id'), nullable=True, unique=None, default=None)
 
    ip_addr = Column(String(255, convert_unicode=False), nullable=True, unique=False, default=None)
 
    ip_addr = Column(String(255), nullable=True, unique=False, default=None)
 
    active = Column(Boolean(), nullable=True, unique=None, default=True)
 
    user = relationship('User')
 

	
 
    @classmethod
 
    def _get_ip_range(cls, ip_addr):
 
        from kallithea.lib import ipaddr
 
        net = ipaddr.IPNetwork(address=ip_addr)
 
        return [str(net.network), str(net.broadcast)]
 

	
 
    def __json__(self):
 
        return dict(
 
          ip_addr=self.ip_addr,
 
@@ -777,28 +777,28 @@ class UserIpMap(Base, BaseModel):
 
    def __unicode__(self):
 
        return u"<%s('user_id:%s=>%s')>" % (self.__class__.__name__,
 
                                            self.user_id, self.ip_addr)
 

	
 
class UserLog(Base, BaseModel):
 
    __tablename__ = 'user_logs'
 
    __table_args__ = (
 
        _table_args_default_dict,
 
    )
 

	
 
    user_log_id = Column(Integer(), nullable=False, unique=True, primary_key=True)
 
    user_id = Column(Integer(), ForeignKey('users.user_id'), nullable=True, unique=None, default=None)
 
    username = Column(String(255, convert_unicode=False), nullable=True, unique=None, default=None)
 
    username = Column(String(255), nullable=True, unique=None, default=None)
 
    repository_id = Column(Integer(), ForeignKey('repositories.repo_id'), nullable=True)
 
    repository_name = Column(Unicode(255), nullable=True, unique=None, default=None)
 
    user_ip = Column(String(255, convert_unicode=False), nullable=True, unique=None, default=None)
 
    user_ip = Column(String(255), nullable=True, unique=None, default=None)
 
    action = Column(UnicodeText(1200000), nullable=True, unique=None, default=None)
 
    action_date = Column(DateTime(timezone=False), nullable=True, unique=None, default=None)
 

	
 
    def __unicode__(self):
 
        return u"<%s('id:%s:%s')>" % (self.__class__.__name__,
 
                                      self.repository_name,
 
                                      self.action)
 

	
 
    @property
 
    def action_as_day(self):
 
        return datetime.date(*self.action_date.timetuple()[:3])
 

	
 
@@ -915,28 +915,28 @@ class UserGroupMember(Base, BaseModel):
 

	
 
class RepositoryField(Base, BaseModel):
 
    __tablename__ = 'repositories_fields'
 
    __table_args__ = (
 
        UniqueConstraint('repository_id', 'field_key'),  # no-multi field
 
        _table_args_default_dict,
 
    )
 

	
 
    PREFIX = 'ex_'  # prefix used in form to not conflict with already existing fields
 

	
 
    repo_field_id = Column(Integer(), nullable=False, unique=True, primary_key=True)
 
    repository_id = Column(Integer(), ForeignKey('repositories.repo_id'), nullable=False, unique=None, default=None)
 
    field_key = Column(String(250, convert_unicode=False))
 
    field_label = Column(String(1024, convert_unicode=False), nullable=False)
 
    field_value = Column(String(10000, convert_unicode=False), nullable=False)
 
    field_desc = Column(String(1024, convert_unicode=False), nullable=False)
 
    field_key = Column(String(250))
 
    field_label = Column(String(1024), nullable=False)
 
    field_value = Column(String(10000), nullable=False)
 
    field_desc = Column(String(1024), nullable=False)
 
    field_type = Column(String(255), nullable=False, unique=None)
 
    created_on = Column(DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
 

	
 
    repository = relationship('Repository')
 

	
 
    @property
 
    def field_key_prefixed(self):
 
        return 'ex_%s' % self.field_key
 

	
 
    @classmethod
 
    def un_prefix_key(cls, key):
 
        if key.startswith(cls.PREFIX):
 
@@ -960,36 +960,36 @@ class Repository(Base, BaseModel):
 

	
 
    DEFAULT_CLONE_URI = '{scheme}://{user}@{netloc}/{repo}'
 
    DEFAULT_CLONE_URI_ID = '{scheme}://{user}@{netloc}/_{repoid}'
 

	
 
    STATE_CREATED = 'repo_state_created'
 
    STATE_PENDING = 'repo_state_pending'
 
    STATE_ERROR = 'repo_state_error'
 

	
 
    repo_id = Column(Integer(), nullable=False, unique=True, primary_key=True)
 
    repo_name = Column(Unicode(255), nullable=False, unique=True, default=None)
 
    repo_state = Column(String(255), nullable=True)
 

	
 
    clone_uri = Column(String(255, convert_unicode=False), nullable=True, unique=False, default=None)
 
    repo_type = Column(String(255, convert_unicode=False), nullable=False, unique=False, default=None)
 
    clone_uri = Column(String(255), nullable=True, unique=False, default=None)
 
    repo_type = Column(String(255), nullable=False, unique=False, default=None)
 
    user_id = Column(Integer(), ForeignKey('users.user_id'), nullable=False, unique=False, default=None)
 
    private = Column(Boolean(), nullable=True, unique=None, default=None)
 
    enable_statistics = Column("statistics", Boolean(), nullable=True, unique=None, default=True)
 
    enable_downloads = Column("downloads", Boolean(), nullable=True, unique=None, default=True)
 
    description = Column(Unicode(10000), nullable=True, unique=None, default=None)
 
    created_on = Column(DateTime(timezone=False), nullable=False, unique=None, default=datetime.datetime.now)
 
    updated_on = Column(DateTime(timezone=False), nullable=False, unique=None, default=datetime.datetime.now)
 
    _landing_revision = Column("landing_revision", String(255, convert_unicode=False), nullable=False, unique=False, default=None)
 
    _landing_revision = Column("landing_revision", String(255), nullable=False, unique=False, default=None)
 
    enable_locking = Column(Boolean(), nullable=False, unique=None, default=False)
 
    _locked = Column("locked", String(255, convert_unicode=False), nullable=True, unique=False, default=None)
 
    _locked = Column("locked", String(255), nullable=True, unique=False, default=None)
 
    _changeset_cache = Column("changeset_cache", LargeBinary(), nullable=True) #JSON data
 

	
 
    fork_id = Column(Integer(), ForeignKey('repositories.repo_id'), nullable=True, unique=False, default=None)
 
    group_id = Column(Integer(), ForeignKey('groups.group_id'), nullable=True, unique=False, default=None)
 

	
 
    user = relationship('User')
 
    fork = relationship('Repository', remote_side=repo_id)
 
    group = relationship('RepoGroup')
 
    repo_to_perm = relationship('UserRepoToPerm', cascade='all', order_by='UserRepoToPerm.repo_to_perm_id')
 
    users_group_to_perm = relationship('UserGroupRepoToPerm', cascade='all')
 
    stats = relationship('Statistics', cascade='all', uselist=False)
 

	
 
@@ -1735,25 +1735,25 @@ class Permission(Base, BaseModel):
 

	
 
        'hg.usergroup.create.false': 0,
 
        'hg.usergroup.create.true': 1,
 

	
 
        'hg.fork.none': 0,
 
        'hg.fork.repository': 1,
 

	
 
        'hg.create.none': 0,
 
        'hg.create.repository': 1
 
    }
 

	
 
    permission_id = Column(Integer(), nullable=False, unique=True, primary_key=True)
 
    permission_name = Column(String(255, convert_unicode=False), nullable=True, unique=None, default=None)
 
    permission_name = Column(String(255), nullable=True, unique=None, default=None)
 

	
 
    def __unicode__(self):
 
        return u"<%s('%s:%s')>" % (
 
            self.__class__.__name__, self.permission_id, self.permission_name
 
        )
 

	
 
    @classmethod
 
    def get_by_key(cls, key):
 
        return cls.query().filter(cls.permission_name == key).scalar()
 

	
 
    @classmethod
 
    def get_default_perms(cls, default_user_id):
kallithea/tests/test.ini
Show inline comments
 
@@ -492,25 +492,24 @@ logview.pylons.util = #eee
 
sqlalchemy.db1.url = sqlite:///%(here)s/kallithea_test.sqlite
 

	
 
# POSTGRESQL
 
#sqlalchemy.db1.url = postgresql://user:pass@localhost/kallithea
 

	
 
# MySQL
 
#sqlalchemy.db1.url = mysql://user:pass@localhost/kallithea?charset=utf8
 

	
 
# see sqlalchemy docs for others
 

	
 
sqlalchemy.db1.echo = false
 
sqlalchemy.db1.pool_recycle = 3600
 
sqlalchemy.db1.convert_unicode = true
 

	
 
################################
 
### LOGGING CONFIGURATION   ####
 
################################
 

	
 
[loggers]
 
keys = root, routes, kallithea, sqlalchemy, beaker, templates, whoosh_indexer
 

	
 
[handlers]
 
keys = console, console_sql
 

	
 
[formatters]
0 comments (0 inline, 0 general)