Database Models for Kallithea

This file was forked by the Kallithea project in July 2014.

Original author and date, and relevant copyright and licensing information is below:

:created_on: Apr 08, 2010

:author: marcink

:copyright: (c) 2013 RhodeCode GmbH, and others.

:license: GPLv3, see LICENSE.md for more details.

"""

import base64

import collections

import datetime

import functools

import hashlib

import logging

import os

import time

import traceback

import ipaddr

import sqlalchemy

from beaker.cache import cache_region, region_invalidate

from sqlalchemy.ext.hybrid import hybrid_property

from sqlalchemy.orm import class_mapper, joinedload, relationship, validates

from tg.i18n import lazy_ugettext as _

from webob.exc import HTTPNotFound

import kallithea

from kallithea.lib.caching_query import FromCache

from kallithea.lib.compat import json

from kallithea.lib.exceptions import DefaultUserException

from kallithea.lib.utils2 import (

    Optional, aslist, get_changeset_safe, get_clone_url, remove_prefix, safe_bytes, safe_int, safe_str, safe_unicode, str2bool, urlreadable)

from kallithea.lib.vcs import get_backend

from kallithea.lib.vcs.backends.base import EmptyChangeset

from kallithea.lib.vcs.utils.helpers import get_scm

from kallithea.lib.vcs.utils.lazy import LazyProperty

from kallithea.model.meta import Base, Session

URL_SEP = '/'

log = logging.getLogger(__name__)

#==============================================================================

# BASE CLASSES

#==============================================================================

        the default user.

        '''

        user = super(User, cls).get_or_404(id_)

        if not allow_default and user.is_default_user:

            raise DefaultUserException()

        return user

    @classmethod

    def get_by_username_or_email(cls, username_or_email, case_insensitive=False, cache=False):

        """

        For anything that looks like an email address, look up by the email address (matching

        case insensitively).

        For anything else, try to look up by the user name.

        This assumes no normal username can have '@' symbol.

        """

        if '@' in username_or_email:

            return User.get_by_email(username_or_email, cache=cache)

        else:

            return User.get_by_username(username_or_email, case_insensitive=case_insensitive, cache=cache)

    @classmethod

    def get_by_username(cls, username, case_insensitive=False, cache=False):

        if case_insensitive:

        else:

            q = cls.query().filter(cls.username == username)

        if cache:

            q = q.options(FromCache(

                            "sql_cache_short",

                            "get_user_%s" % _hash_key(username)

                          )

            )

        return q.scalar()

    @classmethod

    def get_by_api_key(cls, api_key, cache=False, fallback=True):

        if len(api_key) != 40 or not api_key.isalnum():

            return None

        q = cls.query().filter(cls.api_key == api_key)

        if cache:

            q = q.options(FromCache("sql_cache_short",

                                    "get_api_key_%s" % api_key))

        res = q.scalar()

        if fallback and not res:

            # fallback to additional keys

            _res = UserApiKeys.query().filter_by(api_key=api_key, is_expired=False).first()

            if _res:

                res = _res.user

        if res is None or not res.active or res.is_default_user:

            return None

        return res

    @classmethod

    def get_by_email(cls, email, cache=False):

        if cache:

            q = q.options(FromCache("sql_cache_short",

                                    "get_email_key_%s" % email))

        ret = q.scalar()

        if ret is None:

            q = UserEmailMap.query()

            # try fetching in alternate email map

            q = q.options(joinedload(UserEmailMap.user))

            if cache:

                q = q.options(FromCache("sql_cache_short",

                                        "get_email_map_key_%s" % email))

            ret = getattr(q.scalar(), 'user', None)

        return ret

    @classmethod

    def get_from_cs_author(cls, author):

        """

        Tries to get User objects out of commit author string

        :param author:

        """

        from kallithea.lib.helpers import email, author_name

        # Valid email in the attribute passed, see if they're in the system

        _email = email(author)

        if _email:

            user = cls.get_by_email(_email)

            if user is not None:

                return user

        # Maybe we can match by username?

        _author = author_name(author)

            return json.loads(self._group_data)

        except TypeError:

            return {}

    @group_data.setter

    def group_data(self, val):

        try:

            self._group_data = json.dumps(val)

        except Exception:

            log.error(traceback.format_exc())

    def __repr__(self):

        return "<%s %s: %r')>" % (self.__class__.__name__,

                                  self.users_group_id,

                                  self.users_group_name)

    @classmethod

    def guess_instance(cls, value):

        return super(UserGroup, cls).guess_instance(value, UserGroup.get_by_group_name)

    @classmethod

    def get_by_group_name(cls, group_name, cache=False,

                          case_insensitive=False):

        if case_insensitive:

        else:

            q = cls.query().filter(cls.users_group_name == group_name)

        if cache:

            q = q.options(FromCache(

                            "sql_cache_short",

                            "get_group_%s" % _hash_key(group_name)

                          )

            )

        return q.scalar()

    @classmethod

    def get(cls, user_group_id, cache=False):

        user_group = cls.query()

        if cache:

            user_group = user_group.options(FromCache("sql_cache_short",

                                    "get_users_group_%s" % user_group_id))

        return user_group.get(user_group_id)

    def get_api_data(self, with_members=True):

        user_group = self

        data = dict(

            users_group_id=user_group.users_group_id,

            group_name=user_group.users_group_name,

    def changeset_cache(self):

        try:

            cs_cache = json.loads(self._changeset_cache) # might raise on bad data

            cs_cache['raw_id'] # verify data, raise exception on error

            return cs_cache

        except (TypeError, KeyError, ValueError):

            return EmptyChangeset().__json__()

    @changeset_cache.setter

    def changeset_cache(self, val):

        try:

            self._changeset_cache = json.dumps(val)

        except Exception:

            log.error(traceback.format_exc())

    @classmethod

    def query(cls, sorted=False):

        """Add Repository-specific helpers for common query constructs.

        sorted: if True, apply the default ordering (name, case insensitive).

        """

        q = super(Repository, cls).query()

        if sorted:

        return q

    @classmethod

    def url_sep(cls):

        return URL_SEP

    @classmethod

    def normalize_repo_name(cls, repo_name):

        """

        Normalizes os specific repo_name to the format internally stored inside

        database using URL_SEP

        :param cls:

        :param repo_name:

        """

        return cls.url_sep().join(repo_name.split(os.sep))

    @classmethod

    def guess_instance(cls, value):

        return super(Repository, cls).guess_instance(value, Repository.get_by_repo_name)

    @classmethod

    def get_by_repo_name(cls, repo_name, case_insensitive=False):

        """Get the repo, defaulting to database case sensitivity.

        case_insensitive will be slower and should only be specified if necessary."""

        if case_insensitive:

        else:

            q = Session().query(cls).filter(cls.repo_name == repo_name)

        q = q.options(joinedload(Repository.fork)) \

                .options(joinedload(Repository.owner)) \

                .options(joinedload(Repository.group))

        return q.scalar()

    @classmethod

    def get_by_full_path(cls, repo_full_path):

        base_full_path = os.path.realpath(cls.base_path())

        repo_full_path = os.path.realpath(repo_full_path)

        assert repo_full_path.startswith(base_full_path + os.path.sep)

        repo_name = repo_full_path[len(base_full_path) + 1:]

        repo_name = cls.normalize_repo_name(repo_name)

        return cls.get_by_repo_name(repo_name.strip(URL_SEP))

    @classmethod

    def get_repo_forks(cls, repo_id):

        return cls.query().filter(Repository.fork_id == repo_id)

    @classmethod

    def base_path(cls):

        """

        Returns base path where all repos are stored

    SEP = ' » '

    group_id = Column(Integer(), primary_key=True)

    group_name = Column(Unicode(255), nullable=False, unique=True) # full path

    parent_group_id = Column('group_parent_id', Integer(), ForeignKey('groups.group_id'), nullable=True)

    group_description = Column(Unicode(10000), nullable=False)

    owner_id = Column('user_id', Integer(), ForeignKey('users.user_id'), nullable=False)

    created_on = Column(DateTime(timezone=False), nullable=False, default=datetime.datetime.now)

    repo_group_to_perm = relationship('UserRepoGroupToPerm', cascade='all', order_by='UserRepoGroupToPerm.group_to_perm_id')

    users_group_to_perm = relationship('UserGroupRepoGroupToPerm', cascade='all')

    parent_group = relationship('RepoGroup', remote_side=group_id)

    owner = relationship('User')

    @classmethod

    def query(cls, sorted=False):

        """Add RepoGroup-specific helpers for common query constructs.

        sorted: if True, apply the default ordering (name, case insensitive).

        """

        q = super(RepoGroup, cls).query()

        if sorted:

        return q

    def __init__(self, group_name='', parent_group=None):

        self.group_name = group_name

        self.parent_group = parent_group

    def __repr__(self):

        return "<%s %s: %s>" % (self.__class__.__name__,

                                self.group_id, self.group_name)

    @classmethod

    def _generate_choice(cls, repo_group):

        """Return tuple with group_id and name as html literal"""

        from webhelpers2.html import literal

        if repo_group is None:

            return (-1, u'-- %s --' % _('top level'))

        return repo_group.group_id, literal(cls.SEP.join(repo_group.full_path_splitted))

    @classmethod

    def groups_choices(cls, groups):

        """Return tuples with group_id and name as html literal."""

        return sorted((cls._generate_choice(g) for g in groups),

                      key=lambda c: c[1].split(cls.SEP))

    @classmethod

    def url_sep(cls):

        return URL_SEP

    @classmethod

    def guess_instance(cls, value):

        return super(RepoGroup, cls).guess_instance(value, RepoGroup.get_by_group_name)

    @classmethod

    def get_by_group_name(cls, group_name, cache=False, case_insensitive=False):

        group_name = group_name.rstrip('/')

        if case_insensitive:

            gr = cls.query() \

        else:

            gr = cls.query() \

                .filter(cls.group_name == group_name)

        if cache:

            gr = gr.options(FromCache(

                            "sql_cache_short",

                            "get_group_%s" % _hash_key(group_name)

                            )

            )

        return gr.scalar()

    @property

    def parents(self):

        groups = []

        group = self.parent_group

        while group is not None:

            groups.append(group)

            group = group.parent_group

            assert group not in groups, group # avoid recursion on bad db content

        groups.reverse()

        return groups

    @property

    def children(self):

    commit_activity = Column(LargeBinary(1000000), nullable=False) # JSON data

    commit_activity_combined = Column(LargeBinary(), nullable=False) # JSON data

    languages = Column(LargeBinary(1000000), nullable=False) # JSON data

    repository = relationship('Repository', single_parent=True)

class UserFollowing(Base, BaseDbModel):

    __tablename__ = 'user_followings'

    __table_args__ = (

        UniqueConstraint('user_id', 'follows_repository_id', name='uq_user_followings_user_repo'),

        UniqueConstraint('user_id', 'follows_user_id', name='uq_user_followings_user_user'),

        _table_args_default_dict,

    )

    user_following_id = Column(Integer(), primary_key=True)

    user_id = Column(Integer(), ForeignKey('users.user_id'), nullable=False)

    follows_repository_id = Column(Integer(), ForeignKey('repositories.repo_id'), nullable=True)

    follows_user_id = Column(Integer(), ForeignKey('users.user_id'), nullable=True)

    follows_from = Column(DateTime(timezone=False), nullable=False, default=datetime.datetime.now)

    user = relationship('User', primaryjoin='User.user_id==UserFollowing.user_id')

    follows_user = relationship('User', primaryjoin='User.user_id==UserFollowing.follows_user_id')

    @classmethod

    def get_repo_followers(cls, repo_id):

        return cls.query().filter(cls.follows_repository_id == repo_id)

class CacheInvalidation(Base, BaseDbModel):

    __tablename__ = 'cache_invalidation'

    __table_args__ = (

        Index('key_idx', 'cache_key'),

        _table_args_default_dict,

    )

    # cache_id, not used

    cache_id = Column(Integer(), primary_key=True)

    # cache_key as created by _get_cache_key

    cache_key = Column(Unicode(255), nullable=False, unique=True)

    # cache_args is a repo_name

    cache_args = Column(Unicode(255), nullable=False)

    # instance sets cache_active True when it is caching, other instances set

    # cache_active to False to indicate that this cache is invalid

    cache_active = Column(Boolean(), nullable=False, default=False)

    def __init__(self, cache_key, repo_name=''):