"""

    Extract urls from text and make html links out of them

    """

    return url_re.sub(_urlify_text_replace, s)

def urlify_text(s, truncate=None, stylize=False, truncatef=truncate):

    """

    Extract urls from text and make literal html links out of them

    """

    if truncate is not None:

        s = truncatef(s, truncate)

    s = html_escape(s)

    if stylize:

        s = desc_stylize(s)

    s = _urlify_text(s)

    return literal(s)

def _urlify_changeset_replace_f(repository):

    from pylons import url  # doh, we need to re-import url to mock it later

    def urlify_changeset_replace(match_obj):

        rev = match_obj.group(0)

        return '<a class="revision-link" href="%(url)s">%(rev)s</a>' % {

         'url': url('changeset_home', repo_name=repository, revision=rev),

         'rev': rev,

        }

    return urlify_changeset_replace

urilify_changeset_re = r'(?:^|(?<=[\s(),]))([0-9a-fA-F]{12,40})(?=$|\s|[.,:()])'

def urlify_changesets(text_, repository):

    """

    Extract revision ids from changeset and make link from them

    :param text_:

    :param repository: repo name to build the URL with

    """

    urlify_changeset_replace = _urlify_changeset_replace_f(repository)

    return re.sub(urilify_changeset_re, urlify_changeset_replace, text_)

def linkify_others(t, l):

    # attempt at fixing double quoting?

    urls = re.compile(r'(\<a.*?\<\/a\>)',)

    links = []

    for e in urls.split(t):

        if not urls.match(e):

            links.append('<a class="message-link" href="%s">%s</a>' % (l, e))

        else:

            links.append(e)

    return ''.join(links)

def urlify_commit(text_, repository, link_=None):

    """

    Parses given text message and makes proper links.

    issues are linked to given issue-server, and rest is a changeset link

    if link_ is given, in other case it's a plain text

    :param text_:

    :param repository:

    :param link_: changeset link

    """

    newtext = html_escape(text_)

    # urlify changesets - extract revisions and make link out of them

    newtext = urlify_changesets(newtext, repository)

    # extract http/https links and make them real urls

    newtext = _urlify_text(newtext)

    newtext = urlify_issues(newtext, repository, link_)

    return literal(newtext)

def _urlify_issues_replace_f(repository, ISSUE_SERVER_LNK, ISSUE_PREFIX):

    def urlify_issues_replace(match_obj):

        pref = ''

        if match_obj.group().startswith(' '):

            pref = ' '

        issue_id = ''.join(match_obj.groups())

        issue_url = ISSUE_SERVER_LNK.replace('{id}', issue_id)

        if repository:

            issue_url = issue_url.replace('{repo}', repository)

            repo_name = repository.split(URL_SEP)[-1]

            issue_url = issue_url.replace('{repo_name}', repo_name)

        return (

            '%(pref)s<a class="%(cls)s" href="%(url)s">'

            '%(issue-prefix)s%(id-repr)s'

            '</a>'

            ) % {

             'pref': pref,

             'cls': 'issue-tracker-link',

             'url': issue_url,

             'id-repr': issue_id,

             'issue-prefix': ISSUE_PREFIX,

             'serv': ISSUE_SERVER_LNK,

            }

    return urlify_issues_replace

def urlify_issues(newtext, repository, link_=None):

    from kallithea import CONFIG as conf

    # allow multiple issue servers to be used

    valid_indices = [

        x.group(1)

        for x in map(lambda x: re.match(r'issue_pat(.*)', x), conf.keys())

        if x and 'issue_server_link%s' % x.group(1) in conf

        and 'issue_prefix%s' % x.group(1) in conf

    ]

    if valid_indices:

        log.debug('found issue server suffixes `%s` during valuation of: %s',

                  ','.join(valid_indices), newtext)

    for pattern_index in valid_indices:

        ISSUE_PATTERN = conf.get('issue_pat%s' % pattern_index)

        ISSUE_SERVER_LNK = conf.get('issue_server_link%s' % pattern_index)

        ISSUE_PREFIX = conf.get('issue_prefix%s' % pattern_index)

        log.debug('pattern suffix `%s` PAT:%s SERVER_LINK:%s PREFIX:%s',

                  pattern_index, ISSUE_PATTERN, ISSUE_SERVER_LNK,

                  ISSUE_PREFIX)

        URL_PAT = re.compile(ISSUE_PATTERN)

        urlify_issues_replace = _urlify_issues_replace_f(repository, ISSUE_SERVER_LNK, ISSUE_PREFIX)

        newtext = URL_PAT.sub(urlify_issues_replace, newtext)

        log.debug('processed prefix:`%s` => %s', pattern_index, newtext)

    # if we actually did something above

    if link_:

        # wrap not links into final link => link_

        newtext = linkify_others(newtext, link_)

    return newtext

def _mentions_replace(match_obj):

    return '<b>@%s</b>' % match_obj.group(1)

def render_w_mentions(source, repo_name=None):

    """

    Render plain text with revision hashes and issue references urlified

    and with @mention highlighting.

    """

    s = source.rstrip()

    s = safe_unicode(s)

    s = '\n'.join(s.splitlines())

    s = html_escape(s)

    # this sequence of html-ifications seems to be safe and non-conflicting

    # if the issues regexp is sane

    s = _urlify_text(s)

    if repo_name is not None:

        s = urlify_changesets(s, repo_name)

    s = urlify_issues(s, repo_name)

    s = MENTIONS_REGEX.sub(_mentions_replace, s)

    return literal('<code style="white-space:pre-wrap">%s</code>' % s)

def short_ref(ref_type, ref_name):

    if ref_type == 'rev':

        return short_id(ref_name)

    return ref_name

def link_to_ref(repo_name, ref_type, ref_name, rev=None):

    """

    Return full markup for a href to changeset_home for a changeset.

    If ref_type is branch it will link to changelog.

    ref_name is shortened if ref_type is 'rev'.

    if rev is specified show it too, explicitly linking to that revision.

    """

    txt = short_ref(ref_type, ref_name)

    if ref_type == 'branch':

        u = url('changelog_home', repo_name=repo_name, branch=ref_name)

    else:

        u = url('changeset_home', repo_name=repo_name, revision=ref_name)

    l = link_to(repo_name + '#' + txt, u)

    if rev and ref_type != 'rev':

        l = literal('%s (%s)' % (l, link_to(short_id(rev), url('changeset_home', repo_name=repo_name, revision=rev))))

    return l

def changeset_status(repo, revision):

    return ChangesetStatusModel().get_status(repo, revision)

def changeset_status_lbl(changeset_status):

def get_permission_name(key):

    return dict(Permission.PERMS).get(key)

def journal_filter_help():

    return _(textwrap.dedent('''

        Example filter terms:

            repository:vcs

            username:developer

            action:*push*

            ip:127.0.0.1

            date:20120101

            date:[20120101100000 TO 20120102]

        Generate wildcards using '*' character:

            "repository:vcs*" - search everything starting with 'vcs'

            "repository:*vcs*" - search for repository containing 'vcs'

        Optional AND / OR operators in queries

            "repository:vcs OR repository:test"

            "username:test AND repository:test*"

    '''))

def not_mapped_error(repo_name):

    flash(_('%s repository is not mapped to db perhaps'

            ' it was created or renamed from the filesystem'

            ' please run the application again'

            ' in order to rescan repositories') % repo_name, category='error')

def ip_range(ip_addr):

    from kallithea.model.db import UserIpMap

    s, e = UserIpMap._get_ip_range(ip_addr)

    return '%s - %s' % (s, e)

def form(url, method="post", **attrs):

    """Like webhelpers.html.tags.form but automatically using secure_form with

    authentication_token for POST. authentication_token is thus never leaked

    in the URL."""

    if method.lower() == 'get':

        return insecure_form(url, method=method, **attrs)

    # webhelpers will turn everything but GET into POST

    return secure_form(url, method=method, **attrs)

    # cache_active to False to indicate that this cache is invalid

    cache_active = Column(Boolean(), nullable=False, default=False)

    def __init__(self, cache_key, repo_name=''):

        self.cache_key = cache_key

        self.cache_args = repo_name

        self.cache_active = False

    def __unicode__(self):

        return u"<%s('%s:%s[%s]')>" % (

            self.__class__.__name__,

            self.cache_id, self.cache_key, self.cache_active)

    def _cache_key_partition(self):

        prefix, repo_name, suffix = self.cache_key.partition(self.cache_args)

        return prefix, repo_name, suffix

    def get_prefix(self):

        """

        get prefix that might have been used in _get_cache_key to

        generate self.cache_key. Only used for informational purposes

        in repo_edit.html.

        """

        # prefix, repo_name, suffix

        return self._cache_key_partition()[0]

    def get_suffix(self):

        """

        get suffix that might have been used in _get_cache_key to

        generate self.cache_key. Only used for informational purposes

        in repo_edit.html.

        """

        # prefix, repo_name, suffix

        return self._cache_key_partition()[2]

    @classmethod

    def clear_cache(cls):

        """

        Delete all cache keys from database.

        Should only be run when all instances are down and all entries thus stale.

        """

        cls.query().delete()

        Session().commit()

    @classmethod

    def _get_cache_key(cls, key):

        """

        Wrapper for generating a unique cache key for this instance and "key".

        key must / will start with a repo_name which will be stored in .cache_args .

        """

        import kallithea

        prefix = kallithea.CONFIG.get('instance_id', '')

        return "%s%s" % (prefix, key)

    @classmethod

    def set_invalidate(cls, repo_name):

        """

        Mark all caches of a repo as invalid in the database.

        """

        inv_objs = Session().query(cls).filter(cls.cache_args == repo_name).all()

        log.debug('for repo %s got %s invalidation objects',

                  safe_str(repo_name), inv_objs)

        for inv_obj in inv_objs:

            log.debug('marking %s key for invalidation based on repo_name=%s',

                      inv_obj, safe_str(repo_name))

            Session().delete(inv_obj)

        Session().commit()

    @classmethod

    def test_and_set_valid(cls, repo_name, kind, valid_cache_keys=None):

        """

        Mark this cache key as active and currently cached.

        Return True if the existing cache registration still was valid.

        Return False to indicate that it had been invalidated and caches should be refreshed.

        """

        key = (repo_name + '_' + kind) if kind else repo_name

        cache_key = cls._get_cache_key(key)

        if valid_cache_keys and cache_key in valid_cache_keys:

            return True

        inv_obj = cls.query().filter(cls.cache_key == cache_key).scalar()

        if inv_obj is None:

            inv_obj = cls(cache_key, repo_name)

        elif inv_obj.cache_active:

            return True

        inv_obj.cache_active = True

        Session().add(inv_obj)

        try:

            Session().commit()

        except sqlalchemy.exc.IntegrityError:

            log.error('commit of CacheInvalidation failed - retrying')

            Session().rollback()

            inv_obj = cls.query().filter(cls.cache_key == cache_key).scalar()

            if inv_obj is None:

                log.error('failed to create CacheInvalidation entry')

                # TODO: fail badly?

            # else: TOCTOU - another thread added the key at the same time; no further action required

        return False

    @classmethod

    def get_valid_cache_keys(cls):

        """

        Return opaque object with information of which caches still are valid

        and can be used without checking for invalidation.

        """

        return set(inv_obj.cache_key for inv_obj in cls.query().filter(cls.cache_active).all())

class ChangesetComment(Base, BaseModel):

    __tablename__ = 'changeset_comments'

    __table_args__ = (

        Index('cc_revision_idx', 'revision'),

        Index('cc_pull_request_id_idx', 'pull_request_id'),

        _table_args_default_dict,

    )

    comment_id = Column(Integer(), unique=True, primary_key=True)

    repo_id = Column(Integer(), ForeignKey('repositories.repo_id'), nullable=False)

    revision = Column(String(40), nullable=True)

    pull_request_id = Column(Integer(), ForeignKey('pull_requests.pull_request_id'), nullable=True)

    line_no = Column(Unicode(10), nullable=True)

    f_path = Column(Unicode(1000), nullable=True)

    user_id = Column(Integer(), ForeignKey('users.user_id'), nullable=False)

    text = Column(UnicodeText(25000), nullable=False)

    created_on = Column(DateTime(timezone=False), nullable=False, default=datetime.datetime.now)

    modified_at = Column(DateTime(timezone=False), nullable=False, default=datetime.datetime.now)

    author = relationship('User')

    repo = relationship('Repository')

    # status_change is frequently used directly in templates - make it a lazy

    # join to avoid fetching each related ChangesetStatus on demand.

    # There will only be one ChangesetStatus referencing each comment so the join will not explode.

    status_change = relationship('ChangesetStatus',

                                 cascade="all, delete-orphan", lazy='joined')

    pull_request = relationship('PullRequest')

    @classmethod

    def get_users(cls, revision=None, pull_request_id=None):

        """

        Returns user associated with this ChangesetComment. ie those

        who actually commented

        :param cls:

        :param revision:

        """

        q = Session().query(User) \

                .join(ChangesetComment.author)

        if revision is not None:

            q = q.filter(cls.revision == revision)

        elif pull_request_id is not None:

            q = q.filter(cls.pull_request_id == pull_request_id)

        return q.all()

    def url(self):

        anchor = "comment-%s" % self.comment_id

        import kallithea.lib.helpers as h

        if self.revision:

            return h.url('changeset_home', repo_name=self.repo.repo_name, revision=self.revision, anchor=anchor)

        elif self.pull_request_id is not None:

            return self.pull_request.url(anchor=anchor)

    def deletable(self):

        return self.created_on > datetime.datetime.now() - datetime.timedelta(minutes=5)

class ChangesetStatus(Base, BaseModel):

    __tablename__ = 'changeset_statuses'

    __table_args__ = (

        Index('cs_revision_idx', 'revision'),

        Index('cs_version_idx', 'version'),

        Index('cs_pull_request_id_idx', 'pull_request_id'),

        Index('cs_changeset_comment_id_idx', 'changeset_comment_id'),

        Index('cs_pull_request_id_user_id_version_idx', 'pull_request_id', 'user_id', 'version'),

        Index('cs_repo_id_pull_request_id_idx', 'repo_id', 'pull_request_id'),

        UniqueConstraint('repo_id', 'revision', 'version'),

        _table_args_default_dict,

    )

    STATUS_NOT_REVIEWED = DEFAULT = 'not_reviewed'

    STATUS_APPROVED = 'approved'

    STATUS_REJECTED = 'rejected'

    STATUS_UNDER_REVIEW = 'under_review'

    STATUSES = [

        (STATUS_NOT_REVIEWED, _("Not reviewed")),  # (no icon) and default

        (STATUS_APPROVED, _("Approved")),

        (STATUS_REJECTED, _("Rejected")),

        (STATUS_UNDER_REVIEW, _("Under review")),

    ]

    changeset_status_id = Column(Integer(), unique=True, primary_key=True)

    repo_id = Column(Integer(), ForeignKey('repositories.repo_id'), nullable=False)

    user_id = Column(Integer(), ForeignKey('users.user_id'), nullable=False)

    revision = Column(String(40), nullable=True)

    status = Column(String(128), nullable=False, default=DEFAULT)

    changeset_comment_id = Column(Integer(), ForeignKey('changeset_comments.comment_id'), nullable=False)

    modified_at = Column(DateTime(), nullable=False, default=datetime.datetime.now)

    version = Column(Integer(), nullable=False, default=0)

    pull_request_id = Column(Integer(), ForeignKey('pull_requests.pull_request_id'), nullable=True)

    author = relationship('User')

    repo = relationship('Repository')

    comment = relationship('ChangesetComment')

    pull_request = relationship('PullRequest')

    def __unicode__(self):

        return u"<%s('%s:%s')>" % (

            self.__class__.__name__,

            self.status, self.author

        )

    @classmethod

    def get_status_lbl(cls, value):

    @property

    def status_lbl(self):

        return ChangesetStatus.get_status_lbl(self.status)

class PullRequest(Base, BaseModel):

    __tablename__ = 'pull_requests'

    __table_args__ = (

        Index('pr_org_repo_id_idx', 'org_repo_id'),

        Index('pr_other_repo_id_idx', 'other_repo_id'),

        _table_args_default_dict,

    )

    # values for .status

    STATUS_NEW = u'new'

    STATUS_CLOSED = u'closed'

    pull_request_id = Column(Integer(), unique=True, primary_key=True)

    title = Column(Unicode(255), nullable=False)

    description = Column(UnicodeText(10240), nullable=False)

    status = Column(Unicode(255), nullable=False, default=STATUS_NEW) # only for closedness, not approve/reject/etc

    created_on = Column(DateTime(timezone=False), nullable=False, default=datetime.datetime.now)

    updated_on = Column(DateTime(timezone=False), nullable=False, default=datetime.datetime.now)

    user_id = Column(Integer(), ForeignKey('users.user_id'), nullable=False)

    _revisions = Column('revisions', UnicodeText(20500), nullable=False)  # 500 revisions max

    org_repo_id = Column(Integer(), ForeignKey('repositories.repo_id'), nullable=False)

    org_ref = Column(Unicode(255), nullable=False)

    other_repo_id = Column(Integer(), ForeignKey('repositories.repo_id'), nullable=False)

    other_ref = Column(Unicode(255), nullable=False)

    @hybrid_property

    def revisions(self):

        return self._revisions.split(':')

    @revisions.setter

    def revisions(self, val):

        self._revisions = safe_unicode(':'.join(val))

    @property

    def org_ref_parts(self):

        return self.org_ref.split(':')

    @property

    def other_ref_parts(self):

        return self.other_ref.split(':')

    owner = relationship('User')

    reviewers = relationship('PullRequestReviewers',

                             cascade="all, delete-orphan")

    org_repo = relationship('Repository', primaryjoin='PullRequest.org_repo_id==Repository.repo_id')

    other_repo = relationship('Repository', primaryjoin='PullRequest.other_repo_id==Repository.repo_id')

    statuses = relationship('ChangesetStatus', order_by='ChangesetStatus.changeset_status_id')

    comments = relationship('ChangesetComment', order_by='ChangesetComment.comment_id',

                             cascade="all, delete-orphan")

    def get_reviewer_users(self):

        """Like .reviewers, but actually returning the users"""

        return User.query() \

            .join(PullRequestReviewers) \

            .filter(PullRequestReviewers.pull_request == self) \

            .order_by(PullRequestReviewers.pull_requests_reviewers_id) \

            .all()

    def is_closed(self):

        return self.status == self.STATUS_CLOSED

    def user_review_status(self, user_id):

        """Return the user's latest status votes on PR"""

        # note: no filtering on repo - that would be redundant

        status = ChangesetStatus.query() \

            .filter(ChangesetStatus.pull_request == self) \

            .filter(ChangesetStatus.user_id == user_id) \

            .order_by(ChangesetStatus.version) \

            .first()

        return str(status.status) if status else ''

    @classmethod

    def make_nice_id(cls, pull_request_id):

        '''Return pull request id nicely formatted for displaying'''

        return '#%s' % pull_request_id

    def nice_id(self):

        '''Return the id of this pull request, nicely formatted for displaying'''

        return self.make_nice_id(self.pull_request_id)

    def __json__(self):

        return dict(

            revisions=self.revisions

        )

    def url(self, **kwargs):

        canonical = kwargs.pop('canonical', None)

        import kallithea.lib.helpers as h

        b = self.org_ref_parts[1]

        if b != self.other_ref_parts[1]:

            s = '/_/' + b

        else:

            s = '/_/' + self.title

        kwargs['extra'] = urlreadable(s)

        if canonical:

            return h.canonical_url('pullrequest_show', repo_name=self.other_repo.repo_name,

                                   pull_request_id=self.pull_request_id, **kwargs)

        return h.url('pullrequest_show', repo_name=self.other_repo.repo_name,

                     pull_request_id=self.pull_request_id, **kwargs)

class PullRequestReviewers(Base, BaseModel):

    __tablename__ = 'pull_request_reviewers'

    __table_args__ = (

        Index('pull_request_reviewers_user_id_idx', 'user_id'),

        _table_args_default_dict,

    )

    def __init__(self, user=None, pull_request=None):

        self.user = user

        self.pull_request = pull_request

    pull_requests_reviewers_id = Column(Integer(), unique=True, primary_key=True)

    pull_request_id = Column(Integer(), ForeignKey('pull_requests.pull_request_id'), nullable=False)

    user_id = Column(Integer(), ForeignKey('users.user_id'), nullable=False)

    user = relationship('User')

    pull_request = relationship('PullRequest')

class Notification(Base, BaseModel):

    __tablename__ = 'notifications'

    __table_args__ = (

        Index('notification_type_idx', 'type'),

        _table_args_default_dict,

    )

    TYPE_CHANGESET_COMMENT = u'cs_comment'

    TYPE_MESSAGE = u'message'

    TYPE_MENTION = u'mention'

    TYPE_REGISTRATION = u'registration'

    TYPE_PULL_REQUEST = u'pull_request'

    TYPE_PULL_REQUEST_COMMENT = u'pull_request_comment'

    notification_id = Column(Integer(), unique=True, primary_key=True)

    subject = Column(Unicode(512), nullable=False)

    body = Column(UnicodeText(50000), nullable=False)

    created_by = Column(Integer(), ForeignKey('users.user_id'), nullable=False)

    created_on = Column(DateTime(timezone=False), nullable=False, default=datetime.datetime.now)

    type_ = Column('type', Unicode(255), nullable=False)

    created_by_user = relationship('User')

    notifications_to_users = relationship('UserNotification', cascade="all, delete-orphan")

    @property

    def recipients(self):

        return [x.user for x in UserNotification.query()

                .filter(UserNotification.notification == self)

                .order_by(UserNotification.user_id.asc()).all()]

    @classmethod

    def create(cls, created_by, subject, body, recipients, type_=None):

        if type_ is None:

            type_ = Notification.TYPE_MESSAGE

        notification = cls()

        notification.created_by_user = created_by

        notification.subject = subject

        notification.body = body

        notification.type_ = type_

        notification.created_on = datetime.datetime.now()

        for recipient in recipients:

            un = UserNotification()

            un.notification = notification

            un.user_id = recipient.user_id

            # Mark notifications to self "pre-read" - should perhaps just be skipped

            if recipient == created_by:

                un.read = True

            Session().add(un)

        Session().add(notification)

        Session().flush() # assign notificaiton.notification_id

        return notification

    @property

    def description(self):

        from kallithea.model.notification import NotificationModel

        return NotificationModel().make_description(self)

class UserNotification(Base, BaseModel):

    __tablename__ = 'user_to_notification'

    __table_args__ = (

        UniqueConstraint('user_id', 'notification_id'),

        _table_args_default_dict,

    )