except UnicodeEncodeError:

            pass

    try:

        import chardet

        encoding = chardet.detect(unicode_)['encoding']

        if encoding is None:

            raise UnicodeEncodeError()

        return unicode_.encode(encoding)

    except (ImportError, UnicodeEncodeError):

        return unicode_.encode(to_encoding[0], 'replace')

def remove_suffix(s, suffix):

    if s.endswith(suffix):

        s = s[:-1 * len(suffix)]

    return s

def remove_prefix(s, prefix):

    if s.startswith(prefix):

        s = s[len(prefix):]

    return s

def engine_from_config(configuration, prefix='sqlalchemy.', **kwargs):

    """

    Custom engine_from_config functions that makes sure we use NullPool for

    file based sqlite databases. This prevents errors on sqlite. This only

    applies to sqlalchemy versions < 0.7.0

    """

    import sqlalchemy

    from sqlalchemy import engine_from_config as efc

    import logging

    if int(sqlalchemy.__version__.split('.')[1]) < 7:

        # This solution should work for sqlalchemy < 0.7.0, and should use

        # proxy=TimerProxy() for execution time profiling

        from sqlalchemy.pool import NullPool

        url = configuration[prefix + 'url']

        if url.startswith('sqlite'):

            kwargs.update({'poolclass': NullPool})

        return efc(configuration, prefix, **kwargs)

    else:

        import time

        from sqlalchemy import event

        log = logging.getLogger('sqlalchemy.engine')

        BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = xrange(30, 38)

        engine = efc(configuration, prefix, **kwargs)

        def color_sql(sql):

            COLOR_SEQ = "\033[1;%dm"

            COLOR_SQL = YELLOW

            normal = '\x1b[0m'

            return ''.join([COLOR_SEQ % COLOR_SQL, sql, normal])

        if configuration['debug']:

            #attach events only for debug configuration

            def before_cursor_execute(conn, cursor, statement,

                                    parameters, context, executemany):

                context._query_start_time = time.time()

                log.info(color_sql(">>>>> STARTING QUERY >>>>>"))

            def after_cursor_execute(conn, cursor, statement,

                                    parameters, context, executemany):

                total = time.time() - context._query_start_time

                log.info(color_sql("<<<<< TOTAL TIME: %f <<<<<" % total))

            event.listen(engine, "before_cursor_execute",

                         before_cursor_execute)

            event.listen(engine, "after_cursor_execute",

                         after_cursor_execute)

    return engine

def age(prevdate, show_short_version=False, now=None):

    """

    turns a datetime into an age string.

    If show_short_version is True, then it will generate a not so accurate but shorter string,

    example: 2days ago, instead of 2 days and 23 hours ago.

    :param prevdate: datetime object

    :param show_short_version: if it should approximate the date and return a shorter string

    :rtype: unicode

    :returns: unicode words describing age

    """

    now = now or datetime.datetime.now()

    order = ['year', 'month', 'day', 'hour', 'minute', 'second']

    deltas = {}

    future = False

    if prevdate > now:

        now, prevdate = prevdate, now

        future = True

    if future:

        prevdate = prevdate.replace(microsecond=0)

    # Get date parts deltas

    from dateutil import relativedelta

    for part in order:

        d = relativedelta.relativedelta(now, prevdate)

        deltas[part] = getattr(d, part + 's')

    # Fix negative offsets (there is 1 second between 10:59:59 and 11:00:00,

    # not 1 hour, -59 minutes and -59 seconds)

    for num, length in [(5, 60), (4, 60), (3, 24)]:  # seconds, minutes, hours

        part = order[num]

        carry_part = order[num - 1]

        if deltas[part] < 0:

            deltas[part] += length

            deltas[carry_part] -= 1

    # Same thing for days except that the increment depends on the (variable)

    # number of days in the month

    month_lengths = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]

    if deltas['day'] < 0:

        if prevdate.month == 2 and (prevdate.year % 4 == 0 and

            (prevdate.year % 100 != 0 or prevdate.year % 400 == 0)):

            deltas['day'] += 29

        else:

            deltas['day'] += month_lengths[prevdate.month - 1]

        deltas['month'] -= 1

    if deltas['month'] < 0:

        deltas['month'] += 12

        deltas['year'] -= 1

    # In short version, we want nicer handling of ages of more than a year

    if show_short_version:

        if deltas['year'] == 1:

            # ages between 1 and 2 years: show as months

            deltas['month'] += 12

            deltas['year'] = 0

        if deltas['year'] >= 2:

            # ages 2+ years: round

            if deltas['month'] > 6:

                deltas['year'] += 1

                deltas['month'] = 0

    # Format the result

    fmt_funcs = {

        'year': lambda d: ungettext(u'%d year', '%d years', d) % d,

        'month': lambda d: ungettext(u'%d month', '%d months', d) % d,

        'day': lambda d: ungettext(u'%d day', '%d days', d) % d,

        'hour': lambda d: ungettext(u'%d hour', '%d hours', d) % d,

        'minute': lambda d: ungettext(u'%d minute', '%d minutes', d) % d,

        'second': lambda d: ungettext(u'%d second', '%d seconds', d) % d,

    }

    for i, part in enumerate(order):

        value = deltas[part]

        if value == 0:

            continue

        if i < 5:

            sub_part = order[i + 1]

            sub_value = deltas[sub_part]

        else:

            sub_value = 0

        if sub_value == 0 or show_short_version:

            if future:

                return _('in %s') % fmt_funcs[part](value)

            else:

                return _('%s ago') % fmt_funcs[part](value)

        if future:

            return _('in %s and %s') % (fmt_funcs[part](value),

                fmt_funcs[sub_part](sub_value))

        else:

            return _('%s and %s ago') % (fmt_funcs[part](value),

                fmt_funcs[sub_part](sub_value))

    return _('just now')

def uri_filter(uri):

    """

    Removes user:password from given url string

    :param uri:

    :rtype: unicode

    :returns: filtered list of strings

    """

    if not uri:

        return ''

    proto = ''

    for pat in ('https://', 'http://', 'git://'):

        if uri.startswith(pat):

            uri = uri[len(pat):]

            proto = pat

            break

    # remove passwords and username

    uri = uri[uri.find('@') + 1:]

    # get the port

    cred_pos = uri.find(':')

    if cred_pos == -1:

        host, port = uri, None

    else:

        host, port = uri[:cred_pos], uri[cred_pos + 1:]

    return filter(None, [proto, host, port])

def credentials_filter(uri):

    """

    Returns a url with removed credentials

    :param uri:

    """

    uri = uri_filter(uri)

    #check if we have port

    if len(uri) > 2 and uri[2]:

        uri[2] = ':' + uri[2]

    return ''.join(uri)

def get_clone_url(uri_tmpl, qualified_home_url, repo_name, repo_id, **override):

    parsed_url = urlobject.URLObject(qualified_home_url)

    decoded_path = safe_unicode(urllib.unquote(parsed_url.path.rstrip('/')))

    args = {

        'scheme': parsed_url.scheme,

        'user': '',

        'netloc': parsed_url.netloc+decoded_path,  # path if we use proxy-prefix

        'prefix': decoded_path,

        'repo': repo_name,

        'repoid': str(repo_id)

    }

    args.update(override)

    args['user'] = urllib.quote(safe_str(args['user']))

    for k, v in args.items():

        uri_tmpl = uri_tmpl.replace('{%s}' % k, v)

    # remove leading @ sign if it's present. Case of empty user

    url_obj = urlobject.URLObject(uri_tmpl)

    url = url_obj.with_netloc(url_obj.netloc.lstrip('@'))

    return safe_unicode(url)

def get_changeset_safe(repo, rev):

    """

    Safe version of get_changeset if this changeset doesn't exists for a

    repo it returns a Dummy one instead

    :param repo:

    :param rev:

    """

    from kallithea.lib.vcs.backends.base import BaseRepository

    from kallithea.lib.vcs.exceptions import RepositoryError

    from kallithea.lib.vcs.backends.base import EmptyChangeset

    if not isinstance(repo, BaseRepository):

        raise Exception('You must pass an Repository '

                        'object as first argument got %s', type(repo))

    try:

        cs = repo.get_changeset(rev)

    except (RepositoryError, LookupError):

        cs = EmptyChangeset(requested_revision=rev)

    return cs

def datetime_to_time(dt):

    if dt:

        return time.mktime(dt.timetuple())

def time_to_datetime(tm):

    if tm:

        if isinstance(tm, basestring):

            try:

                tm = float(tm)

            except ValueError:

                return

        return datetime.datetime.fromtimestamp(tm)

# Must match regexp in kallithea/public/js/base.js MentionsAutoComplete()

# Check char before @ - it must not look like we are in an email addresses.

# Matching is greedy so we don't have to look beyond the end.

MENTIONS_REGEX = re.compile(r'(?:^|(?<=[^a-zA-Z0-9]))@([a-zA-Z0-9][-_.a-zA-Z0-9]*[a-zA-Z0-9])')

def extract_mentioned_usernames(text):

    r"""

    Returns list of (possible) usernames @mentioned in given text.

    >>> extract_mentioned_usernames('@1-2.a_X,@1234 not@not @ddd@not @n @ee @ff @gg, @gg;@hh @n\n@zz,')

    ['1-2.a_X', '1234', 'ddd', 'ee', 'ff', 'gg', 'hh', 'zz']

    """

    return MENTIONS_REGEX.findall(text)

def extract_mentioned_users(text):

    """ Returns set of actual database Users @mentioned in given text. """

    from kallithea.model.db import User

    result = set()

    for name in extract_mentioned_usernames(text):

        user = User.get_by_username(name, case_insensitive=True)

        if user is not None and user.username != User.DEFAULT_USER:

            result.add(user)

    return result

class AttributeDict(dict):

    def __getattr__(self, attr):

        return self.get(attr, None)

    __setattr__ = dict.__setitem__

    __delattr__ = dict.__delitem__

def fix_PATH(os_=None):

    """

    Get current active python path, and append it to PATH variable to fix issues

    of subprocess calls and different python versions

    """

    if os_ is None:

        import os

    else:

        os = os_

    cur_path = os.path.split(sys.executable)[0]

    if not os.environ['PATH'].startswith(cur_path):

        os.environ['PATH'] = '%s:%s' % (cur_path, os.environ['PATH'])

def obfuscate_url_pw(engine):

    from sqlalchemy.engine import url as sa_url

    from sqlalchemy.exc import ArgumentError

    try:

        _url = sa_url.make_url(engine or '')

    except ArgumentError:

        return engine

    if _url.password:

        _url.password = 'XXXXX'

    return str(_url)

def get_server_url(environ):

    req = webob.Request(environ)

    return req.host_url + req.script_name

def _extract_extras(env=None):

    """

    Extracts the Kallithea extras data from os.environ, and wraps it into named

    AttributeDict object

    """

    if not env:

        env = os.environ

    try:

        extras = json.loads(env['KALLITHEA_EXTRAS'])

    except KeyError:

        extras = {}

    try:

        for k in ['username', 'repository', 'locked_by', 'scm', 'make_lock',

                  'action', 'ip']:

            extras[k]

    except KeyError as e:

        raise Exception('Missing key %s in os.environ %s' % (e, extras))

    return AttributeDict(extras)

def _set_extras(extras):

    # RC_SCM_DATA can probably be removed in the future, but for compatibility now...

    os.environ['KALLITHEA_EXTRAS'] = os.environ['RC_SCM_DATA'] = json.dumps(extras)

def get_current_authuser():

    """

    Gets kallithea user from threadlocal tmpl_context variable if it's

    defined, else returns None.

    """

    from pylons import tmpl_context

    if hasattr(tmpl_context, 'authuser'):

        return tmpl_context.authuser

    return None

class OptionalAttr(object):

    """

    Special Optional Option that defines other attribute. Example::

        def test(apiuser, userid=Optional(OAttr('apiuser')):

            user = Optional.extract(userid)

            # calls

    """

    def __init__(self, attr_name):

        self.attr_name = attr_name

    def __repr__(self):

        return '<OptionalAttr:%s>' % self.attr_name

    def __call__(self):

        return self

#alias

OAttr = OptionalAttr

class Optional(object):

    """

    Defines an optional parameter::

        param = param.getval() if isinstance(param, Optional) else param

        param = param() if isinstance(param, Optional) else param

    is equivalent of::

        param = Optional.extract(param)

    """

    def __init__(self, type_):

        self.type_ = type_

    def __repr__(self):

        return '<Optional:%s>' % self.type_.__repr__()

    def __call__(self):

        return self.getval()

    def getval(self):

        """

        returns value from this Optional instance

        """

        if isinstance(self.type_, OAttr):

            # use params name

            return self.type_.attr_name

        return self.type_

    @classmethod

    def extract(cls, val):

        """

        Extracts value from Optional() instance

        :param val:

        :return: original value if it's not Optional instance else

            value of instance

        """

        if isinstance(val, cls):

            return val.getval()

        return val

def urlreadable(s, _cleanstringsub=re.compile('[^-a-zA-Z0-9./]+').sub):

    return _cleanstringsub('_', safe_str(s)).rstrip('_')

# -*- coding: utf-8 -*-

# This program is free software: you can redistribute it and/or modify

# it under the terms of the GNU General Public License as published by

# the Free Software Foundation, either version 3 of the License, or

# (at your option) any later version.

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU General Public License for more details.

#

# You should have received a copy of the GNU General Public License

# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""

kallithea.model.gist

~~~~~~~~~~~~~~~~~~~~

gist model for Kallithea

This file was forked by the Kallithea project in July 2014.

Original author and date, and relevant copyright and licensing information is below:

:created_on: May 9, 2013

:author: marcink

:copyright: (c) 2013 RhodeCode GmbH, and others.

:license: GPLv3, see LICENSE.md for more details.

"""

import os

import time

import logging

import traceback

import shutil

    time_to_datetime, AttributeDict

from kallithea.lib.compat import json

from kallithea.model.base import BaseModel

from kallithea.model.db import Gist, User

from kallithea.model.repo import RepoModel

from kallithea.model.scm import ScmModel

log = logging.getLogger(__name__)

GIST_STORE_LOC = '.rc_gist_store'

GIST_METADATA_FILE = '.rc_gist_metadata'

class GistModel(BaseModel):

    def __delete_gist(self, gist):

        """

        removes gist from filesystem

        :param gist: gist object

        """

        root_path = RepoModel().repos_path

        rm_path = os.path.join(root_path, GIST_STORE_LOC, gist.gist_access_id)

        log.info("Removing %s", rm_path)

        shutil.rmtree(rm_path)

    def _store_metadata(self, repo, gist_id, gist_access_id, user_id, gist_type,

                        gist_expires):

        """

        store metadata inside the gist, this can be later used for imports

        or gist identification

        """

        metadata = {

            'metadata_version': '1',

            'gist_db_id': gist_id,

            'gist_access_id': gist_access_id,

            'gist_owner_id': user_id,

            'gist_type': gist_type,

            'gist_expires': gist_expires,

            'gist_updated': time.time(),

        }

        with open(os.path.join(repo.path, '.hg', GIST_METADATA_FILE), 'wb') as f:

            f.write(json.dumps(metadata))

    def get_gist(self, gist):

        return Gist.guess_instance(gist)

    def get_gist_files(self, gist_access_id, revision=None):

        """

        Get files for given gist

        :param gist_access_id:

        """

        repo = Gist.get_by_access_id(gist_access_id)

        cs = repo.scm_instance.get_changeset(revision)

        return cs, [n for n in cs.get_node('/')]

    def create(self, description, owner, gist_mapping,

               gist_type=Gist.GIST_PUBLIC, lifetime=-1):

        """

        :param description: description of the gist

        :param owner: user who created this gist

        :param gist_mapping: mapping {filename:{'content':content},...}

        :param gist_type: type of gist private/public

        :param lifetime: in minutes, -1 == forever

        """

        owner = User.guess_instance(owner)

        lifetime = safe_int(lifetime, -1)

        gist_expires = time.time() + (lifetime * 60) if lifetime != -1 else -1

        log.debug('set GIST expiration date to: %s',

                  time_to_datetime(gist_expires)

                   if gist_expires != -1 else 'forever')

        #create the Database version

        gist = Gist()

        gist.gist_description = description

        gist.gist_access_id = gist_id

        gist.owner_id = owner.user_id

        gist.gist_expires = gist_expires

        gist.gist_type = safe_unicode(gist_type)

        self.sa.add(gist)

        self.sa.flush() # make database assign gist.gist_id

        if gist_type == Gist.GIST_PUBLIC:

            # use DB ID for easy to use GIST ID

            gist_id = safe_unicode(gist.gist_id)

            gist.gist_access_id = gist_id

            self.sa.add(gist)

        gist_repo_path = os.path.join(GIST_STORE_LOC, gist_id)

        log.debug('Creating new %s GIST repo in %s', gist_type, gist_repo_path)

        repo = RepoModel()._create_filesystem_repo(

            repo_name=gist_id, repo_type='hg', repo_group=GIST_STORE_LOC)

        processed_mapping = {}

        for filename in gist_mapping:

            if filename != os.path.basename(filename):

                raise Exception('Filename cannot be inside a directory')

            content = gist_mapping[filename]['content']

            #TODO: expand support for setting explicit lexers

#             if lexer is None:

#                 try:

#                     guess_lexer = pygments.lexers.guess_lexer_for_filename

#                     lexer = guess_lexer(filename,content)

#                 except pygments.util.ClassNotFound:

#                     lexer = 'text'

            processed_mapping[filename] = {'content': content}

        # now create single multifile commit

        message = 'added file'

        message += 's: ' if len(processed_mapping) > 1 else ': '

        message += ', '.join([x for x in processed_mapping])

        #fake Kallithea Repository object

        fake_repo = AttributeDict(dict(

            repo_name=gist_repo_path,

            scm_instance_no_cache=lambda: repo,

        ))

        ScmModel().create_nodes(

            user=owner.user_id, repo=fake_repo,

            message=message,

            nodes=processed_mapping,

            trigger_push_hook=False

        )

        self._store_metadata(repo, gist.gist_id, gist.gist_access_id,

                             owner.user_id, gist.gist_type, gist.gist_expires)

        return gist

    def delete(self, gist, fs_remove=True):

        gist = Gist.guess_instance(gist)

        try:

            self.sa.delete(gist)

            if fs_remove:

                self.__delete_gist(gist)

            else:

                log.debug('skipping removal from filesystem')

        except Exception:

            log.error(traceback.format_exc())

            raise

    def update(self, gist, description, owner, gist_mapping, gist_type,

               lifetime):

        gist = Gist.guess_instance(gist)

        gist_repo = gist.scm_instance

        lifetime = safe_int(lifetime, -1)

        if lifetime == 0:  # preserve old value

            gist_expires = gist.gist_expires

        else:

            gist_expires = time.time() + (lifetime * 60) if lifetime != -1 else -1

        #calculate operation type based on given data

        gist_mapping_op = {}

        for k, v in gist_mapping.items():

            # add, mod, del

            if not v['org_filename'] and v['filename']:

                op = 'add'

            elif v['org_filename'] and not v['filename']:

                op = 'del'

            else:

                op = 'mod'

            v['op'] = op

            gist_mapping_op[k] = v

        gist.gist_description = description

        gist.gist_expires = gist_expires

        gist.owner = owner

        gist.gist_type = gist_type

        message = 'updated file'

        message += 's: ' if len(gist_mapping) > 1 else ': '

        message += ', '.join([x for x in gist_mapping])

        #fake Kallithea Repository object

        fake_repo = AttributeDict(dict(

            repo_name=gist_repo.path,

            scm_instance_no_cache=lambda: gist_repo,

        ))

        self._store_metadata(gist_repo, gist.gist_id, gist.gist_access_id,

                             owner.user_id, gist.gist_type, gist.gist_expires)

        ScmModel().update_nodes(

            user=owner.user_id,

            repo=fake_repo,

            message=message,

            nodes=gist_mapping_op,

            trigger_push_hook=False

        )

        return gist