# get our arglist and check if we provided them as args

        for arg, default in func_kwargs.items():

            if arg == USER_SESSION_ATTR:

                # USER_SESSION_ATTR is something translated from API key and

                # this is checked before so we don't need validate it

                continue

            # skip the required param check if it's default value is

            # NotImplementedType (default_empty)

            if default == default_empty and arg not in self._request_params:

                raise JSONRPCErrorResponse(

                    retid=self._req_id,

                    message='Missing non optional `%s` arg in JSON DATA' % arg,

                )

        extra = set(self._request_params).difference(func_kwargs)

        if extra:

            raise JSONRPCErrorResponse(

                retid=self._req_id,

                message='Unknown %s arg in JSON DATA' %

                        ', '.join('`%s`' % arg for arg in extra),

            )

        self._rpc_args = {}

        self._rpc_args.update(self._request_params)

        self._rpc_args['action'] = self._req_method

        self._rpc_args['environ'] = environ

        log.info('IP: %s Request to %s time: %.3fs' % (

            self._get_ip_addr(environ),

            get_path_info(environ), time.time() - start)

        )

        state.set_action(self._rpc_call, [])

        state.set_params(self._rpc_args)

        return state

    def _rpc_call(self, action, environ, **rpc_args):

        """

        Call the specified RPC Method

        """

        raw_response = ''

        try:

            raw_response = getattr(self, action)(**rpc_args)

            if isinstance(raw_response, HTTPError):

                self._error = str(raw_response)

        except JSONRPCError as e:

        except Exception as e:

            log.error('Encountered unhandled exception: %s',

                      traceback.format_exc(),)

            json_exc = JSONRPCError('Internal server error')

        if self._error is not None:

            raw_response = None

        response = dict(id=self._req_id, result=raw_response, error=self._error)

        try:

            return ascii_bytes(ext_json.dumps(response))

        except TypeError as e:

            log.error('API FAILED. Error encoding response for %s %s: %s\n%s', action, rpc_args, e, traceback.format_exc())

            return ascii_bytes(ext_json.dumps(

                dict(

                    id=self._req_id,

                    result=None,

                    error="Error encoding response",

                )

            ))

    def _find_method(self):

        """

        Return method named by `self._req_method` in controller if able

        """

        log.debug('Trying to find JSON-RPC method: %s', self._req_method)

        if self._req_method.startswith('_'):

            raise AttributeError("Method not allowed")

        try:

            func = getattr(self, self._req_method, None)

        except UnicodeEncodeError:

            raise AttributeError("Problem decoding unicode in requested "

                                 "method name.")

        if isinstance(func, types.MethodType):

            return func

        else:

            raise AttributeError("No such method: %s" % (self._req_method,))

    def failed(self):

        return False

    traceback = None # if failed

    task_id = None

def task(f_org):

    """Wrapper of celery.task.task, running async if CELERY_ON

    """

    if CELERY_ON:

        def f_async(*args, **kwargs):

            log.info('executing %s task', f_org.__name__)

            try:

                f_org(*args, **kwargs)

            finally:

                log.info('executed %s task', f_org.__name__)

        f_async.__name__ = f_org.__name__

        from kallithea.lib import celerypylons

        runner = celerypylons.task(ignore_result=True)(f_async)

        def f_wrapped(*args, **kwargs):

            t = runner.apply_async(args=args, kwargs=kwargs)

            log.info('executing task %s in async mode - id %s', f_org, t.task_id)

            return t

    else:

        def f_wrapped(*args, **kwargs):

            log.info('executing task %s in sync', f_org.__name__)

            try:

                result = f_org(*args, **kwargs)

            except Exception as e:

                log.error('exception executing sync task %s in sync: %r', f_org.__name__, e)

                raise # TODO: return this in FakeTask as with async tasks?

            return FakeTask(result)

    return f_wrapped

def __get_lockkey(func, *fargs, **fkwargs):

    params = list(fargs)

    params.extend(['%s-%s' % ar for ar in fkwargs.items()])

    func_name = str(func.__name__) if hasattr(func, '__name__') else str(func)

    lockkey = 'task_%s.lock' % \

    return lockkey

def locked_task(func):

    def __wrapper(func, *fargs, **fkwargs):

        lockkey = __get_lockkey(func, *fargs, **fkwargs)

        lockkey_path = config.get('cache_dir') or config['app_conf']['cache_dir']  # Backward compatibility for TurboGears < 2.4

        log.info('running task with lockkey %s', lockkey)

        try:

            l = DaemonLock(os.path.join(lockkey_path, lockkey))

            ret = func(*fargs, **fkwargs)

            l.release()

            return ret

        except LockHeld:

            log.info('LockHeld')

            return 'Task with key %s already running' % lockkey

    return decorator(__wrapper, func)

def get_session():

    sa = meta.Session()

    return sa

def dbsession(func):

    def __wrapper(func, *fargs, **fkwargs):

        try:

            ret = func(*fargs, **fkwargs)

            return ret

        finally:

            if CELERY_ON and not CELERY_EAGER:

                meta.Session.remove()

    return decorator(__wrapper, func)

class _Message(object):

    """A message returned by ``pop_flash_messages()``.

    Converting the message to a string returns the message text. Instances

    also have the following attributes:

    * ``category``: the category specified when the message was created.

    * ``message``: the html-safe message text.

    """

    def __init__(self, category, message):

        self.category = category

        self.message = message

def _session_flash_messages(append=None, clear=False):

    """Manage a message queue in tg.session: return the current message queue

    after appending the given message, and possibly clearing the queue."""

    key = 'flash'

    from tg import session

    if key in session:

        flash_messages = session[key]

    else:

        if append is None:  # common fast path - also used for clearing empty queue

            return []  # don't bother saving

        flash_messages = []

        session[key] = flash_messages

    if append is not None and append not in flash_messages:

        flash_messages.append(append)

    if clear:

        session.pop(key, None)

    session.save()

    return flash_messages

def flash(message, category, logf=None):

    """

    Show a message to the user _and_ log it through the specified function

    category: notice (default), warning, error, success

    logf: a custom log function - such as log.debug

    logf defaults to log.info, unless category equals 'success', in which

    case logf defaults to log.debug.

    """

    assert category in ('error', 'success', 'warning'), category

    if hasattr(message, '__html__'):

        # render to HTML for storing in cookie

    else:

        # Apply str - the message might be an exception with __str__

        # Escape, so we can trust the result without further escaping, without any risk of injection

    if logf is None:

        logf = log.info

        if category == 'success':

            logf = log.debug

    logf('Flash %s: %s', category, safe_message)

    _session_flash_messages(append=(category, safe_message))

def pop_flash_messages():

    """Return all accumulated messages and delete them from the session.

    The return value is a list of ``Message`` objects.

    """

    return [_Message(category, message) for category, message in _session_flash_messages(clear=True)]

age = lambda x, y=False: _age(x, y)

capitalize = lambda x: x.capitalize()

email = author_email

short_id = lambda x: x[:12]

hide_credentials = lambda x: ''.join(credentials_filter(x))

def show_id(cs):

    """

    Configurable function that shows ID

    by default it's r123:fffeeefffeee

    :param cs: changeset instance

    """

    from kallithea import CONFIG

    def_len = safe_int(CONFIG.get('show_sha_length', 12))

    show_rev = str2bool(CONFIG.get('show_revision_number', False))

    raw_id = cs.raw_id[:def_len]

    if show_rev:

        return 'r%s:%s' % (cs.revision, raw_id)

    else:

        return raw_id

def fmt_date(date):

    if date:

        return date.strftime("%Y-%m-%d %H:%M:%S")

    return ""

# -*- coding: utf-8 -*-

import json

import urllib.parse

import urllib.request

class RecaptchaResponse(object):

    def __init__(self, is_valid, error_code=None):

        self.is_valid = is_valid

        self.error_code = error_code

    def __repr__(self):

        return '<RecaptchaResponse:%s>' % (self.is_valid)

def submit(g_recaptcha_response, private_key, remoteip):

    """

    Submits a reCAPTCHA request for verification. Returns RecaptchaResponse for the request

    g_recaptcha_response -- The value of g_recaptcha_response from the form

    private_key -- your reCAPTCHA private key

    remoteip -- the user's IP address

    """

    if not (g_recaptcha_response and len(g_recaptcha_response)):

        return RecaptchaResponse(is_valid=False, error_code='incorrect-captcha-sol')

    def encode_if_necessary(s):

            return s.encode('utf-8')

        return s

    params = urllib.parse.urlencode({

        'secret': encode_if_necessary(private_key),

        'remoteip': encode_if_necessary(remoteip),

        'response': encode_if_necessary(g_recaptcha_response),

    }).encode('ascii')

    req = urllib.request.Request(

        url="https://www.google.com/recaptcha/api/siteverify",

        data=params,

        headers={

            "Content-type": "application/x-www-form-urlencoded",

            "User-agent": "reCAPTCHA Python"

        }

    )

    httpresp = urllib.request.urlopen(req)

    return_values = json.loads(httpresp.read())

    httpresp.close()

    if not (isinstance(return_values, dict)):

        return RecaptchaResponse(is_valid=False, error_code='incorrect-captcha-sol')

    elif (("success" in return_values) and ((return_values["success"] is True) or (return_values["success"] == "true"))):

        return RecaptchaResponse(is_valid=True)

    elif (("error-codes" in return_values) and isinstance(return_values["error-codes"], list) and (len(return_values["error-codes"]) > 0)):

        return RecaptchaResponse(is_valid=False, error_code=return_values["error-codes"][0])

    else:

        return RecaptchaResponse(is_valid=False, error_code='incorrect-captcha-sol')

    """

    Generates a random (presumably unique) API key.

    This value is used in URLs and "Bearer" HTTP Authorization headers,

    which in practice means it should only contain URL-safe characters

    (RFC 3986):

        unreserved = ALPHA / DIGIT / "-" / "." / "_" / "~"

    """

    # Hexadecimal certainly qualifies as URL-safe.

    return ascii_str(binascii.hexlify(os.urandom(20)))

def safe_int(val, default=None):

    """

    Returns int() of val if val is not convertable to int use default

    instead

    :param val:

    :param default:

    """

    try:

        val = int(val)

    except (ValueError, TypeError):

        val = default

    return val

def remove_suffix(s, suffix):

    if s.endswith(suffix):

        s = s[:-1 * len(suffix)]

    return s

def remove_prefix(s, prefix):

    if s.startswith(prefix):

        s = s[len(prefix):]

    return s

def age(prevdate, show_short_version=False, now=None):

    """

    turns a datetime into an age string.

    If show_short_version is True, then it will generate a not so accurate but shorter string,

    example: 2days ago, instead of 2 days and 23 hours ago.

    :param prevdate: datetime object

    :param show_short_version: if it should approximate the date and return a shorter string

    """

    now = now or datetime.datetime.now()

    order = ['year', 'month', 'day', 'hour', 'minute', 'second']

    deltas = {}

    future = False

    if prevdate > now:

        now, prevdate = prevdate, now

        future = True

    if future:

        prevdate = prevdate.replace(microsecond=0)

    # Get date parts deltas

    from dateutil import relativedelta

    for part in order:

        d = relativedelta.relativedelta(now, prevdate)

        deltas[part] = getattr(d, part + 's')

    # Fix negative offsets (there is 1 second between 10:59:59 and 11:00:00,

    # not 1 hour, -59 minutes and -59 seconds)

    for num, length in [(5, 60), (4, 60), (3, 24)]:  # seconds, minutes, hours

        part = order[num]

        carry_part = order[num - 1]

        if deltas[part] < 0:

            deltas[part] += length

            deltas[carry_part] -= 1

    # Same thing for days except that the increment depends on the (variable)

    # number of days in the month

    month_lengths = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]

    if deltas['day'] < 0:

        if prevdate.month == 2 and (prevdate.year % 4 == 0 and

            (prevdate.year % 100 != 0 or prevdate.year % 400 == 0)

        ):

            deltas['day'] += 29

        else:

            deltas['day'] += month_lengths[prevdate.month - 1]

        deltas['month'] -= 1

    if deltas['month'] < 0:

        deltas['month'] += 12

        deltas['year'] -= 1

    # In short version, we want nicer handling of ages of more than a year

    if show_short_version:

        if deltas['year'] == 1:

            # ages between 1 and 2 years: show as months

            deltas['year'] = 0

        if deltas['year'] >= 2:

            # ages 2+ years: round

            if deltas['month'] > 6:

                deltas['year'] += 1

                deltas['month'] = 0

    # Format the result

    fmt_funcs = {

        'year': lambda d: ungettext(u'%d year', '%d years', d) % d,

        'month': lambda d: ungettext(u'%d month', '%d months', d) % d,

        'day': lambda d: ungettext(u'%d day', '%d days', d) % d,

        'hour': lambda d: ungettext(u'%d hour', '%d hours', d) % d,

        'minute': lambda d: ungettext(u'%d minute', '%d minutes', d) % d,

        'second': lambda d: ungettext(u'%d second', '%d seconds', d) % d,

    }

    for i, part in enumerate(order):

        value = deltas[part]

        if value == 0:

            continue

        if i < 5:

            sub_part = order[i + 1]

            sub_value = deltas[sub_part]

        else:

            sub_value = 0

        if sub_value == 0 or show_short_version:

            if future:

                return _('in %s') % fmt_funcs[part](value)

            else:

                return _('%s ago') % fmt_funcs[part](value)

        if future:

            return _('in %s and %s') % (fmt_funcs[part](value),

                fmt_funcs[sub_part](sub_value))

        else:

            return _('%s and %s ago') % (fmt_funcs[part](value),

                fmt_funcs[sub_part](sub_value))

    return _('just now')

def uri_filter(uri):

    """

    Removes user:password from given url string

    :param uri:

    :returns: filtered list of strings

    """

    if not uri:

        return []

    proto = ''

    for pat in ('https://', 'http://', 'git://'):

        if uri.startswith(pat):

            uri = uri[len(pat):]

            proto = pat

            break

    # remove passwords and username

    uri = uri[uri.find('@') + 1:]

    # get the port

    cred_pos = uri.find(':')

    if cred_pos == -1:

        host, port = uri, None

    else:

        host, port = uri[:cred_pos], uri[cred_pos + 1:]

    return [_f for _f in [proto, host, port] if _f]

def credentials_filter(uri):

    """

    Returns a url with removed credentials

    :param uri:

    """

    uri = uri_filter(uri)

    # check if we have port

    if len(uri) > 2 and uri[2]:

        uri[2] = ':' + uri[2]

    return ''.join(uri)

def get_clone_url(clone_uri_tmpl, prefix_url, repo_name, repo_id, username=None):

    parsed_url = urlobject.URLObject(prefix_url)

    prefix = urllib.parse.unquote(parsed_url.path.rstrip('/'))

    try:

        system_user = pwd.getpwuid(os.getuid()).pw_name

    except Exception: # TODO: support all systems - especially Windows

        system_user = 'kallithea' # hardcoded default value ...

        :param config_file: A path to file which should be used to retrieve

          configuration from (might also be a list of file paths)

        """

        raise NotImplementedError

    # =========== #

    # WORKDIR API #

    # =========== #

    @LazyProperty

    def workdir(self):

        """

        Returns ``Workdir`` instance for this repository.

        """

        raise NotImplementedError

class BaseChangeset(object):

    """

    Each backend should implement it's changeset representation.

    **Attributes**

        ``repository``

            repository object within which changeset exists

        ``raw_id``

            raw changeset representation (i.e. full 40 length sha for git

            backend)

        ``short_id``

            shortened (if apply) version of ``raw_id``; it would be simple

            shortcut for ``raw_id[:12]`` for git/mercurial backends or same

            as ``raw_id`` for subversion

        ``revision``

            revision number as integer

        ``files``

            list of ``FileNode`` (``Node`` with NodeKind.FILE) objects

        ``dirs``

            list of ``DirNode`` (``Node`` with NodeKind.DIR) objects

        ``nodes``

            combined list of ``Node`` objects

        ``author``

        ``message``

        ``parents``

            list of parent changesets

        ``last``

            ``True`` if this is last changeset in repository, ``False``

            otherwise; trying to access this attribute while there is no

            changesets would raise ``EmptyRepositoryError``

    """

    def __str__(self):

        return '<%s at %s:%s>' % (self.__class__.__name__, self.revision,

            self.short_id)

    def __repr__(self):

        return self.__str__()

    def __eq__(self, other):

        if type(self) is not type(other):

            return False

        return self.raw_id == other.raw_id

    def __json__(self, with_file_list=False):

        if with_file_list:

            return dict(

                short_id=self.short_id,

                raw_id=self.raw_id,

                revision=self.revision,

                message=self.message,

                date=self.date,

                author=self.author,

                added=[el.path for el in self.added],

                changed=[el.path for el in self.changed],

                removed=[el.path for el in self.removed],

            )

        else:

            return dict(

                short_id=self.short_id,

                raw_id=self.raw_id,

                revision=self.revision,

                message=self.message,

                date=self.date,

                author=self.author,

            )

    @LazyProperty

    def last(self):

        if self.repository is None:

            raise ChangesetError("Cannot check if it's most recent revision")

                msg = "Not valid repository at %s. Original error was %s" \

                    % (self.name, err)

            raise RepositoryError(msg)

    @LazyProperty

    def in_memory_changeset(self):

        return MercurialInMemoryChangeset(self)

    @LazyProperty

    def description(self):

        _desc = self._repo.ui.config(b'web', b'description', None, untrusted=True)

        return safe_str(_desc or b'unknown')

    @LazyProperty

    def contact(self):

        return safe_str(mercurial.hgweb.common.get_contact(self._repo.ui.config)

                            or b'Unknown')

    @LazyProperty

    def last_change(self):

        """

        Returns last change made on this repository as datetime object

        """

        return date_fromtimestamp(self._get_mtime(), makedate()[1])

    def _get_mtime(self):

        try:

            return time.mktime(self.get_changeset().date.timetuple())

        except RepositoryError:

            # fallback to filesystem

            cl_path = os.path.join(self.path, '.hg', "00changelog.i")

            st_path = os.path.join(self.path, '.hg', "store")

            if os.path.exists(cl_path):

                return os.stat(cl_path).st_mtime

            else:

                return os.stat(st_path).st_mtime

    def _get_revision(self, revision):

        """

        Given any revision identifier, returns a 40 char string with revision hash.

        :param revision: str or int or None

        """

        if self._empty:

            raise EmptyRepositoryError("There are no changesets yet")

        if revision in [-1, None]:

            revision = b'tip'

            revision = safe_bytes(revision)

        try:

            if isinstance(revision, int):

                return ascii_str(self._repo[revision].hex())

            return ascii_str(mercurial.scmutil.revsymbol(self._repo, revision).hex())

        except (IndexError, ValueError, mercurial.error.RepoLookupError, TypeError):

            msg = "Revision %r does not exist for %s" % (safe_str(revision), self.name)

            raise ChangesetDoesNotExistError(msg)

        except (LookupError, ):

            msg = "Ambiguous identifier `%s` for %s" % (safe_str(revision), self.name)

            raise ChangesetDoesNotExistError(msg)

    def get_ref_revision(self, ref_type, ref_name):

        """

        Returns revision number for the given reference.

        """

        if ref_type == 'rev' and not ref_name.strip('0'):

            return self.EMPTY_CHANGESET

        # lookup up the exact node id

        _revset_predicates = {

                'branch': 'branch',

                'book': 'bookmark',

                'tag': 'tag',

                'rev': 'id',

            }

        # avoid expensive branch(x) iteration over whole repo

        rev_spec = "%%s & %s(%%s)" % _revset_predicates[ref_type]

        try:

            revs = self._repo.revs(rev_spec, ref_name, ref_name)

        except LookupError:

            msg = "Ambiguous identifier %s:%s for %s" % (ref_type, ref_name, self.name)

            raise ChangesetDoesNotExistError(msg)

        except mercurial.error.RepoLookupError:

            msg = "Revision %s:%s does not exist for %s" % (ref_type, ref_name, self.name)

            raise ChangesetDoesNotExistError(msg)

        if revs:

            revision = revs.last()

        else:

            # TODO: just report 'not found'?

            revision = ref_name

        return self._get_revision(revision)

    def _get_archives(self, archive_name='tip'):

        allowed = self.baseui.configlist(b"web", b"allow_archive",

                                         untrusted=True)

        for name, ext in [(b'zip', '.zip'), (b'gz', '.tar.gz'), (b'bz2', '.tar.bz2')]:

        Done_reading does not mean that the iterator's buffer is empty.

        Iterator might have done reading from underlying source, but the read

        chunks might still be available for serving through .next() method.

        :returns: An Bool value.

        """

        return self.worker.EOF.is_set()

    @property

    def length(self):

        """

        returns int.

        This is the length of the queue of chunks, not the length of

        the combined contents in those chunks.

        __len__() cannot be meaningfully implemented because this

        reader is just flying through a bottomless pit content and

        can only know the length of what it already saw.

        If __len__() on WSGI server per PEP 3333 returns a value,

        the response's length will be set to that. In order not to

        confuse WSGI PEP3333 servers, we will not implement __len__

        at all.

        """

        return len(self.data)

    def prepend(self, x):

        self.data.appendleft(x)

    def append(self, x):

        self.data.append(x)

    def extend(self, o):

        self.data.extend(o)

    def __getitem__(self, i):

        return self.data[i]

class SubprocessIOChunker(object):

    """

    Processor class wrapping handling of subprocess IO.

    In a way, this is a "communicate()" replacement with a twist.

    - We are multithreaded. Writing in and reading out, err are all sep threads.

    - We support concurrent (in and out) stream processing.

      chunks. The object behaves as an iterable. You can "for chunk in obj:" us.

    - We are non-blocking in more respects than communicate()

      (reading from subprocess out pauses when internal buffer is full, but

       does not block the parent calling code. On the flip side, reading from

       slow-yielding subprocess may block the iteration until data shows up. This

       does not block the parallel inpipe reading occurring parallel thread.)

    The purpose of the object is to allow us to wrap subprocess interactions into

    an iterable that can be passed to a WSGI server as the application's return

    value. Because of stream-processing-ability, WSGI does not have to read ALL

    of the subprocess's output and buffer it, before handing it to WSGI server for

    HTTP response. Instead, the class initializer reads just a bit of the stream

    to figure out if error occurred or likely to occur and if not, just hands the

    further iteration over subprocess output to the server for completion of HTTP

    response.

    The real or perceived subprocess error is trapped and raised as one of

    EnvironmentError family of exceptions

    Example usage:

    #    try:

    #        answer = SubprocessIOChunker(

    #            cmd,

    #            input,

    #            buffer_size = 65536,

    #            chunk_size = 4096

    #            )

    #    except (EnvironmentError) as e:

    #        print str(e)

    #        raise e

    #

    #    return answer

    """

    def __init__(self, cmd, inputstream=None, buffer_size=65536,

                 chunk_size=4096, starting_values=None, **kwargs):

        """

        Initializes SubprocessIOChunker

        :param cmd: A Subprocess.Popen style "cmd". Can be string or array of strings

        :param inputstream: (Default: None) A file-like, string, or file pointer.

        :param buffer_size: (Default: 65536) A size of total buffer per stream in bytes.

        :param chunk_size: (Default: 4096) A max size of a chunk. Actual chunk may be smaller.

        :param starting_values: (Default: []) An array of strings to put in front of output que.

        """

        starting_values = starting_values or []