# -*- coding: utf-8 -*-

"""

Kallithea wrapper of Celery

The Celery configuration is in the Kallithea ini file but must be converted to an

entirely different format before Celery can use it.

We read the configuration from tg.config at module import time. This module can

thus not be imported in global scope but must be imported on demand in function

scope after tg.config has been initialized.

To make sure that the config really has been initialized, we check one of the

mandatory settings.

"""

import celery

import celery.result as result

import tg

from celery.bin import worker

from celery.task import task

def celery_config(config):

    """Return Celery config object populated from relevant settings in a config dict, such as tg.config"""

    # Verify .ini file configuration has been loaded

    assert config['celery.imports'] == 'kallithea.lib.celerylib.tasks', 'Kallithea Celery configuration has not been loaded'

    class CeleryConfig(object):

        pass

    celery_config = CeleryConfig()

    PREFIXES = """ADMINS BROKER CASSANDRA CELERYBEAT CELERYD CELERYMON CELERY EMAIL SERVER""".split()

    LIST_PARAMS = """CELERY_IMPORTS ADMINS ROUTES CELERY_ACCEPT_CONTENT""".split()

    for config_key, config_value in sorted(config.items()):

        celery_key = config_key.replace('.', '_').upper()

        if celery_key.split('_', 1)[0] not in PREFIXES:

            continue

            continue

        if celery_key in LIST_PARAMS:

            celery_value = config_value.split()

        elif config_value.isdigit():

            celery_value = int(config_value)

        elif config_value.lower() in ['true', 'false']:

            celery_value = config_value.lower() == 'true'

        else:

            celery_value = config_value

        setattr(celery_config, celery_key, celery_value)

    return celery_config

def make_app():

    """Create celery app from the TurboGears configuration file"""

    app = celery.Celery()

    app.config_from_object(celery_config(tg.config))

    return app

    return literal(

        ('(' + json.dumps(value) + ')')

        # In JSON, the following can only appear in string literals.

        .replace('&', r'\x26')

        .replace('<', r'\x3c')

        .replace('>', r'\x3e')

    )

def jshtml(val):

    """HTML escapes a string value, then converts the resulting string

    to its corresponding JavaScript representation (see `js`).

    This is used when a plain-text string (possibly containing special

    HTML characters) will be used by a script in an HTML context (e.g.

    element.innerHTML or jQuery's 'html' method).

    If in doubt, err on the side of using `jshtml` over `js`, since it's

    better to escape too much than too little.

    """

    return js(escape(val))

def shorter(s, size=20, firstline=False, postfix='...'):

    """Truncate s to size, including the postfix string if truncating.

    If firstline, truncate at newline.

    """

    if firstline:

        s = s.split('\n', 1)[0].rstrip()

    if len(s) > size:

        return s[:size - len(postfix)] + postfix

    return s

def reset(name, value, id=NotGiven, **attrs):

    """Create a reset button, similar to webhelpers2.html.tags.submit ."""

    return _input("reset", name, value, id, attrs)

def select(name, selected_values, options, id=NotGiven, **attrs):

    """Convenient wrapper of webhelpers2 to let it accept options as a tuple list"""

    if isinstance(options, list):

        option_list = options

        # Handle old value,label lists ... where value also can be value,label lists

        options = Options()

        for x in option_list:

            if isinstance(x, tuple) and len(x) == 2:

                value, label = x

                value = label = x

            else:

                log.error('invalid select option %r', x)

                raise

            if isinstance(value, list):

                og = options.add_optgroup(label)

                for x in value:

                    if isinstance(x, tuple) and len(x) == 2:

                        group_value, group_label = x

                        group_value = group_label = x

                    else:

                        log.error('invalid select option %r', x)

                        raise

                    og.add_option(group_label, group_value)

            else:

                options.add_option(label, value)

    return webhelpers2_select(name, selected_values, options, id=id, **attrs)

safeid = _make_safe_id_component

def FID(raw_id, path):

    """

    Creates a unique ID for filenode based on it's hash of path and revision

    it's safe to use in urls

    :param raw_id:

    :param path:

    """

    return 'C-%s-%s' % (short_id(raw_id), hashlib.md5(safe_bytes(path)).hexdigest()[:12])

class _FilesBreadCrumbs(object):

    def __call__(self, repo_name, rev, paths):

        if isinstance(paths, str):

            paths = safe_unicode(paths)

        url_l = [link_to(repo_name, url('files_home',

                                        repo_name=repo_name,

                                        revision=rev, f_path=''),

                         class_='ypjax-link')]

        paths_l = paths.split('/')

        for cnt, p in enumerate(paths_l):

            if p != '':

                url_l.append(link_to(p,

                                     url('files_home',

                                         repo_name=repo_name,

                                         revision=rev,

                                         f_path='/'.join(paths_l[:cnt + 1])

                                         ),

                                     class_='ypjax-link'

                                     )

                             )

        return literal('/'.join(url_l))

from kallithea.lib.rcmail.exceptions import BadHeaders, InvalidMessage

from kallithea.lib.rcmail.response import MailResponse

class Attachment(object):

    """

    Encapsulates file attachment information.

    :param filename: filename of attachment

    :param content_type: file mimetype

    :param data: the raw file data, either as string or file obj

    :param disposition: content-disposition (if any)

    """

    def __init__(self,

                 filename=None,

                 content_type=None,

                 data=None,

                 disposition=None):

        self.filename = filename

        self.content_type = content_type

        self.disposition = disposition or 'attachment'

        self._data = data

    @property

    def data(self):

            return self._data

        self._data = self._data.read()

        return self._data

class Message(object):

    """

    Encapsulates an email message.

    :param subject: email subject header

    :param recipients: list of email addresses

    :param body: plain text message

    :param html: HTML message

    :param sender: email sender address

    :param cc: CC list

    :param bcc: BCC list

    :param extra_headers: dict of extra email headers

    :param attachments: list of Attachment instances

    :param recipients_separator: alternative separator for any of

        'From', 'To', 'Delivered-To', 'Cc', 'Bcc' fields

    """

    def __init__(self,

                 subject=None,

                 recipients=None,

                 body=None,

                 html=None,

                 sender=None,

                 cc=None,

                 bcc=None,

                 extra_headers=None,

                 attachments=None,

                 recipients_separator="; "):

        self.subject = subject or ''

        self.sender = sender

        self.body = body

        self.html = html

        self.recipients = recipients or []

        self.attachments = attachments or []

        self.cc = cc or []

        self.bcc = bcc or []

        self.extra_headers = extra_headers or {}

        self.recipients_separator = recipients_separator

    @property

This file was forked by the Kallithea project in July 2014.

Original author and date, and relevant copyright and licensing information is below:

:created_on: Sep 13, 2010

:author: marcink

:copyright: (c) 2013 RhodeCode GmbH, and others.

:license: GPLv3, see LICENSE.md for more details.

"""

import logging

import smtplib

import time

from email.utils import formatdate

from ssl import SSLError

from kallithea.lib.rcmail.message import Message

from kallithea.lib.rcmail.utils import DNS_NAME

class SmtpMailer(object):

    """SMTP mailer class

    mailer = SmtpMailer(mail_from, user, passwd, mail_server, smtp_auth

                        mail_port, ssl, tls)

    mailer.send(recipients, subject, body, attachment_files)

    :param recipients might be a list of string or single string

    :param attachment_files is a dict of {filename:location}

        it tries to guess the mimetype and attach the file

    """

    def __init__(self, mail_from, user, passwd, mail_server, smtp_auth=None,

                 mail_port=None, ssl=False, tls=False, debug=False):

        self.mail_from = mail_from

        self.mail_server = mail_server

        self.mail_port = mail_port

        self.user = user

        self.passwd = passwd

        self.ssl = ssl

        self.tls = tls

        self.debug = debug

        self.auth = smtp_auth

    def send(self, recipients=None, subject='', body='', html='',

             attachment_files=None, headers=None):

        recipients = recipients or []

            recipients = [recipients]

        if headers is None:

            headers = {}

        headers.setdefault('Date', formatdate(time.time()))

        msg = Message(subject, recipients, body, html, self.mail_from,

                      recipients_separator=", ", extra_headers=headers)

        raw_msg = msg.to_message()

        if self.ssl:

            smtp_serv = smtplib.SMTP_SSL(self.mail_server, self.mail_port,

                                         local_hostname=DNS_NAME.get_fqdn())

        else:

            smtp_serv = smtplib.SMTP(self.mail_server, self.mail_port,

                                     local_hostname=DNS_NAME.get_fqdn())

        if self.tls:

            smtp_serv.ehlo()

            smtp_serv.starttls()

        if self.debug:

            smtp_serv.set_debuglevel(1)

        smtp_serv.ehlo()

        if self.auth:

            smtp_serv.esmtp_features["auth"] = self.auth

        # if server requires authorization you must provide login and password

        # but only if we have them

        if self.user and self.passwd:

            smtp_serv.login(self.user, self.passwd)

        smtp_serv.sendmail(msg.sender, msg.send_to, raw_msg.as_string())

        logging.info('MAIL SENT TO: %s' % recipients)

        try:

            smtp_serv.quit()

        except SSLError:

            # SSL error might sometimes be raised in tls connections on closing

            smtp_serv.close()

    """

    by_id_match = re.match(r'^_(\d+)$', s)

    if by_id_match is None:

        return None

    return int(by_id_match.group(1))

def fix_repo_id_name(path):

    """

    Rewrite repo_name for _<ID> permanent URLs.

    Given a path, if the first path element is like _<ID>, return the path with

    this part expanded to the corresponding full repo name, else return the

    provided path.

    """

    first, rest = path, ''

    if '/' in path:

        first, rest_ = path.split('/', 1)

        rest = '/' + rest_

    repo_id = _get_permanent_id(first)

    if repo_id is not None:

        repo = Repository.get(repo_id)

        if repo is not None:

            return repo.repo_name + rest

    return path

def action_logger(user, action, repo, ipaddr='', commit=False):

    """

    Action logger for various actions made by users

    :param user: user that made this action, can be a unique username string or

        object containing user_id attribute

    :param action: action to log, should be on of predefined unique actions for

        easy translations

    :param repo: string name of repository or object containing repo_id,

        that action was made on

    :param ipaddr: optional IP address from what the action was made

    """

    # if we don't get explicit IP address try to get one from registered user

    # in tmpl context var

    if not ipaddr:

        ipaddr = getattr(get_current_authuser(), 'ip_addr', '')

    if getattr(user, 'user_id', None):

        user_obj = User.get(user.user_id)

        user_obj = User.get_by_username(user)

    else:

        raise Exception('You have to provide a user object or a username')

    if getattr(repo, 'repo_id', None):

        repo_obj = Repository.get(repo.repo_id)

        repo_name = repo_obj.repo_name

        repo_name = repo.lstrip('/')

        repo_obj = Repository.get_by_repo_name(repo_name)

    else:

        repo_obj = None

        repo_name = u''

    user_log = UserLog()

    user_log.user_id = user_obj.user_id

    user_log.username = user_obj.username

    user_log.action = safe_unicode(action)

    user_log.repository = repo_obj

    user_log.repository_name = repo_name

    user_log.action_date = datetime.datetime.now()

    user_log.user_ip = ipaddr

    meta.Session().add(user_log)

    log.info('Logging action:%s on %s by user:%s ip:%s',

             action, safe_unicode(repo), user_obj, ipaddr)

    if commit:

        meta.Session().commit()

def get_filesystem_repos(path):

    """

    Scans given path for repos and return (name,(type,path)) tuple

    :param path: path to scan for repositories

    :param recursive: recursive search and return names with subdirs in front

    """

    # remove ending slash for better results

    path = safe_str(path.rstrip(os.sep))

    log.debug('now scanning in %s', path)

    def isdir(*n):

        return os.path.isdir(os.path.join(*n))

    for root, dirs, _files in os.walk(path):

        recurse_dirs = []

        for subdir in dirs:

            # skip removed repos

            if REMOVED_REPO_PAT.match(subdir):

                continue

            # skip .<something> dirs TODO: rly? then we should prevent creating them ...

            if subdir.startswith('.'):

:license: GPLv3, see LICENSE.md for more details.

"""

from __future__ import print_function

import binascii

import datetime

import json

import os

import pwd

import re

import time

import urllib

import urlobject

from tg.i18n import ugettext as _

from tg.i18n import ungettext

from webhelpers2.text import collapse, remove_formatting, strip_tags

from kallithea.lib.vcs.utils import ascii_bytes, ascii_str, safe_bytes, safe_str, safe_unicode  # re-export

from kallithea.lib.vcs.utils.lazy import LazyProperty

def str2bool(_str):

    """

    returns True/False value from given string, it tries to translate the

    string into boolean

    :param _str: string value to translate into boolean

    :rtype: boolean

    :returns: boolean from given string

    """

    if _str is None:

        return False

    if _str in (True, False):

        return _str

    _str = str(_str).strip().lower()

    return _str in ('t', 'true', 'y', 'yes', 'on', '1')

def aslist(obj, sep=None, strip=True):

    """

    Returns given string separated by sep as list

    :param obj:

    :param sep:

    :param strip:

    """

        lst = obj.split(sep)

        if strip:

            lst = [v.strip() for v in lst]

        return lst

    elif isinstance(obj, (list, tuple)):

        return obj

    elif obj is None:

        return []

    else:

        return [obj]

def convert_line_endings(line, mode):

    """

    Converts a given line  "line end" according to given mode

    Available modes are::

        0 - Unix

        1 - Mac

        2 - DOS

    :param line: given line to convert

    :param mode: mode to convert to

    :rtype: str

    :return: converted line according to mode

    """

    if mode == 0:

        line = line.replace('\r\n', '\n')

        line = line.replace('\r', '\n')

    elif mode == 1:

        line = line.replace('\r\n', '\r')

        line = line.replace('\n', '\r')

    elif mode == 2:

        line = re.sub("\r(?!\n)|(?<!\r)\n", "\r\n", line)

    return line

def detect_mode(line, default):

    """

    Detects line break for given line, if line break couldn't be found

    given default value is returned

    :param line: str line

    :param default: default

    :rtype: int

    :return: value of line end on of 0 - Unix, 1 - Mac, 2 - DOS

    """

    if line.endswith('\r\n'):

        'scheme': parsed_url.scheme,

        'user': safe_unicode(urllib.quote(safe_str(username or ''))),

        'netloc': parsed_url.netloc + prefix,  # like "hostname:port/prefix" (with optional ":port" and "/prefix")

        'prefix': prefix, # undocumented, empty or starting with /

        'repo': repo_name,

        'repoid': str(repo_id),

        'system_user': safe_unicode(system_user),

        'hostname': parsed_url.hostname,

    }

    url = re.sub('{([^{}]+)}', lambda m: args.get(m.group(1), m.group(0)), clone_uri_tmpl)

    # remove leading @ sign if it's present. Case of empty user

    url_obj = urlobject.URLObject(url)

    if not url_obj.username:

        url_obj = url_obj.with_username(None)

    return safe_unicode(url_obj)

def get_changeset_safe(repo, rev):

    """

    Safe version of get_changeset if this changeset doesn't exists for a

    repo it returns a Dummy one instead

    :param repo:

    :param rev:

    """

    from kallithea.lib.vcs.backends.base import BaseRepository

    from kallithea.lib.vcs.exceptions import RepositoryError

    from kallithea.lib.vcs.backends.base import EmptyChangeset

    if not isinstance(repo, BaseRepository):

        raise Exception('You must pass an Repository '

                        'object as first argument got %s' % type(repo))

    try:

        cs = repo.get_changeset(rev)

    except (RepositoryError, LookupError):

        cs = EmptyChangeset(requested_revision=rev)

    return cs

def datetime_to_time(dt):

    if dt:

        return time.mktime(dt.timetuple())

def time_to_datetime(tm):

    if tm:

            try:

                tm = float(tm)

            except ValueError:

                return

        return datetime.datetime.fromtimestamp(tm)

# Must match regexp in kallithea/public/js/base.js MentionsAutoComplete()

# Check char before @ - it must not look like we are in an email addresses.

# Matching is greedy so we don't have to look beyond the end.

MENTIONS_REGEX = re.compile(r'(?:^|(?<=[^a-zA-Z0-9]))@([a-zA-Z0-9][-_.a-zA-Z0-9]*[a-zA-Z0-9])')

def extract_mentioned_usernames(text):

    r"""

    Returns list of (possible) usernames @mentioned in given text.

    >>> extract_mentioned_usernames('@1-2.a_X,@1234 not@not @ddd@not @n @ee @ff @gg, @gg;@hh @n\n@zz,')

    ['1-2.a_X', '1234', 'ddd', 'ee', 'ff', 'gg', 'gg', 'hh', 'zz']

    """

    return MENTIONS_REGEX.findall(text)

def extract_mentioned_users(text):

    """ Returns set of actual database Users @mentioned in given text. """

    from kallithea.model.db import User

    result = set()

    for name in extract_mentioned_usernames(text):

        user = User.get_by_username(name, case_insensitive=True)

        if user is not None and not user.is_default_user:

            result.add(user)

    return result

class AttributeDict(dict):

    def __getattr__(self, attr):

        return self.get(attr, None)

    __setattr__ = dict.__setitem__

    __delattr__ = dict.__delitem__

def obfuscate_url_pw(engine):

    from sqlalchemy.engine import url as sa_url

    from sqlalchemy.exc import ArgumentError

    try:

        _url = sa_url.make_url(engine or '')

    except ArgumentError:

        return engine

        url = self._get_url(url)

        cmd = ['pull', '--ff-only', url]

        # If error occurs run_git_command raises RepositoryError already

        self.run_git_command(cmd)

    def fetch(self, url):

        """

        Tries to pull changes from external location.

        """

        url = self._get_url(url)

        so = self.run_git_command(['ls-remote', '-h', url])

        cmd = ['fetch', url, '--']

        for line in (x for x in so.splitlines()):

            sha, ref = line.split('\t')

            cmd.append('+%s:%s' % (ref, ref))

        self.run_git_command(cmd)

    def _update_server_info(self):

        """

        runs gits update-server-info command in this repo instance

        """

        from dulwich.server import update_server_info

        try:

            update_server_info(self._repo)

        except OSError as e:

            if e.errno not in [errno.ENOENT, errno.EROFS]:

                raise

            # Workaround for dulwich crashing on for example its own dulwich/tests/data/repos/simple_merge.git/info/refs.lock

            log.error('Ignoring %s running update-server-info: %s', type(e).__name__, e)

    @LazyProperty

    def workdir(self):

        """

        Returns ``Workdir`` instance for this repository.

        """

        return GitWorkdir(self)

    def get_config_value(self, section, name, config_file=None):

        """

        Returns configuration value for a given [``section``] and ``name``.

        :param section: Section we want to retrieve value from

        :param name: Name of configuration we want to retrieve

        :param config_file: A path to file which should be used to retrieve

          configuration from (might also be a list of file paths)

        """

        if config_file is None:

            config_file = []

            config_file = [config_file]

        def gen_configs():

            for path in config_file + self._config_files:

                try:

                    yield ConfigFile.from_path(path)

                except (IOError, OSError, ValueError):

                    continue

        for config in gen_configs():

            try:

                return config.get(section, name)

            except KeyError:

                continue

        return None

    def get_user_name(self, config_file=None):

        """

        Returns user's name from global configuration file.

        :param config_file: A path to file which should be used to retrieve

          configuration from (might also be a list of file paths)

        """

        return self.get_config_value('user', 'name', config_file)

    def get_user_email(self, config_file=None):

        """

        Returns user's email from global configuration file.

        :param config_file: A path to file which should be used to retrieve

          configuration from (might also be a list of file paths)

        """

        return self.get_config_value('user', 'email', config_file)

import os

import posixpath

import mercurial.archival

import mercurial.node

import mercurial.obsutil

from kallithea.lib.vcs.backends.base import BaseChangeset

from kallithea.lib.vcs.conf import settings

from kallithea.lib.vcs.exceptions import ChangesetDoesNotExistError, ChangesetError, ImproperArchiveTypeError, NodeDoesNotExistError, VCSError

from kallithea.lib.vcs.nodes import (

    AddedFileNodesGenerator, ChangedFileNodesGenerator, DirNode, FileNode, NodeKind, RemovedFileNodesGenerator, RootNode, SubModuleNode)

from kallithea.lib.vcs.utils import ascii_bytes, ascii_str, date_fromtimestamp, safe_bytes, safe_str, safe_unicode

from kallithea.lib.vcs.utils.lazy import LazyProperty

from kallithea.lib.vcs.utils.paths import get_dirs_for_path

class MercurialChangeset(BaseChangeset):

    """

    Represents state of the repository at a revision.

    """

    def __init__(self, repository, revision):

        self.repository = repository

        self._ctx = repository._repo[ascii_bytes(revision)]

        self.raw_id = ascii_str(self._ctx.hex())

        self.revision = self._ctx._rev

        self.nodes = {}

    @LazyProperty

    def tags(self):

        return [safe_unicode(tag) for tag in self._ctx.tags()]

    @LazyProperty

    def branch(self):

        return safe_unicode(self._ctx.branch())

    @LazyProperty

    def branches(self):

        return [safe_unicode(self._ctx.branch())]

    @LazyProperty

    def closesbranch(self):

        return self._ctx.closesbranch()

    @LazyProperty

    def obsolete(self):

        return self._ctx.obsolete()

    @LazyProperty

    def bumped(self):

        return self._ctx.phasedivergent()

    @LazyProperty

    def divergent(self):

        return self._ctx.contentdivergent()

    @LazyProperty

    def extinct(self):

        return self._ctx.extinct()

    @LazyProperty

    def unstable(self):

        return self._ctx.orphan()

    @LazyProperty

    def phase(self):

        if(self._ctx.phase() == 1):

            return 'Draft'

        elif(self._ctx.phase() == 2):

            return 'Secret'

        else:

        if filter_ or max_revisions:

            if filter_:

                revspec = b' and '.join(filter_)

            else:

                revspec = b'all()'

            if max_revisions:

                revspec = b'limit(%s, %d)' % (revspec, max_revisions)

            revisions = mercurial.scmutil.revrange(self._repo, [revspec])

        else:

            revisions = self.revisions

        # this is very much a hack to turn this into a list; a better solution

        # would be to get rid of this function entirely and use revsets

        revs = list(revisions)[start_pos:end_pos]

        if reverse:

            revs.reverse()

        return CollectionGenerator(self, revs)

    def pull(self, url):

        """

        Tries to pull changes from external location.

        """

        other = mercurial.hg.peer(self._repo, {}, safe_bytes(self._get_url(url)))

        try:

            mercurial.exchange.pull(self._repo, other, heads=None, force=None)

        except mercurial.error.Abort as err:

            # Propagate error but with vcs's type

            raise RepositoryError(str(err))

    @LazyProperty

    def workdir(self):

        """

        Returns ``Workdir`` instance for this repository.

        """

        return MercurialWorkdir(self)

    def get_config_value(self, section, name=None, config_file=None):

        """