import celery.result as result

import tg

from celery.bin import worker

from celery.task import task

def celery_config(config):

    """Return Celery config object populated from relevant settings in a config dict, such as tg.config"""

    # Verify .ini file configuration has been loaded

    assert config['celery.imports'] == 'kallithea.lib.celerylib.tasks', 'Kallithea Celery configuration has not been loaded'

    class CeleryConfig(object):

        pass

    celery_config = CeleryConfig()

    PREFIXES = """ADMINS BROKER CASSANDRA CELERYBEAT CELERYD CELERYMON CELERY EMAIL SERVER""".split()

    LIST_PARAMS = """CELERY_IMPORTS ADMINS ROUTES CELERY_ACCEPT_CONTENT""".split()

    for config_key, config_value in sorted(config.items()):

        celery_key = config_key.replace('.', '_').upper()

        if celery_key.split('_', 1)[0] not in PREFIXES:

            continue

            continue

        if celery_key in LIST_PARAMS:

            celery_value = config_value.split()

        elif config_value.isdigit():

            celery_value = int(config_value)

        elif config_value.lower() in ['true', 'false']:

            celery_value = config_value.lower() == 'true'

        else:

            celery_value = config_value

        setattr(celery_config, celery_key, celery_value)

    return celery_config

def make_app():

    """Create celery app from the TurboGears configuration file"""

    app = celery.Celery()

    app.config_from_object(celery_config(tg.config))

    return app

    """Truncate s to size, including the postfix string if truncating.

    If firstline, truncate at newline.

    """

    if firstline:

        s = s.split('\n', 1)[0].rstrip()

    if len(s) > size:

        return s[:size - len(postfix)] + postfix

    return s

def reset(name, value, id=NotGiven, **attrs):

    """Create a reset button, similar to webhelpers2.html.tags.submit ."""

    return _input("reset", name, value, id, attrs)

def select(name, selected_values, options, id=NotGiven, **attrs):

    """Convenient wrapper of webhelpers2 to let it accept options as a tuple list"""

    if isinstance(options, list):

        option_list = options

        # Handle old value,label lists ... where value also can be value,label lists

        options = Options()

        for x in option_list:

            if isinstance(x, tuple) and len(x) == 2:

                value, label = x

                value = label = x

            else:

                log.error('invalid select option %r', x)

                raise

            if isinstance(value, list):

                og = options.add_optgroup(label)

                for x in value:

                    if isinstance(x, tuple) and len(x) == 2:

                        group_value, group_label = x

                        group_value = group_label = x

                    else:

                        log.error('invalid select option %r', x)

                        raise

                    og.add_option(group_label, group_value)

            else:

                options.add_option(label, value)

    return webhelpers2_select(name, selected_values, options, id=id, **attrs)

safeid = _make_safe_id_component

def FID(raw_id, path):

    """

    Creates a unique ID for filenode based on it's hash of path and revision

    it's safe to use in urls

    :param raw_id:

    :param path:

    """

    return 'C-%s-%s' % (short_id(raw_id), hashlib.md5(safe_bytes(path)).hexdigest()[:12])

class Attachment(object):

    """

    Encapsulates file attachment information.

    :param filename: filename of attachment

    :param content_type: file mimetype

    :param data: the raw file data, either as string or file obj

    :param disposition: content-disposition (if any)

    """

    def __init__(self,

                 filename=None,

                 content_type=None,

                 data=None,

                 disposition=None):

        self.filename = filename

        self.content_type = content_type

        self.disposition = disposition or 'attachment'

        self._data = data

    @property

    def data(self):

            return self._data

        self._data = self._data.read()

        return self._data

class Message(object):

    """

    Encapsulates an email message.

    :param subject: email subject header

    :param recipients: list of email addresses

    :param body: plain text message

    :param html: HTML message

    :param sender: email sender address

    :param cc: CC list

    :param bcc: BCC list

    :param extra_headers: dict of extra email headers

    :param attachments: list of Attachment instances

    :param recipients_separator: alternative separator for any of

        'From', 'To', 'Delivered-To', 'Cc', 'Bcc' fields

    """

    def __init__(self,

                 subject=None,

    mailer.send(recipients, subject, body, attachment_files)

    :param recipients might be a list of string or single string

    :param attachment_files is a dict of {filename:location}

        it tries to guess the mimetype and attach the file

    """

    def __init__(self, mail_from, user, passwd, mail_server, smtp_auth=None,

                 mail_port=None, ssl=False, tls=False, debug=False):

        self.mail_from = mail_from

        self.mail_server = mail_server

        self.mail_port = mail_port

        self.user = user

        self.passwd = passwd

        self.ssl = ssl

        self.tls = tls

        self.debug = debug

        self.auth = smtp_auth

    def send(self, recipients=None, subject='', body='', html='',

             attachment_files=None, headers=None):

        recipients = recipients or []

            recipients = [recipients]

        if headers is None:

            headers = {}

        headers.setdefault('Date', formatdate(time.time()))

        msg = Message(subject, recipients, body, html, self.mail_from,

                      recipients_separator=", ", extra_headers=headers)

        raw_msg = msg.to_message()

        if self.ssl:

            smtp_serv = smtplib.SMTP_SSL(self.mail_server, self.mail_port,

                                         local_hostname=DNS_NAME.get_fqdn())

        else:

            smtp_serv = smtplib.SMTP(self.mail_server, self.mail_port,

                                     local_hostname=DNS_NAME.get_fqdn())

        if self.tls:

            smtp_serv.ehlo()

            smtp_serv.starttls()

        if self.debug:

            smtp_serv.set_debuglevel(1)

        smtp_serv.ehlo()

        if self.auth:

    return path

def action_logger(user, action, repo, ipaddr='', commit=False):

    """

    Action logger for various actions made by users

    :param user: user that made this action, can be a unique username string or

        object containing user_id attribute

    :param action: action to log, should be on of predefined unique actions for

        easy translations

    :param repo: string name of repository or object containing repo_id,

        that action was made on

    :param ipaddr: optional IP address from what the action was made

    """

    # if we don't get explicit IP address try to get one from registered user

    # in tmpl context var

    if not ipaddr:

        ipaddr = getattr(get_current_authuser(), 'ip_addr', '')

    if getattr(user, 'user_id', None):

        user_obj = User.get(user.user_id)

        user_obj = User.get_by_username(user)

    else:

        raise Exception('You have to provide a user object or a username')

    if getattr(repo, 'repo_id', None):

        repo_obj = Repository.get(repo.repo_id)

        repo_name = repo_obj.repo_name

        repo_name = repo.lstrip('/')

        repo_obj = Repository.get_by_repo_name(repo_name)

    else:

        repo_obj = None

        repo_name = u''

    user_log = UserLog()

    user_log.user_id = user_obj.user_id

    user_log.username = user_obj.username

    user_log.action = safe_unicode(action)

    user_log.repository = repo_obj

    user_log.repository_name = repo_name

    user_log.action_date = datetime.datetime.now()

    user_log.user_ip = ipaddr

    meta.Session().add(user_log)

    log.info('Logging action:%s on %s by user:%s ip:%s',

             action, safe_unicode(repo), user_obj, ipaddr)

    if commit:

        meta.Session().commit()

    """

    returns True/False value from given string, it tries to translate the

    string into boolean

    :param _str: string value to translate into boolean

    :rtype: boolean

    :returns: boolean from given string

    """

    if _str is None:

        return False

    if _str in (True, False):

        return _str

    _str = str(_str).strip().lower()

    return _str in ('t', 'true', 'y', 'yes', 'on', '1')

def aslist(obj, sep=None, strip=True):

    """

    Returns given string separated by sep as list

    :param obj:

    :param sep:

    :param strip:

    """

        lst = obj.split(sep)

        if strip:

            lst = [v.strip() for v in lst]

        return lst

    elif isinstance(obj, (list, tuple)):

        return obj

    elif obj is None:

        return []

    else:

        return [obj]

def convert_line_endings(line, mode):

    """

    Converts a given line  "line end" according to given mode

    Available modes are::

        0 - Unix

        1 - Mac

        2 - DOS

    :param line: given line to convert

    :param mode: mode to convert to

    :rtype: str

    :param repo:

    :param rev:

    """

    from kallithea.lib.vcs.backends.base import BaseRepository

    from kallithea.lib.vcs.exceptions import RepositoryError

    from kallithea.lib.vcs.backends.base import EmptyChangeset

    if not isinstance(repo, BaseRepository):

        raise Exception('You must pass an Repository '

                        'object as first argument got %s' % type(repo))

    try:

        cs = repo.get_changeset(rev)

    except (RepositoryError, LookupError):

        cs = EmptyChangeset(requested_revision=rev)

    return cs

def datetime_to_time(dt):

    if dt:

        return time.mktime(dt.timetuple())

def time_to_datetime(tm):

    if tm:

            try:

                tm = float(tm)

            except ValueError:

                return

        return datetime.datetime.fromtimestamp(tm)

# Must match regexp in kallithea/public/js/base.js MentionsAutoComplete()

# Check char before @ - it must not look like we are in an email addresses.

# Matching is greedy so we don't have to look beyond the end.

MENTIONS_REGEX = re.compile(r'(?:^|(?<=[^a-zA-Z0-9]))@([a-zA-Z0-9][-_.a-zA-Z0-9]*[a-zA-Z0-9])')

def extract_mentioned_usernames(text):

    r"""

    Returns list of (possible) usernames @mentioned in given text.

    >>> extract_mentioned_usernames('@1-2.a_X,@1234 not@not @ddd@not @n @ee @ff @gg, @gg;@hh @n\n@zz,')

    ['1-2.a_X', '1234', 'ddd', 'ee', 'ff', 'gg', 'gg', 'hh', 'zz']

    """

    return MENTIONS_REGEX.findall(text)

def extract_mentioned_users(text):

        except OSError as e:

            if e.errno not in [errno.ENOENT, errno.EROFS]:

                raise

            # Workaround for dulwich crashing on for example its own dulwich/tests/data/repos/simple_merge.git/info/refs.lock

            log.error('Ignoring %s running update-server-info: %s', type(e).__name__, e)

    @LazyProperty

    def workdir(self):

        """

        Returns ``Workdir`` instance for this repository.

        """

        return GitWorkdir(self)

    def get_config_value(self, section, name, config_file=None):

        """

        Returns configuration value for a given [``section``] and ``name``.

        :param section: Section we want to retrieve value from

        :param name: Name of configuration we want to retrieve

        :param config_file: A path to file which should be used to retrieve

          configuration from (might also be a list of file paths)

        """

        if config_file is None:

            config_file = []

            config_file = [config_file]

        def gen_configs():

            for path in config_file + self._config_files:

                try:

                    yield ConfigFile.from_path(path)

                except (IOError, OSError, ValueError):

                    continue

        for config in gen_configs():

            try:

                return config.get(section, name)

            except KeyError:

                continue

        return None

    def get_user_name(self, config_file=None):

        """

        Returns user's name from global configuration file.

        :param config_file: A path to file which should be used to retrieve

          configuration from (might also be a list of file paths)

        """

        return self.get_config_value('user', 'name', config_file)

import os

import posixpath

import mercurial.archival

import mercurial.node

import mercurial.obsutil

from kallithea.lib.vcs.backends.base import BaseChangeset

from kallithea.lib.vcs.conf import settings

from kallithea.lib.vcs.exceptions import ChangesetDoesNotExistError, ChangesetError, ImproperArchiveTypeError, NodeDoesNotExistError, VCSError

from kallithea.lib.vcs.nodes import (

    AddedFileNodesGenerator, ChangedFileNodesGenerator, DirNode, FileNode, NodeKind, RemovedFileNodesGenerator, RootNode, SubModuleNode)

from kallithea.lib.vcs.utils import ascii_bytes, ascii_str, date_fromtimestamp, safe_bytes, safe_str, safe_unicode

from kallithea.lib.vcs.utils.lazy import LazyProperty

from kallithea.lib.vcs.utils.paths import get_dirs_for_path

class MercurialChangeset(BaseChangeset):

    """

    Represents state of the repository at a revision.

    """

    def __init__(self, repository, revision):

        self.repository = repository

        self._ctx = repository._repo[ascii_bytes(revision)]

        self.raw_id = ascii_str(self._ctx.hex())

        self.revision = self._ctx._rev

        self.nodes = {}

    @LazyProperty

    def tags(self):

        return [safe_unicode(tag) for tag in self._ctx.tags()]

    @LazyProperty

    def branch(self):

        return safe_unicode(self._ctx.branch())

    @LazyProperty

    def branches(self):

        return [safe_unicode(self._ctx.branch())]

    @LazyProperty

    def closesbranch(self):

        return self._ctx.closesbranch()

    @LazyProperty

    def obsolete(self):

        return self._ctx.obsolete()

        try:

            mercurial.exchange.pull(self._repo, other, heads=None, force=None)

        except mercurial.error.Abort as err:

            # Propagate error but with vcs's type

            raise RepositoryError(str(err))

    @LazyProperty

    def workdir(self):

        """

        Returns ``Workdir`` instance for this repository.

        """

        return MercurialWorkdir(self)

    def get_config_value(self, section, name=None, config_file=None):

        """

        Returns configuration value for a given [``section``] and ``name``.

        :param section: Section we want to retrieve value from

        :param name: Name of configuration we want to retrieve

        :param config_file: A path to file which should be used to retrieve

          configuration from (might also be a list of file paths)

        """

        if config_file is None:

            config_file = []

            config_file = [config_file]

        config = self._repo.ui

        if config_file:

            config = mercurial.ui.ui()

            for path in config_file:

                config.readconfig(safe_bytes(path))

        return config.config(safe_bytes(section), safe_bytes(name))

    def get_user_name(self, config_file=None):

        """

        Returns user's name from global configuration file.

        :param config_file: A path to file which should be used to retrieve

          configuration from (might also be a list of file paths)

        """

        username = self.get_config_value('ui', 'username', config_file=config_file)

        if username:

            return author_name(username)

        return None

    def get_user_email(self, config_file=None):

        """

        Returns user's email from global configuration file.

"""

import datetime

import re

import time

def makedate():

    lt = time.localtime()

    if lt[8] == 1 and time.daylight:

        tz = time.altzone

    else:

        tz = time.timezone

    return time.mktime(lt), tz

def aslist(obj, sep=None, strip=True):

    """

    Returns given string separated by sep as list

    :param obj:

    :param sep:

    :param strip:

    """

        lst = obj.split(sep)

        if strip:

            lst = [v.strip() for v in lst]

        return lst

    elif isinstance(obj, (list, tuple)):

        return obj

    elif obj is None:

        return []

    else:

        return [obj]

def date_fromtimestamp(unixts, tzoffset=0):

    """

    Makes a local datetime object out of unix timestamp

    :param unixts:

    :param tzoffset:

    """

    return datetime.datetime.fromtimestamp(float(unixts))

def safe_int(val, default=None):

    @classmethod

    def get(cls, id_):

        if id_:

            return cls.query().get(id_)

    @classmethod

    def guess_instance(cls, value, callback=None):

        """Haphazardly attempt to convert `value` to a `cls` instance.

        If `value` is None or already a `cls` instance, return it. If `value`

        is a number (or looks like one if you squint just right), assume it's

        a database primary key and let SQLAlchemy sort things out. Otherwise,

        fall back to resolving it using `callback` (if specified); this could

        e.g. be a function that looks up instances by name (though that won't

        work if the name begins with a digit). Otherwise, raise Exception.

        """

        if value is None:

            return None

        if isinstance(value, cls):

            return value

        if isinstance(value, int):

            return cls.get(value)

            return cls.get(int(value))

        if callback is not None:

            return callback(value)

        raise Exception(

            'given object must be int, long or Instance of %s '

            'got %s, no callback provided' % (cls, type(value))

        )

    @classmethod

    def get_or_404(cls, id_):

        try:

            id_ = int(id_)

        except (TypeError, ValueError):

            raise HTTPNotFound

        res = cls.query().get(id_)

        if res is None:

            raise HTTPNotFound

        return res

    @classmethod

    def delete(cls, id_):

        obj = cls.query().get(id_)

                    perm_checker=HasRepoGroupPermissionLevel,

                    extra_kwargs=extra_kwargs)

class UserGroupList(_PermCheckIterator):

    def __init__(self, db_user_group_list, perm_level, extra_kwargs=None):

        super(UserGroupList, self).__init__(obj_list=db_user_group_list,

                    obj_attr='users_group_name', perm_set=[perm_level],

                    perm_checker=HasUserGroupPermissionLevel,

                    extra_kwargs=extra_kwargs)

class ScmModel(object):

    """

    Generic Scm Model

    """

    def __get_repo(self, instance):

        cls = Repository

        if isinstance(instance, cls):

            return instance

        elif isinstance(instance, int):

            return cls.get(instance)

            if instance.isdigit():

                return cls.get(int(instance))

            return cls.get_by_repo_name(instance)

        elif instance is not None:

            raise Exception('given object must be int, basestr or Instance'

                            ' of %s got %s' % (type(cls), type(instance)))

    @LazyProperty

    def repos_path(self):

        """

        Gets the repositories root path from database

        """

        q = Ui.query().filter(Ui.ui_key == '/').one()

        return q.ui_value

    def repo_scan(self, repos_path=None):

        """

        Listing of repositories in given path. This path should not be a

        repository itself. Return a dictionary of repository objects mapping to

        vcs instances.

        :param repos_path: path to directory containing repositories

        """

        Commits specified nodes to repo.

        :param user: Kallithea User object or user_id, the committer

        :param repo: Kallithea Repository object

        :param message: commit message

        :param nodes: mapping {filename:{'content':content},...}

        :param parent_cs: parent changeset, can be empty than it's initial commit

        :param author: author of commit, cna be different that committer only for git

        :param trigger_push_hook: trigger push hooks

        :returns: new committed changeset

        """

        user = User.guess_instance(user)

        scm_instance = repo.scm_instance_no_cache()

        processed_nodes = []

        for f_path in nodes:

            content = nodes[f_path]['content']

            f_path = self._sanitize_path(f_path)

            f_path = safe_str(f_path)

            # decoding here will force that we have proper encoded values

            # in any other case this will throw exceptions and deny commit

                content = safe_str(content)

            else:

                content = content.read()

            processed_nodes.append((f_path, content))

        message = safe_unicode(message)

        committer = user.full_contact

        author = safe_unicode(author) if author else committer

        IMC = self._get_IMC_module(scm_instance.alias)

        imc = IMC(scm_instance)

        if not parent_cs:

            parent_cs = EmptyChangeset(alias=scm_instance.alias)

        if isinstance(parent_cs, EmptyChangeset):

            # EmptyChangeset means we we're editing empty repository

            parents = None

        else:

            parents = [parent_cs]

        # add multiple nodes

        for path, content in processed_nodes:

            imc.add(FileNode(path, content=content))

            new_user_group.user_group_description = description

            new_user_group.users_group_active = active

            if group_data:

                new_user_group.group_data = group_data

            Session().add(new_user_group)

            self._create_default_perms(new_user_group)

            self.grant_user_permission(user_group=new_user_group,

                                       user=owner, perm='usergroup.admin')

            return new_user_group

        except Exception:

            log.error(traceback.format_exc())

            raise

    def update(self, user_group, form_data):

        try:

            user_group = UserGroup.guess_instance(user_group)

            for k, v in form_data.items():

                if k == 'users_group_members':

                    members_list = []

                    if v:

                        for u_id in set(v):

                            member = UserGroupMember(user_group.users_group_id, u_id)

                            members_list.append(member)

                            Session().add(member)

                    user_group.members = members_list

                setattr(user_group, k, v)

            # Flush to make db assign users_group_member_id to newly

            # created UserGroupMembers.

            Session().flush()

        except Exception:

            log.error(traceback.format_exc())

            raise

    def delete(self, user_group, force=False):

        """

        Deletes user group, unless force flag is used

        raises exception if there are members in that group, else deletes

        group and users

        :param user_group:

        :param force:

        """

        user_group = UserGroup.guess_instance(user_group)

if not BASE_URI.endswith('/'):

    BASE_URI += '/'

print('Crawling @ %s' % BASE_URI)

BASE_URI += '%s'

PROJECT_PATH = os.path.join('/', 'home', 'username', 'repos')

PROJECTS = [

    # 'linux-magx-pbranch',

    'CPython',

    'kallithea',

]

cj = cookielib.FileCookieJar(os.path.join(tempfile.gettempdir(), 'rc_test_cookie.txt'))

o = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj))

o.addheaders = [

    ('User-agent', 'kallithea-crawler'),

    ('Accept-Language', 'en - us, en;q = 0.5')

]

urllib2.install_opener(o)

def _get_repo(proj):

        repo = vcs.get_repo(os.path.join(PROJECT_PATH, proj))

        proj = proj

    else:

        repo = proj

        proj = repo.name

    return repo, proj

def test_changelog_walk(proj, pages=100):

    repo, proj = _get_repo(proj)

    total_time = 0

    for i in range(1, pages):

        page = '/'.join((proj, 'changelog',))

        full_uri = (BASE_URI % page) + '?' + urllib.urlencode({'page': i})

        s = time.time()

        f = o.open(full_uri)

        assert f.url == full_uri, 'URL:%s does not match %s' % (f.url, full_uri)