import logging

import os

import platform

import sys

import alembic.config

import mercurial

import tg

from alembic.migration import MigrationContext

from alembic.script.base import ScriptDirectory

from sqlalchemy import create_engine

from tg.configuration import AppConfig

from tg.support.converters import asbool

import kallithea.lib.locale

import kallithea.model.base

import kallithea.model.meta

from kallithea.lib.middleware.https_fixup import HttpsFixup

from kallithea.lib.middleware.permanent_repo_url import PermanentRepoUrl

from kallithea.lib.middleware.simplegit import SimpleGit

from kallithea.lib.middleware.simplehg import SimpleHg

from kallithea.lib.middleware.wrapper import RequestWrapper

from kallithea.lib.utils import check_git_version, load_rcextensions, make_ui, set_app_settings, set_indexer_config, set_vcs_config

log = logging.getLogger(__name__)

class KallitheaAppConfig(AppConfig):

    # Note: AppConfig has a misleading name, as it's not the application

    # configuration, but the application configurator. The AppConfig values are

    # used as a template to create the actual configuration, which might

    # overwrite or extend the one provided by the configurator template.

    # To make it clear, AppConfig creates the config and sets into it the same

    # values that AppConfig itself has. Then the values from the config file and

    # gearbox options are loaded and merged into the configuration. Then an

    # after_init_config(conf) method of AppConfig is called for any change that

    # might depend on options provided by configuration files.

    def __init__(self):

        super(KallitheaAppConfig, self).__init__()

        self['package'] = kallithea

        self['prefer_toscawidgets2'] = False

        self['use_toscawidgets'] = False

        script_dir = ScriptDirectory.from_config(alembic_cfg)

        available_heads = sorted(script_dir.get_heads())

        engine = create_engine(dbconf)

        with engine.connect() as conn:

            context = MigrationContext.configure(conn)

            current_heads = sorted(str(s) for s in context.get_current_heads())

        if current_heads != available_heads:

            log.error('Failed to run Kallithea:\n\n'

                      'The database version does not match the Kallithea version.\n'

                      'Please read the documentation on how to upgrade or downgrade the database.\n'

                      'Current database version id(s): %s\n'

                      'Expected database version id(s): %s\n'

                      'If you are a developer and you know what you are doing, you can add `ignore_alembic_revision = True` '

                      'to your .ini file to skip the check.\n' % (' '.join(current_heads), ' '.join(available_heads)))

            sys.exit(1)

    # store some globals into kallithea

    kallithea.CELERY_ON = str2bool(config.get('use_celery'))

    kallithea.CELERY_EAGER = str2bool(config.get('celery.always.eager'))

    kallithea.CONFIG = config

    load_rcextensions(root_path=config['here'])

    config['base_path'] = repos_path

    set_app_settings(config)

    instance_id = kallithea.CONFIG.get('instance_id', '*')

    if instance_id == '*':

        instance_id = '%s-%s' % (platform.uname()[1], os.getpid())

        kallithea.CONFIG['instance_id'] = instance_id

    # update kallithea.CONFIG with the meanwhile changed 'config'

    kallithea.CONFIG.update(config)

    # configure vcs and indexer libraries (they are supposed to be independent

    # as much as possible and thus avoid importing tg.config or

    # kallithea.CONFIG).

    set_vcs_config(kallithea.CONFIG)

    set_indexer_config(kallithea.CONFIG)

    check_git_version()

    kallithea.model.meta.Session.remove()

tg.hooks.register('configure_new_app', setup_configuration)

~~~~~~~~~~~~~~~~~~~~~~~~~~

Home controller for Kallithea

This file was forked by the Kallithea project in July 2014.

Original author and date, and relevant copyright and licensing information is below:

:created_on: Feb 18, 2010

:author: marcink

:copyright: (c) 2013 RhodeCode GmbH, and others.

:license: GPLv3, see LICENSE.md for more details.

"""

import logging

from sqlalchemy import or_

from tg import request

from tg import tmpl_context as c

from tg.i18n import ugettext as _

from webob.exc import HTTPBadRequest

from kallithea.lib import helpers as h

from kallithea.lib.auth import HasRepoPermissionLevelDecorator, LoginRequired

from kallithea.lib.base import BaseController, jsonify, render

from kallithea.model.db import RepoGroup, Repository, User, UserGroup

from kallithea.model.repo import RepoModel

from kallithea.model.scm import UserGroupList

log = logging.getLogger(__name__)

class HomeController(BaseController):

    def about(self):

        return render('/about.html')

    @LoginRequired(allow_default_user=True)

    def index(self):

        c.group = None

        repo_groups_list = self.scm_model.get_repo_groups()

        repos_list = Repository.query(sorted=True).filter_by(group=None).all()

        c.data = RepoModel().get_repos_as_dict(repos_list,

                                               repo_groups_list=repo_groups_list,

                                               short_name=True)

            for res_dict in res:

                for child in (res_dict['children']):

                    child['obj'].pop('_changeset_cache', None)  # bytes cannot be encoded in json ... but this value isn't relevant on client side at all ...

            data = {

                'more': False,

                'results': res,

            }

            return data

        else:

            raise HTTPBadRequest()

    @LoginRequired(allow_default_user=True)

    @HasRepoPermissionLevelDecorator('read')

    @jsonify

    def repo_refs_data(self, repo_name):

        repo = Repository.get_by_repo_name(repo_name).scm_instance

        res = []

        _branches = repo.branches.items()

        if _branches:

            res.append({

                'text': _('Branch'),

            })

        _closed_branches = repo.closed_branches.items()

        if _closed_branches:

            res.append({

                'text': _('Closed Branches'),

            })

        _tags = repo.tags.items()

        if _tags:

            res.append({

                'text': _('Tag'),

            })

        _bookmarks = repo.bookmarks.items()

        if _bookmarks:

            res.append({

                'text': _('Bookmark'),

            })

        data = {

            'more': False,

            'results': res

        }

        return data

    @LoginRequired()

    @jsonify

    def users_and_groups_data(self):

        """

        Returns 'results' with a list of users and user groups.

        You can either use the 'key' GET parameter to get a user by providing

        the exact user key or you can use the 'query' parameter to

        search for users by user key, first name and last name.

        'types' defaults to just 'users' but can be set to 'users,groups' to

        get both users and groups.

        No more than 500 results (of each kind) will be returned.

        """

        types = request.GET.get('types', 'users').split(',')

        key = request.GET.get('key', '')

        query = request.GET.get('query', '')

        results = []

        self.authfunc = authfunc

        self._rc_auth_http_code = auth_http_code

    def build_authentication(self, environ):

        head = paste.httpheaders.WWW_AUTHENTICATE.tuples('Basic realm="%s"' % self.realm)

        # Consume the whole body before sending a response

        try:

            request_body_size = int(environ.get('CONTENT_LENGTH', 0))

        except (ValueError):

            request_body_size = 0

        environ['wsgi.input'].read(request_body_size)

        if self._rc_auth_http_code and self._rc_auth_http_code == '403':

            # return 403 if alternative http return code is specified in

            # Kallithea config

            return paste.httpexceptions.HTTPForbidden(headers=head)

        return paste.httpexceptions.HTTPUnauthorized(headers=head)

    def authenticate(self, environ):

        authorization = paste.httpheaders.AUTHORIZATION(environ)

        if not authorization:

            return self.build_authentication(environ)

        (authmeth, auth) = authorization.split(' ', 1)

        if 'basic' != authmeth.lower():

            return self.build_authentication(environ)

        _parts = auth.split(':', 1)

        if len(_parts) == 2:

            username, password = _parts

            if self.authfunc(username, password, environ) is not None:

                return username

        return self.build_authentication(environ)

    __call__ = authenticate

class BaseVCSController(object):

    """Base controller for handling Mercurial/Git protocol requests

    (coming from a VCS client, and not a browser).

    """

    scm_alias = None # 'hg' / 'git'

    def __init__(self, application, config):

        self.application = application

        self.config = config

        # base path of repo locations

        self.basepath = self.config['base_path']

        # authenticate this VCS request using the authentication modules

        self.authenticate = BasicAuth('', auth_modules.authenticate,

_header_next_check = re.compile(br'''(?!@)(?!literal )(?!delta )''')

def _get_header(vcs, diff_chunk):

    """

    Parses a Git diff for a single file (header and chunks) and returns a tuple with:

    1. A dict with meta info:

        a_path, b_path, similarity_index, rename_from, rename_to,

        old_mode, new_mode, new_file_mode, deleted_file_mode,

        a_blob_id, b_blob_id, b_mode, a_file, b_file

    2. An iterator yielding lines with simple HTML markup.

    """

    match = None

    if vcs == 'git':

        match = _git_header_re.match(diff_chunk)

    elif vcs == 'hg':

        match = _hg_header_re.match(diff_chunk)

    if match is None:

        raise Exception('diff not recognized as valid %s diff' % vcs)

    rest = diff_chunk[match.end():]

    if rest and _header_next_check.match(rest):

    diff_lines = (_escaper(m.group(0)) for m in re.finditer(br'.*\n|.+$', rest)) # don't split on \r as str.splitlines do

    return meta_info, diff_lines

_chunk_re = re.compile(r'^@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))? @@(.*)')

_newline_marker = re.compile(r'^\\ No newline at end of file')

def _parse_lines(diff_lines):

    """

    Given an iterator of diff body lines, parse them and return a dict per

    line and added/removed totals.

    """

    added = deleted = 0

    old_line = old_end = new_line = new_end = None

    chunks = []

    try:

        line = next(diff_lines)

        while True:

            lines = []

            chunks.append(lines)

# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""

kallithea.lib.hooks

~~~~~~~~~~~~~~~~~~~

Hooks run by Kallithea

This file was forked by the Kallithea project in July 2014.

Original author and date, and relevant copyright and licensing information is below:

:created_on: Aug 6, 2010

:author: marcink

:copyright: (c) 2013 RhodeCode GmbH, and others.

:license: GPLv3, see LICENSE.md for more details.

"""

import os

import sys

import time

import mercurial.scmutil

from kallithea.lib import helpers as h

from kallithea.lib.exceptions import UserCreationError

from kallithea.lib.utils import action_logger, make_ui

from kallithea.lib.vcs.backends.base import EmptyChangeset

from kallithea.model.db import Repository, User

def _get_scm_size(alias, root_path):

    if not alias.startswith('.'):

        alias += '.'

    size_scm, size_root = 0, 0

    for path, dirs, files in os.walk(root_path):

        if path.find(alias) != -1:

            for f in files:

                try:

                    size_scm += os.path.getsize(os.path.join(path, f))

                except OSError:

                    pass

        else:

            for f in files:

                try:

                    size_root += os.path.getsize(os.path.join(path, f))

                except OSError:

                    pass

    size_scm_f = h.format_byte_size(size_scm)

    size_root_f = h.format_byte_size(size_root)

    size_total_f = h.format_byte_size(size_root + size_scm)

    return size_scm_f, size_root_f, size_total_f

def repo_size(ui, repo, hooktype=None, **kwargs):

    """Show size of Mercurial repository, to be called after push."""

    last_cs = repo[len(repo) - 1]

    msg = ('Repository size .hg: %s Checkout: %s Total: %s\n'

           'Last revision is now r%s:%s\n') % (

        size_hg_f, size_root_f, size_total_f, last_cs.rev(), ascii_str(last_cs.hex())[:12]

    )

    ui.status(safe_bytes(msg))

def log_pull_action(ui, repo, **kwargs):

    """Logs user last pull action

    Called as Mercurial hook outgoing.pull_logger or from Kallithea before invoking Git.

    Does *not* use the action from the hook environment but is always 'pull'.

    """

    ex = get_hook_environment()

    user = User.get_by_username(ex.username)

    action = 'pull'

    action_logger(user, action, ex.repository, ex.ip, commit=True)

    # extension hook call

    from kallithea import EXTENSIONS

This file was forked by the Kallithea project in July 2014.

Original author and date, and relevant copyright and licensing information is below:

:created_on: Apr 18, 2010

:author: marcink

:copyright: (c) 2013 RhodeCode GmbH, and others.

:license: GPLv3, see LICENSE.md for more details.

"""

import datetime

import logging

import os

import re

import sys

import traceback

from distutils.version import StrictVersion

import beaker.cache

import mercurial.config

import mercurial.ui

from tg.i18n import ugettext as _

import kallithea.config.conf

from kallithea.lib.exceptions import HgsubversionImportError

from kallithea.lib.vcs.backends.git.repository import GitRepository

from kallithea.lib.vcs.backends.hg.repository import MercurialRepository

from kallithea.lib.vcs.conf import settings

from kallithea.lib.vcs.exceptions import RepositoryError, VCSError

from kallithea.lib.vcs.utils.fakemod import create_module

from kallithea.lib.vcs.utils.helpers import get_scm

from kallithea.model import meta

from kallithea.model.db import RepoGroup, Repository, Setting, Ui, User, UserGroup, UserLog

log = logging.getLogger(__name__)

REMOVED_REPO_PAT = re.compile(r'rm__\d{8}_\d{6}_\d{6}_.*')

#==============================================================================

# PERM DECORATOR HELPERS FOR EXTRACTING NAMES FOR PERM CHECKS

#==============================================================================

def get_repo_slug(request):

    _repo = request.environ['pylons.routes_dict'].get('repo_name')

    if _repo:

        _repo = _repo.rstrip('/')

    return _repo

#==============================================================================

git_req_ver = StrictVersion('1.7.4')

def check_git_version():

    """

    Checks what version of git is installed on the system, and raise a system exit

    if it's too old for Kallithea to work properly.

    """

    if 'git' not in kallithea.BACKENDS:

        return None

    if not settings.GIT_EXECUTABLE_PATH:

        log.warning('No git executable configured - check "git_path" in the ini file.')

        return None

    try:

        stdout, stderr = GitRepository._run_git_command(['--version'])

    except RepositoryError as e:

        # message will already have been logged as error

        log.warning('No working git executable found - check "git_path" in the ini file.')

        return None

    if stderr:

    if not stdout:

        log.warning('No working git executable found - check "git_path" in the ini file.')

        return None

    m = re.search(r"\d+.\d+.\d+", output)

    if m:

        ver = StrictVersion(m.group(0))

        log.debug('Git executable: "%s", version %s (parsed from: "%s")',

                  settings.GIT_EXECUTABLE_PATH, ver, output)

        if ver < git_req_ver:

            log.error('Kallithea detected %s version %s, which is too old '

                      'for the system to function properly. '

                      'Please upgrade to version %s or later. '

                      'If you strictly need Mercurial repositories, you can '

                      'clear the "git_path" setting in the ini file.',

                      settings.GIT_EXECUTABLE_PATH, ver, git_req_ver)

            log.error("Terminating ...")

            sys.exit(1)

    else:

        ver = StrictVersion('0.0.0')

        log.warning('Error finding version number in "%s --version" stdout:\n%s',

                    settings.GIT_EXECUTABLE_PATH, output)

    return ver

#===============================================================================

# CACHE RELATED METHODS

        """

        return self.changed, self.added, self.removed

    @LazyProperty

    def tags(self):

        _tags = []

        for tname, tsha in self.repository.tags.items():

            if tsha == self.raw_id:

                _tags.append(tname)

        return _tags

    @LazyProperty

    def branch(self):

        # Note: This function will return one branch name for the changeset -

        # that might not make sense in Git where branches() is a better match

        # for the basic model

        heads = self.repository._heads(reverse=False)

        ref = heads.get(self._commit.id)

        if ref:

            return safe_str(ref)

    @LazyProperty

    def branches(self):

        heads = self.repository._heads(reverse=True)

    def _fix_path(self, path):

        """

        Paths are stored without trailing slash so we need to get rid off it if

        needed.

        """

        if path.endswith('/'):

            path = path.rstrip('/')

        return path

    def _get_id_for_path(self, path):

        # FIXME: Please, spare a couple of minutes and make those codes cleaner;

        if path not in self._paths:

            path = path.strip('/')

            # set root tree

            tree = self.repository._repo[self._tree_id]

            if path == '':

                self._paths[''] = tree.id

                return tree.id

            splitted = path.split('/')

            dirs, name = splitted[:-1], splitted[-1]

            curdir = ''

            # initially extract things from root dir

            for item, stat, id in tree.items():

                if curdir:

                self._paths[name] = id

                self._stat_modes[name] = stat

            for dir in dirs:

                if curdir:

                    curdir = '/'.join((curdir, dir))

                else:

                    curdir = dir

                dir_id = None

                for item, stat, id in tree.items():

                        dir_id = id

                if dir_id:

                    # Update tree

                    tree = self.repository._repo[dir_id]

                    if not isinstance(tree, objects.Tree):

                        raise ChangesetError('%s is not a directory' % curdir)

                else:

                    raise ChangesetError('%s have not been found' % curdir)

                # cache all items from the given traversed tree

                for item, stat, id in tree.items():

                    if curdir:

                    self._paths[name] = id

                    self._stat_modes[name] = stat

            if path not in self._paths:

                raise NodeDoesNotExistError("There is no file nor directory "

                    "at the given path '%s' at revision %s"

                    % (path, self.short_id))

        return self._paths[path]

    def _get_kind(self, path):

        obj = self.repository._repo[self._get_id_for_path(path)]

        if isinstance(obj, objects.Blob):

            return NodeKind.FILE

        elif isinstance(obj, objects.Tree):

            return NodeKind.DIR

    def _get_filectx(self, path):

        path = self._fix_path(path)

        if self._get_kind(path) != NodeKind.FILE:

            raise ChangesetError("File does not exist for revision %s at "

                " '%s'" % (self.raw_id, path))

        return path

    def _get_file_nodes(self):

        return chain(*(t[2] for t in self.walk()))

        chunk = popen.stdout.read(buffer_size)

        while chunk:

            stream.write(chunk)

            chunk = popen.stdout.read(buffer_size)

        # Make sure all descriptors would be read

        popen.communicate()

    def get_nodes(self, path):

        """

        Returns combined ``DirNode`` and ``FileNode`` objects list representing

        state of changeset at the given ``path``. If node at the given ``path``

        is not instance of ``DirNode``, ChangesetError would be raised.

        """

        if self._get_kind(path) != NodeKind.DIR:

            raise ChangesetError("Directory does not exist for revision %s at "

                " '%s'" % (self.revision, path))

        path = self._fix_path(path)

        id = self._get_id_for_path(path)

        tree = self.repository._repo[id]

        dirnodes = []

        filenodes = []

        als = self.repository.alias

        for name, stat, id in tree.items():

            if path != '':

            if objects.S_ISGITLINK(stat):

                root_tree = self.repository._repo[self._tree_id]

                cf = ConfigFile.from_file(BytesIO(self.repository._repo.get_object(root_tree[b'.gitmodules'][1]).data))

                url = ascii_str(cf.get(('submodule', obj_path), 'url'))

                dirnodes.append(SubModuleNode(obj_path, url=url, changeset=ascii_str(id),

                                              alias=als))

                continue

            obj = self.repository._repo.get_object(id)

            if obj_path not in self._stat_modes:

                self._stat_modes[obj_path] = stat

            if isinstance(obj, objects.Tree):

                dirnodes.append(DirNode(obj_path, changeset=self))

            elif isinstance(obj, objects.Blob):

                filenodes.append(FileNode(obj_path, changeset=self, mode=stat))

            else:

                raise ChangesetError("Requested object should be Tree "

                                     "or Blob, is %r" % type(obj))

        nodes = dirnodes + filenodes

        for node in nodes:

            if node.path not in self.nodes:

                self.nodes[node.path] = node

        nodes.sort()

        return nodes

        """

        Gets a fast accessible file changes for given changeset

        """

        added, modified, deleted = self._changes_cache

        return list(added.union(modified).union(deleted))

    @LazyProperty

    def _changes_cache(self):

        added = set()

        modified = set()

        deleted = set()

        _r = self.repository._repo

        parents = self.parents

        if not self.parents:

            parents = [EmptyChangeset()]

        for parent in parents:

            if isinstance(parent, EmptyChangeset):

                oid = None

            else:

                oid = _r[parent._commit.id].tree

            changes = _r.object_store.tree_changes(oid, _r[self._commit.id].tree)

            for (oldpath, newpath), (_, _), (_, _) in changes:

                if newpath and oldpath:

                elif newpath and not oldpath:

                elif not newpath and oldpath:

        return added, modified, deleted

    def _get_paths_for_status(self, status):

        """

        Returns sorted list of paths for given ``status``.

        :param status: one of: *added*, *modified* or *deleted*

        """

        added, modified, deleted = self._changes_cache

        return sorted({

            'added': list(added),

            'modified': list(modified),

            'deleted': list(deleted)}[status]

        )

    @LazyProperty

    def added(self):

        """

        Returns list of added ``FileNode`` objects.

        """

        if not self.parents:

            return list(self._get_file_nodes())

        return AddedFileNodesGenerator([n for n in

                                self._get_paths_for_status('added')], self)

        except RepositoryError:

            idx_loc = '' if self.bare else '.git'

            # fallback to filesystem

            in_path = os.path.join(self.path, idx_loc, "index")

            he_path = os.path.join(self.path, idx_loc, "HEAD")

            if os.path.exists(in_path):

                return os.stat(in_path).st_mtime

            else:

                return os.stat(he_path).st_mtime

    @LazyProperty

    def description(self):

        return safe_str(self._repo.get_description() or b'unknown')

    @LazyProperty

    def contact(self):

        undefined_contact = u'Unknown'

        return undefined_contact

    @property

    def branches(self):

        if not self.revisions:

            return {}

        sortkey = lambda ctx: ctx[0]

                     for key, (sha, type_) in self._parsed_refs.items() if type_ == b'H']

        return OrderedDict(sorted(_branches, key=sortkey, reverse=False))

    @LazyProperty

    def closed_branches(self):

        return {}

    @LazyProperty

    def tags(self):

        return self._get_tags()

    def _get_tags(self):

        if not self.revisions:

            return {}

        sortkey = lambda ctx: ctx[0]

                 for key, (sha, type_) in self._parsed_refs.items() if type_ == b'T']

        return OrderedDict(sorted(_tags, key=sortkey, reverse=True))

    def tag(self, name, user, revision=None, message=None, date=None,

            **kwargs):

        """

        Creates and returns a tag for the given ``revision``.

        :param name: name for new tag

        :param user: full username, i.e.: "Joe Doe <joe.doe@example.com>"

        :param revision: changeset id for which new tag would be created

        :param message: message of the tag's commit

        :param date: date of tag's commit

        :raises TagAlreadyExistError: if tag with same name already exists

        """

        if name in self.tags:

            raise TagAlreadyExistError("Tag %s already exists" % name)

        changeset = self.get_changeset(revision)

        message = message or "Added tag %s for commit %s" % (name,

            changeset.raw_id)

        self._repo.refs[b"refs/tags/%s" % safe_bytes(name)] = changeset._commit.id

        self._parsed_refs = self._get_parsed_refs()

        self.tags = self._get_tags()

        return changeset

    def remove_tag(self, name, user, message=None, date=None):

        """

        Removes tag with the given ``name``.

        :param name: name of the tag to be removed

        :param user: full username, i.e.: "Joe Doe <joe.doe@example.com>"

        :param message: message of the tag's removal commit

        :param date: date of tag's removal commit

        :raises TagDoesNotExistError: if tag with given name does not exists

        """

        if name not in self.tags:

            raise TagDoesNotExistError("Tag %s does not exist" % name)

        # self._repo.refs is a DiskRefsContainer, and .path gives the full absolute path of '.git'

        try:

            os.remove(tagpath)

            self._parsed_refs = self._get_parsed_refs()

            self.tags = self._get_tags()

        except OSError as e:

            raise RepositoryError(e.strerror)

    @LazyProperty