if c.gist.is_expired:

            log.error('Gist expired at %s',

                      time_to_datetime(c.gist.gist_expires))

            raise HTTPNotFound()

        try:

            c.file_changeset, c.files = GistModel().get_gist_files(gist_id,

                                                            revision=revision)

        except VCSError:

            log.error(traceback.format_exc())

            raise HTTPNotFound()

        if format == 'raw':

            response.content_type = 'text/plain'

            return content

        return render('admin/gists/show.html')

    @LoginRequired()

    def edit(self, gist_id, format='html'):

        c.gist = Gist.get_or_404(gist_id)

        if c.gist.is_expired:

            log.error('Gist expired at %s',

                      time_to_datetime(c.gist.gist_expires))

            raise HTTPNotFound()

                h.flash(_('Cannot compare repositories without using common ancestor'), category='error')

                raise HTTPBadRequest

            rev1 = c.a_rev

        diff_limit = None if fulldiff else self.cut_off_limit

        log.debug('running diff between %s and %s in %s',

                  rev1, c.cs_rev, org_repo.scm_instance.path)

        raw_diff = diffs.get_diff(org_repo.scm_instance, rev1=rev1, rev2=c.cs_rev,

                                      ignore_whitespace=ignore_whitespace,

                                      context=line_context)

        c.limited_diff = diff_processor.limited_diff

        c.file_diff_data = []

        c.lines_added = 0

        c.lines_deleted = 0

        for f in diff_processor.parsed:

            st = f['stats']

            c.lines_added += st['added']

            c.lines_deleted += st['deleted']

            filename = f['filename']

            fid = h.FID('', filename)

            html_diff = diffs.as_html(enable_comments=False, parsed_lines=[f])

            c.file_diff_data.append((fid, None, f['operation'], f['old_filename'], filename, html_diff, st))

        # rev link

        _url = h.canonical_url('changeset_home', repo_name=c.db_repo.repo_name,

                   revision=cs.raw_id)

        desc_msg.append('changeset: <a href="%s">%s</a>' % (_url, cs.raw_id[:8]))

        desc_msg.append('<pre>')

        desc_msg.append(h.urlify_text(cs.message))

        desc_msg.append('\n')

        desc_msg.extend(changes)

        if str2bool(CONFIG.get('rss_include_diff', False)):

            desc_msg.append('\n\n')

        desc_msg.append('</pre>')

        return [safe_unicode(chunk) for chunk in desc_msg]

    def _feed(self, repo_name, feeder):

        """Produce a simple feed"""

        @cache_region('long_term', '_get_feed_from_cache')

        def _get_feed_from_cache(*_cache_keys):  # parameters are not really used - only as caching key

            header = dict(

                title=_('%s %s feed') % (c.site_name, repo_name),

                link=h.canonical_url('summary_home', repo_name=repo_name),

                description=_('Changes on %s repository') % repo_name,

from tg import tmpl_context as c

from tg.i18n import ugettext as _

from webob.exc import HTTPFound, HTTPNotFound

from kallithea.config.routing import url

from kallithea.controllers.changeset import _context_url, _ignorews_url, anchor_url, get_ignore_ws, get_line_ctx

from kallithea.lib import diffs

from kallithea.lib import helpers as h

from kallithea.lib.auth import HasRepoPermissionLevelDecorator, LoginRequired

from kallithea.lib.base import BaseRepoController, jsonify, render

from kallithea.lib.exceptions import NonRelativePathError

from kallithea.lib.utils import action_logger

from kallithea.lib.vcs.backends.base import EmptyChangeset

from kallithea.lib.vcs.conf import settings

from kallithea.lib.vcs.exceptions import (

    ChangesetDoesNotExistError, ChangesetError, EmptyRepositoryError, ImproperArchiveTypeError, NodeAlreadyExistsError, NodeDoesNotExistError, NodeError, RepositoryError, VCSError)

from kallithea.lib.vcs.nodes import FileNode

from kallithea.model.db import Repository

from kallithea.model.repo import RepoModel

from kallithea.model.scm import ScmModel

log = logging.getLogger(__name__)

        r_post = request.POST

        c.cs = self.__get_cs(revision)

        c.file = self.__get_filenode(c.cs, f_path)

        if c.file.is_binary:

            raise HTTPFound(location=url('files_home', repo_name=c.repo_name,

                            revision=c.cs.raw_id, f_path=f_path))

        c.default_message = _('Edited file %s via Kallithea') % (f_path)

        c.f_path = f_path

        if r_post:

            sl = old_content.splitlines(1)

            first_line = sl[0] if sl else ''

            # modes:  0 - Unix, 1 - Mac, 2 - DOS

            mode = detect_mode(first_line, 0)

            content = convert_line_endings(r_post.get('content', ''), mode)

            message = r_post.get('message') or c.default_message

            author = request.authuser.full_contact

            if content == old_content:

                h.flash(_('No changes'), category='warning')

                raise HTTPFound(location=url('changeset_home', repo_name=c.repo_name,

        c.context_url = _context_url

        fulldiff = request.GET.get('fulldiff')

        diff_limit = None if fulldiff else self.cut_off_limit

        # we swap org/other ref since we run a simple diff on one repo

        log.debug('running diff between %s and %s in %s',

                  c.a_rev, c.cs_rev, org_scm_instance.path)

        try:

            raw_diff = diffs.get_diff(org_scm_instance, rev1=safe_str(c.a_rev), rev2=safe_str(c.cs_rev),

                                      ignore_whitespace=ignore_whitespace, context=line_context)

        except ChangesetDoesNotExistError:

            raw_diff = _("The diff can't be shown - the PR revisions could not be found.")

        c.limited_diff = diff_processor.limited_diff

        c.file_diff_data = []

        c.lines_added = 0

        c.lines_deleted = 0

        for f in diff_processor.parsed:

            st = f['stats']

            c.lines_added += st['added']

            c.lines_deleted += st['deleted']

            filename = f['filename']

            fid = h.FID('', filename)

            html_diff = diffs.as_html(enable_comments=True, parsed_lines=[f])

from tg import tmpl_context as c

from tg.i18n import ugettext as _

from webob.exc import HTTPBadRequest

import kallithea.lib.helpers as h

from kallithea.config.conf import ALL_EXTS, ALL_READMES, LANGUAGES_EXTENSIONS_MAP

from kallithea.lib.auth import HasRepoPermissionLevelDecorator, LoginRequired

from kallithea.lib.base import BaseRepoController, jsonify, render

from kallithea.lib.celerylib.tasks import get_commits_stats

from kallithea.lib.compat import json

from kallithea.lib.markup_renderer import MarkupRenderer

from kallithea.lib.page import Page

from kallithea.lib.vcs.backends.base import EmptyChangeset

from kallithea.lib.vcs.exceptions import ChangesetError, EmptyRepositoryError, NodeDoesNotExistError

from kallithea.lib.vcs.nodes import FileNode

from kallithea.model.db import Statistics

log = logging.getLogger(__name__)

README_FILES = [''.join([x[0][0], x[1][0]]) for x in

                    sorted(list(itertools.product(ALL_READMES, ALL_EXTS)),

                           key=lambda y:y[0][1] + y[1][1])]

                cs = db_repo.get_landing_changeset()

                if isinstance(cs, EmptyChangeset):

                    raise EmptyRepositoryError()

                renderer = MarkupRenderer()

                for f in README_FILES:

                    try:

                        readme = cs.get_node(f)

                        if not isinstance(readme, FileNode):

                            continue

                        readme_file = f

                        log.debug('Found README file `%s` rendering...',

                                  readme_file)

                                                      filename=f)

                        break

                    except NodeDoesNotExistError:

                        continue

            except ChangesetError:

                log.error(traceback.format_exc())

                pass

            except EmptyRepositoryError:

                pass

            return readme_data, readme_file

Original author and date, and relevant copyright and licensing information is below:

:created_on: Dec 4, 2011

:author: marcink

:copyright: (c) 2013 RhodeCode GmbH, and others.

:license: GPLv3, see LICENSE.md for more details.

"""

from pygments import highlight

from pygments.formatters import HtmlFormatter

from kallithea.lib.vcs.exceptions import VCSError

from kallithea.lib.vcs.nodes import FileNode

def annotate_highlight(filenode, annotate_from_changeset_func=None,

        order=None, headers=None, **options):

    """

    Returns html portion containing annotated table with 3 columns: line

    numbers, changeset information and pygmentized line of code.

    :param filenode: FileNode object

    :param annotate_from_changeset_func: function taking changeset and

      returning single annotate cell; needs break line at the end

    :param order: ordered sequence of ``ls`` (line numbers column),

      ``annotate`` (annotate column), ``code`` (code column); Default is

      ``['ls', 'annotate', 'code']``

    :param headers: dictionary with headers (keys are whats in ``order``

      parameter)

    """

    from kallithea.lib.pygmentsutils import get_custom_lexer

    options['linenos'] = True

    formatter = AnnotateHtmlFormatter(filenode=filenode, order=order,

        headers=headers,

        annotate_from_changeset_func=annotate_from_changeset_func, **options)

    lexer = get_custom_lexer(filenode.extension) or filenode.lexer

    return highlighted

class AnnotateHtmlFormatter(HtmlFormatter):

    def __init__(self, filenode, annotate_from_changeset_func=None,

            order=None, **options):

        """

        If ``annotate_from_changeset_func`` is passed it should be a function

        which returns string from the given changeset. For example, we may pass

        following function as ``annotate_from_changeset_func``::

    can be used to render it in a HTML template.

    """

    _diff_git_re = re.compile('^diff --git', re.MULTILINE)

    def __init__(self, diff, vcs='hg', diff_limit=None, inline_diff=True):

        """

        :param diff:   a text in diff format

        :param vcs: type of version control hg or git

        :param diff_limit: define the size of diff that is considered "big"

            based on that parameter cut off will be triggered, set to None

            to show full diff

        """

        self._diff = diff

        self.adds = 0

        self.removes = 0

        self.diff_limit = diff_limit

        self.limited_diff = False

        self.vcs = vcs

        self.parsed = self._parse_gitdiff(inline_diff=inline_diff)

    def _parse_gitdiff(self, inline_diff):

        """Parse self._diff and return a list of dicts with meta info and chunks for each file.

        Might set limited_diff.

    (?:^copy[ ]from[ ](?P<copy_from>.+)\n

       ^copy[ ]to[ ](?P<copy_to>.+)(?:\n|$))?

    (?:^new[ ]file[ ]mode[ ](?P<new_file_mode>.+)(?:\n|$))?

    (?:^deleted[ ]file[ ]mode[ ](?P<deleted_file_mode>.+)(?:\n|$))?

    (?:^index[ ](?P<a_blob_id>[0-9A-Fa-f]+)

        \.\.(?P<b_blob_id>[0-9A-Fa-f]+)[ ]?(?P<b_mode>.+)?(?:\n|$))?

    (?:^(?P<bin_patch>GIT[ ]binary[ ]patch)(?:\n|$))?

    (?:^---[ ](a/(?P<a_file>.+?)|/dev/null)\t?(?:\n|$))?

    (?:^\+\+\+[ ](b/(?P<b_file>.+?)|/dev/null)\t?(?:\n|$))?

""", re.VERBOSE | re.MULTILINE)

def _get_header(vcs, diff_chunk):

    """

    Parses a Git diff for a single file (header and chunks) and returns a tuple with:

    1. A dict with meta info:

        a_path, b_path, similarity_index, rename_from, rename_to,

        old_mode, new_mode, new_file_mode, deleted_file_mode,

        a_blob_id, b_blob_id, b_mode, a_file, b_file

    2. An iterator yielding lines with simple HTML markup.

    """

    match = None

    if vcs == 'git':

        match = _git_header_re.match(diff_chunk)

    elif vcs == 'hg':

        match = _hg_header_re.match(diff_chunk)

    if match is None:

        raise Exception('diff not recognized as valid %s diff' % vcs)

    meta_info = match.groupdict()

    rest = diff_chunk[match.end():]

        raise Exception('cannot parse %s diff header: %r followed by %r' % (vcs, diff_chunk[:match.end()], rest[:1000]))

    diff_lines = (_escaper(m.group(0)) for m in re.finditer(r'.*\n|.+$', rest)) # don't split on \r as str.splitlines do

    return meta_info, diff_lines

_chunk_re = re.compile(r'^@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))? @@(.*)')

_newline_marker = re.compile(r'^\\ No newline at end of file')

def _parse_lines(diff_lines):

    """

    Given an iterator of diff body lines, parse them and return a dict per

def markup_whitespace(s):

    return _whitespace_re.sub(_markup_whitespace, s)

def pygmentize(filenode, **kwargs):

    """

    pygmentize function using pygments

    :param filenode:

    """

    lexer = get_custom_lexer(filenode.extension) or filenode.lexer

    return literal(markup_whitespace(

def pygmentize_annotation(repo_name, filenode, **kwargs):

    """

    pygmentize function for annotation

    :param filenode:

    """

    color_dict = {}

    def gen_color(n=10000):

        """

        Adding doc to writer this function itself fetches data from

        the instance of vcs backend

        """

        try:

            node = self.get_node(repo, path, index_rev)

        except (ChangesetError, NodeDoesNotExistError):

            log.debug("    >> %s - not found in %s %s", path, repo, index_rev)

            return 0, 0

        indexed = indexed_w_content = 0

        if self.is_indexable_node(node):

                log.warning('    >> %s - no text content', path)

                u_content = u''

            else:

                log.debug('    >> %s', path)

                indexed_w_content += 1

        else:

            log.debug('    >> %s - not indexable', path)

            # just index file name without it's content

            u_content = u''

            indexed += 1

        p = safe_unicode(path)

        writer.add_document(

            fileid=p,

            owner=unicode(repo.contact),

                except KeyError:

                    # put curdir back into dirnames and stops

                    dirnames.insert(0, curdir)

                    break

                else:

                    # If found, updates parent

                    parent = self.repository._repo[dir_id]

                    ancestors.append((curdir, parent))

            # Now parent is deepest existing tree and we need to create subtrees

            # for dirnames (in reverse order) [this only applies for nodes from added]

            new_trees = []

            node_path = safe_bytes(node.name)

            if dirnames:

                # If there are trees which should be created we need to build

                # them now (in reverse order)

                reversed_dirnames = list(reversed(dirnames))

                curtree = objects.Tree()

                curtree[node_path] = node.mode, blob.id

                new_trees.append(curtree)

                for dirname in reversed_dirnames[:-1]:

                    newtree = objects.Tree()

                    #newtree.add(stat.S_IFDIR, dirname, curtree.id)

            Marks given path as added/changed/removed in a given _repo. This is

            for internal mercurial commit function.

            """

            # check if this path is removed

            if path in (node.path for node in self.removed):

                return None

            # check if this path is added

            for node in self.added:

                if node.path == path:

                    return memfilectx(_repo, memctx, path=node.path,

                        islink=False,

                        isexec=node.is_executable,

                        copysource=False)

            # or changed

            for node in self.changed:

                if node.path == path:

                    return memfilectx(_repo, memctx, path=node.path,

                        islink=False,

                        isexec=node.is_executable,

                        copysource=False)

            raise RepositoryError("Given path haven't been marked as added,"

                                  "changed or removed (%s)" % path)

        parents = [None, None]

        for i, parent in enumerate(self.parents):

            if parent is not None:

                parents[i] = parent._ctx.node()

    :created_on: Apr 8, 2010

    :copyright: (c) 2010-2011 by Marcin Kuzminski, Lukasz Balcerzak.

"""

import functools

import mimetypes

import posixpath

import stat

from kallithea.lib.vcs.backends.base import EmptyChangeset

from kallithea.lib.vcs.exceptions import NodeError, RemovedFileNodeError

from kallithea.lib.vcs.utils.lazy import LazyProperty

class NodeKind:

    SUBMODULE = -1

    DIR = 1

    FILE = 2

class NodeState:

    ADDED = u'added'

    CHANGED = u'changed'

        would raise ``NodeError`` exception.

        :param path: relative path to the node

        :param content: content may be passed to constructor

        :param changeset: if given, will use it to lazily fetch content

        :param mode: octal representation of ST_MODE (i.e. 0100644)

        """

        if content and changeset:

            raise NodeError("Cannot use both content and changeset")

        super(FileNode, self).__init__(path, kind=NodeKind.FILE)

        self.changeset = changeset

        self._content = content

        self._mode = mode or 0o100644

    @LazyProperty

    def mode(self):

        """

        Returns lazily mode of the FileNode. If ``changeset`` is not set, would

        use value given at initialization or 0100644 (default).

        """

        if self.changeset:

            mode = self.changeset.get_file_mode(self.path)

        else:

            mode = self._mode

        return mode

        if self.changeset:

            content = self.changeset.get_file_content(self.path)

        else:

            content = self._content

        return content

    @LazyProperty

    def size(self):

        if self.changeset:

            return self.changeset.get_file_size(self.path)

        raise NodeError("Cannot retrieve size of the file without related "

            "changeset attribute")

    @LazyProperty

    def message(self):

        if self.changeset:

            return self.last_changeset.message

        raise NodeError("Cannot retrieve message of the file without related "

    @LazyProperty

    def mimetype_main(self):

        return self.mimetype.split('/')[0]

    @LazyProperty

    def lexer(self):

        """

        Returns pygment's lexer class. Would try to guess lexer taking file's

        content, name and mimetype.

        """

        from pygments import lexers

        try:

        except lexers.ClassNotFound:

            lexer = lexers.TextLexer(stripnl=False)

        # returns first alias

        return lexer

    @LazyProperty

    def lexer_alias(self):

        """

        Returns first alias of the lexer guessed for this file.

        """

        return self.lexer.aliases[0]

        elif self.path in (node.path for node in self.changeset.added):

            return NodeState.ADDED

        elif self.path in (node.path for node in self.changeset.changed):

            return NodeState.CHANGED

        else:

            return NodeState.NOT_CHANGED

    @property

    def is_binary(self):

        """

        Returns True if file has binary content.

        """

    def is_browser_compatible_image(self):

        return self.mimetype in [

            "image/gif",

            "image/jpeg",

            "image/png",

            "image/bmp"

        ]

    @LazyProperty

    def extension(self):

        """Returns filenode extension"""

from pygments import highlight

from pygments.formatters import HtmlFormatter

from kallithea.lib.vcs.exceptions import VCSError

from kallithea.lib.vcs.nodes import FileNode

def annotate_highlight(filenode, annotate_from_changeset_func=None,

        order=None, headers=None, **options):

    """

    Returns html portion containing annotated table with 3 columns: line

    numbers, changeset information and pygmentized line of code.

    :param filenode: FileNode object

    :param annotate_from_changeset_func: function taking changeset and

      returning single annotate cell; needs break line at the end

    :param order: ordered sequence of ``ls`` (line numbers column),

      ``annotate`` (annotate column), ``code`` (code column); Default is

      ``['ls', 'annotate', 'code']``

    :param headers: dictionary with headers (keys are whats in ``order``

      parameter)

    """

    options['linenos'] = True

    formatter = AnnotateHtmlFormatter(filenode=filenode, order=order,

        headers=headers,

        annotate_from_changeset_func=annotate_from_changeset_func, **options)

class AnnotateHtmlFormatter(HtmlFormatter):

    def __init__(self, filenode, annotate_from_changeset_func=None,

            order=None, **options):

        """

        If ``annotate_from_changeset_func`` is passed it should be a function

        which returns string from the given changeset. For example, we may pass

        following function as ``annotate_from_changeset_func``::

            def changeset_to_anchor(changeset):

                </div>

            </div>

            % for cnt, file in enumerate(c.files):

                <div id="body" class="panel panel-default form-inline">

                    <div class="panel-heading">

                        <input type="hidden" value="${h.safe_unicode(file.path)}" name="org_files">

                        <input class="form-control" id="filename_${h.FID('f',file.path)}" name="files" size="30" type="text" value="${h.safe_unicode(file.path)}">

                        <select class="form-control" id="mimetype_${h.FID('f',file.path)}" name="mimetypes"></select>

                    </div>

                    <div class="panel-body no-padding">

                        <div id="editor_container">

                        </div>

                    </div>

                </div>

                ## dynamic edit box.

                <script type="text/javascript">

                    $(document).ready(function(){

                        var myCodeMirror = initCodeMirror(${h.js('editor_' + h.FID('f',file.path))}, ${h.jshtml(request.script_name)}, '');

                        //inject new modes

                        var $mimetype_select = $(${h.js('#mimetype_' + h.FID('f',file.path))});

                        $mimetype_select.each(function(){

                    <span class="pull-right buttons">

                      ${h.link_to(_('Show Annotation'),h.url('files_annotate_home',repo_name=c.repo_name,revision=c.cs.raw_id,f_path=c.f_path),class_="btn btn-default btn-xs")}

                      ${h.link_to(_('Show as Raw'),h.url('files_raw_home',repo_name=c.repo_name,revision=c.cs.raw_id,f_path=c.f_path),class_="btn btn-default btn-xs")}

                      ${h.link_to(_('Download as Raw'),h.url('files_rawfile_home',repo_name=c.repo_name,revision=c.cs.raw_id,f_path=c.f_path),class_="btn btn-default btn-xs")}

                      % if h.HasRepoPermissionLevel('write')(c.repo_name):

                       % if not c.file.is_binary:

                        ${h.link_to(_('Source'),h.url('files_home',repo_name=c.repo_name,revision=c.cs.raw_id,f_path=c.f_path),class_="btn btn-default btn-xs")}

                       % endif

                      % endif

                    </span>

              </div>

              <div class="panel-body no-padding">

              </div>

            </div>

            <div>

              <div class="form-group">

                  <label>${_('Commit Message')}</label>

                  <textarea class="form-control" id="commit" name="message" placeholder="${c.default_message}"></textarea>

              </div>

              <div class="form-group buttons">

                ${h.submit('commit',_('Commit Changes'),class_="btn btn-success")}

                ${h.reset('reset',_('Reset'),class_="btn btn-default")}

              </div>

            </div>

        changed = ['setup.py', 'tests/test_nodes.py', 'vcs/backends/hg.py',

            'vcs/nodes.py']

        assert set(changed) == set([f.path for f in chset.changed])

    def test_commit_message_is_unicode(self):

        for cs in self.repo:

            assert isinstance(cs.message, unicode)

    def test_changeset_author_is_unicode(self):

        for cs in self.repo:

            assert isinstance(cs.author, unicode)

        changeset = self.repo.get_changeset()

        for node in changeset.get_node('/'):

            if node.is_file():

    def test_wrong_path(self):

        # There is 'setup.py' in the root dir but not there:

        path = 'foo/bar/setup.py'

        tip = self.repo.get_changeset()

        with pytest.raises(VCSError):

            tip.get_node(path)

    def test_author_email(self):

        assert 'marcin@python-blog.com' == self.repo.get_changeset('c1214f7e79e02fc37156ff215cd71275450cffc3').author_email

        assert 'lukasz.balcerzak@python-center.pl' == self.repo.get_changeset('ff7ca51e58c505fec0dd2491de52c622bb7a806b').author_email

        assert '' == self.repo.get_changeset('8430a588b43b5d6da365400117c89400326e7992').author_email

            chset.get_node(path)

        # but it would be one of ``removed`` (changeset's attribute)

        assert path in [rf.path for rf in chset.removed]

    def test_commit_message_is_unicode(self):

        for cm in self.repo:

            assert isinstance(cm.message, unicode)

    def test_changeset_author_is_unicode(self):

        for cm in self.repo:

            assert isinstance(cm.author, unicode)

        test_changeset = self.repo.get_changeset(100)

        for node in test_changeset.get_node('/'):

            if node.is_file():

    def test_wrong_path(self):

        # There is 'setup.py' in the root dir but not there:

        path = 'foo/bar/setup.py'

        with pytest.raises(VCSError):

            self.repo.get_changeset().get_node(path)

    def test_archival_file(self):

        # TODO:

        pass

    def test_archival_as_generator(self):