stats['ops'][MOD_FILENODE] = 'modified file'

            # a real non-binary diff

            if head['a_file'] or head['b_file']:

                chunks, added, deleted = _parse_lines(diff_lines)

                stats['binary'] = False

                stats['added'] = added

                stats['deleted'] = deleted

                # explicit mark that it's a modified file

                if op == 'modified':

                    stats['ops'][MOD_FILENODE] = 'modified file'

            else:  # Git binary patch (or empty diff)

                # Git binary patch

                if head['bin_patch']:

                    stats['ops'][BIN_FILENODE] = 'binary diff not shown'

                chunks = []

            if op == 'removed' and chunks:

                # a way of seeing deleted content could perhaps be nice - but

                # not with the current UI

                chunks = []

            chunks.insert(0, [{

                'old_lineno': '',

                'new_lineno': '',

                'action':     'context',

                'line':       msg,

                } for _op, msg in stats['ops'].iteritems()

                  if _op not in [MOD_FILENODE]])

            _files.append({

                'old_filename':     head['a_path'],

                'filename':         head['b_path'],

                'old_revision':     head['a_blob_id'],

                'new_revision':     head['b_blob_id'],

                'chunks':           chunks,

                'operation':        op,

                'stats':            stats,

            })

        if not inline_diff:

            return _files

        # highlight inline changes when one del is followed by one add

        for diff_data in _files:

            for chunk in diff_data['chunks']:

                lineiter = iter(chunk)

                try:

                    while True:

                        # find a first del line

                        while peekline['action'] != 'del':

                        delline = peekline

                        # if not followed by add, eat all following del lines

                        if peekline['action'] != 'add':

                            while peekline['action'] == 'del':

                            continue

                        # found an add - make sure it is the only one

                        addline = peekline

                        try:

                        except StopIteration:

                            # add was last line - ok

                            _highlight_inline_diff(delline, addline)

                            raise

                        if peekline['action'] != 'add':

                            # there was only one add line - ok

                            _highlight_inline_diff(delline, addline)

                except StopIteration:

                    pass

        return _files

    def stat(self):

        """

        Returns tuple of added, and removed lines for this instance

        """

        return self.adds, self.removes

_escape_re = re.compile(r'(&)|(<)|(>)|(\t)|(\r)|(?<=.)( \n| $)')

def _escaper(string):

    """

    Do HTML escaping/markup

    """

    def substitute(m):

        groups = m.groups()

        if groups[0]:

            return '&'

        if groups[1]:

            return '<'

        if groups[2]:

            return '>'

        if groups[3]:

            return '<u>\t</u>'

        if groups[4]:

            return '<u class="cr"></u>'

        if groups[5]:

            return ' <i></i>'

        assert False

    return _escape_re.sub(substitute, safe_unicode(string))

_git_header_re = re.compile(br"""

    ^diff[ ]--git[ ]a/(?P<a_path>.+?)[ ]b/(?P<b_path>.+?)\n

    (?:^\+\+\+[ ](b/(?P<b_file>.+?)|/dev/null)\t?(?:\n|$))?

""", re.VERBOSE | re.MULTILINE)

_header_next_check = re.compile(br'''(?!@)(?!literal )(?!delta )''')

def _get_header(vcs, diff_chunk):

    """

    Parses a Git diff for a single file (header and chunks) and returns a tuple with:

    1. A dict with meta info:

        a_path, b_path, similarity_index, rename_from, rename_to,

        old_mode, new_mode, new_file_mode, deleted_file_mode,

        a_blob_id, b_blob_id, b_mode, a_file, b_file

    2. An iterator yielding lines with simple HTML markup.

    """

    match = None

    if vcs == 'git':

        match = _git_header_re.match(diff_chunk)

    elif vcs == 'hg':

        match = _hg_header_re.match(diff_chunk)

    if match is None:

        raise Exception('diff not recognized as valid %s diff' % vcs)

    meta_info = match.groupdict()

    rest = diff_chunk[match.end():]

    if rest and _header_next_check.match(rest):

        raise Exception('cannot parse %s diff header: %r followed by %r' % (vcs, diff_chunk[:match.end()], rest[:1000]))

    diff_lines = (_escaper(m.group(0)) for m in re.finditer(br'.*\n|.+$', rest)) # don't split on \r as str.splitlines do

    return meta_info, diff_lines

_chunk_re = re.compile(r'^@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))? @@(.*)')

_newline_marker = re.compile(r'^\\ No newline at end of file')

def _parse_lines(diff_lines):

    """

    Given an iterator of diff body lines, parse them and return a dict per

    line and added/removed totals.

    """

    added = deleted = 0

    old_line = old_end = new_line = new_end = None

    chunks = []

    try:

        while True:

            lines = []

            chunks.append(lines)

            match = _chunk_re.match(line)

            if not match:

                raise Exception('error parsing diff @@ line %r' % line)

            gr = match.groups()

            (old_line, old_end,

             new_line, new_end) = [int(x or 1) for x in gr[:-1]]

            old_line -= 1

            new_line -= 1

            context = len(gr) == 5

            old_end += old_line

            new_end += new_line

            if context:

                # skip context only if it's first line

                if int(gr[0]) > 1:

                    lines.append({

                        'old_lineno': '...',

                        'new_lineno': '...',

                        'action':     'context',

                        'line':       line,

                    })

            while old_line < old_end or new_line < new_end:

                if not line:

                    raise Exception('error parsing diff - empty line at -%s+%s' % (old_line, new_line))

                affects_old = affects_new = False

                command = line[0]

                if command == '+':

                    affects_new = True

                    action = 'add'

                    added += 1

                elif command == '-':

                    affects_old = True

                    action = 'del'

                    deleted += 1

                elif command == ' ':

                    affects_old = affects_new = True

                    action = 'unmod'

                else:

                    raise Exception('error parsing diff - unknown command in line %r at -%s+%s' % (line, old_line, new_line))

                if not _newline_marker.match(line):

                    old_line += affects_old

                    new_line += affects_new

                    lines.append({

                        'old_lineno':   affects_old and old_line or '',

                        'new_lineno':   affects_new and new_line or '',

                        'action':       action,

                        'line':         line[1:],

                    })

                if _newline_marker.match(line):

                    # we need to append to lines, since this is not

                    # counted in the line specs of diff

                    lines.append({

                        'old_lineno':   '...',

                        'new_lineno':   '...',

                        'action':       'context',

                        'line':         line,

                    })

            if old_line > old_end:

                raise Exception('error parsing diff - more than %s "-" lines at -%s+%s' % (old_end, old_line, new_line))

            if new_line > new_end:

                raise Exception('error parsing diff - more than %s "+" lines at -%s+%s' % (new_end, old_line, new_line))

    except StopIteration:

        pass

    if old_line != old_end or new_line != new_end:

        raise Exception('diff processing broken when old %s<>%s or new %s<>%s line %r' % (old_line, old_end, new_line, new_end, line))

    return chunks, added, deleted

# Used for inline highlighter word split, must match the substitutions in _escaper

_token_re = re.compile(r'()(&amp;|&lt;|&gt;|<u>\t</u>|<u class="cr"></u>| <i></i>|\W+?)')

def _highlight_inline_diff(old, new):

    """

    Highlight simple add/remove in two lines given as info dicts. They are

    modified in place and given markup with <del>/<ins>.

    """

    assert old['action'] == 'del'

    assert new['action'] == 'add'

    oldwords = _token_re.split(old['line'])

    newwords = _token_re.split(new['line'])

    sequence = difflib.SequenceMatcher(None, oldwords, newwords)

    oldfragments, newfragments = [], []

    for tag, i1, i2, j1, j2 in sequence.get_opcodes():

        oldfrag = ''.join(oldwords[i1:i2])

        newfrag = ''.join(newwords[j1:j2])

        if tag != 'equal':

            if oldfrag:

                oldfrag = '<del>%s</del>' % oldfrag

            if newfrag:

                newfrag = '<ins>%s</ins>' % newfrag

        oldfragments.append(oldfrag)

        newfragments.append(newfrag)

    old['line'] = "".join(oldfragments)

    new['line'] = "".join(newfragments)

    q = v * (1.0 - s * f)

    t = v * (1.0 - s * (1.0 - f))

    i = i % 6

    if i == 0:

        return v, t, p

    if i == 1:

        return q, v, p

    if i == 2:

        return p, v, t

    if i == 3:

        return p, q, v

    if i == 4:

        return t, p, v

    if i == 5:

        return v, p, q

def gen_color(n=10000):

    """generator for getting n of evenly distributed colors using

    hsv color and golden ratio. It always return same order of colors

    :returns: RGB tuple

    """

    golden_ratio = 0.618033988749895

    h = 0.22717784590367374

    for _unused in xrange(n):

        h += golden_ratio

        h %= 1

        HSV_tuple = [h, 0.95, 0.95]

        RGB_tuple = hsv_to_rgb(*HSV_tuple)

        yield [str(int(x * 256)) for x in RGB_tuple]

def pygmentize_annotation(repo_name, filenode, **kwargs):

    """

    pygmentize function for annotation

    :param filenode:

    """

    cgenerator = gen_color()

    color_dict = {}

    def get_color_string(cs):

        if cs in color_dict:

            col = color_dict[cs]

        else:

        return "color: rgb(%s)! important;" % (', '.join(col))

    def url_func(changeset):

        author = escape(changeset.author)

        date = changeset.date

        message = escape(changeset.message)

        tooltip_html = ("<b>Author:</b> %s<br/>"

                        "<b>Date:</b> %s</b><br/>"

                        "<b>Message:</b> %s") % (author, date, message)

        lnk_format = show_id(changeset)

        uri = link_to(

                lnk_format,

                url('changeset_home', repo_name=repo_name,

                    revision=changeset.raw_id),

                style=get_color_string(changeset.raw_id),

                **{'data-toggle': 'popover',

                   'data-content': tooltip_html}

              )

        uri += '\n'

        return uri

    return literal(markup_whitespace(annotate_highlight(filenode, url_func, **kwargs)))

class _Message(object):

    """A message returned by ``pop_flash_messages()``.

    Converting the message to a string returns the message text. Instances

    also have the following attributes:

    * ``category``: the category specified when the message was created.

    * ``message``: the html-safe message text.

    """

    def __init__(self, category, message):

        self.category = category

        self.message = message

def _session_flash_messages(append=None, clear=False):

    """Manage a message queue in tg.session: return the current message queue

    after appending the given message, and possibly clearing the queue."""

    key = 'flash'

    from tg import session

    if key in session:

        flash_messages = session[key]

CHGSETS_SCHEMA = Schema(

    raw_id=ID(unique=True, stored=True),

    date=NUMERIC(stored=True),

    last=BOOLEAN(),

    owner=TEXT(analyzer=EMAILADDRANALYZER),

    # this field preserves case of repository name for exact matching

    # and unique-ness in index table

    repository_rawname=ID(unique=True),

    repository=ID(stored=True, analyzer=ICASEIDANALYZER),

    author=TEXT(stored=True, analyzer=EMAILADDRANALYZER),

    message=FieldType(format=Characters(), analyzer=ANALYZER,

                      scorable=True, stored=True),

    parents=TEXT(),

    added=TEXT(analyzer=PATHANALYZER),

    removed=TEXT(analyzer=PATHANALYZER),

    changed=TEXT(analyzer=PATHANALYZER),

)

CHGSET_IDX_NAME = 'CHGSET_INDEX'

# used only to generate queries in journal

JOURNAL_SCHEMA = Schema(

    username=ID(),

    date=DATETIME(),

    action=TEXT(),

    repository=ID(),

    ip=TEXT(),

)

class WhooshResultWrapper(object):

    def __init__(self, search_type, searcher, matcher, highlight_items,

                 repo_location):

        self.search_type = search_type

        self.searcher = searcher

        self.matcher = matcher

        self.highlight_items = highlight_items

        self.fragment_size = 200

        self.repo_location = repo_location

    @LazyProperty

    def doc_ids(self):

        docs_id = []

        while self.matcher.is_active():

            docnum = self.matcher.id()

            chunks = [offsets for offsets in self.get_chunks()]

            docs_id.append([docnum, chunks])

        return docs_id

    def __str__(self):

        return '<%s at %s>' % (self.__class__.__name__, len(self.doc_ids))

    def __repr__(self):

        return self.__str__()

    def __len__(self):

        return len(self.doc_ids)

    def __iter__(self):

        """

        Allows Iteration over results,and lazy generate content

        *Requires* implementation of ``__getitem__`` method.

        """

        for docid in self.doc_ids:

            yield self.get_full_content(docid)

    def __getitem__(self, key):

        """

        Slicing of resultWrapper

        """

        i, j = key.start, key.stop

        slices = []

        for docid in self.doc_ids[i:j]:

            slices.append(self.get_full_content(docid))

        return slices

    def get_full_content(self, docid):

        res = self.searcher.stored_fields(docid[0])

        log.debug('result: %s', res)

        if self.search_type == 'content':

            full_repo_path = os.path.join(self.repo_location, res['repository'])

            f_path = res['path'].split(full_repo_path)[-1]

            f_path = f_path.lstrip(os.sep)

            content_short = self.get_short_content(res, docid[1])

            res.update({'content_short': content_short,

                        'content_short_hl': self.highlight(content_short),

                        'f_path': f_path

            })

        elif self.search_type == 'path':

            full_repo_path = os.path.join(self.repo_location, res['repository'])

            f_path = res['path'].split(full_repo_path)[-1]

            f_path = f_path.lstrip(os.sep)

            res.update({'f_path': f_path})

Wrap app to measure request and response time ... all the way to the response

WSGI iterator has been closed.

This file was forked by the Kallithea project in July 2014.

Original author and date, and relevant copyright and licensing information is below:

:created_on: May 23, 2013

:author: marcink

:copyright: (c) 2013 RhodeCode GmbH, and others.

:license: GPLv3, see LICENSE.md for more details.

"""

import logging

import time

from kallithea.lib.base import _get_ip_addr, get_path_info

log = logging.getLogger(__name__)

class Meter:

    def __init__(self, start_response):

        self._start_response = start_response

        self._start = time.time()

        self._size = 0

    def duration(self):

        return time.time() - self._start

    def start_response(self, status, response_headers, exc_info=None):

        write = self._start_response(status, response_headers, exc_info)

        def metered_write(s):

            self.measure(s)

            write(s)

        return metered_write

    def measure(self, chunk):

        self._size += len(chunk)

    def size(self):

        return self._size

class ResultIter:

    def __init__(self, result, meter, description):

        self._result_close = getattr(result, 'close', None) or (lambda: None)

        self._meter = meter

        self._description = description

    def __iter__(self):

        return self

        chunk = self._next()

        self._meter.measure(chunk)

        return chunk

    def close(self):

        self._result_close()

        log.info("%s responded after %.3fs with %s bytes", self._description, self._meter.duration(), self._meter.size())

class RequestWrapper(object):

    def __init__(self, app, config):

        self.application = app

        self.config = config

    def __call__(self, environ, start_response):

        meter = Meter(start_response)

        description = "Request from %s for %s" % (

            _get_ip_addr(environ),

            get_path_info(environ),

        )

        try:

            result = self.application(environ, meter.start_response)

        finally:

            log.info("%s responding after %.3fs", description, meter.duration())

        return ResultIter(result, meter, description)

                    raise IOError(

                        "Timed out while waiting for input from subprocess.")

            t.append(b)

            da.set()

            try:

                b = s.read(cs)

            except ValueError: # probably "I/O operation on closed file"

                b = ''

        self.EOF.set()

        da.set()  # for cases when done but there was no input.

class BufferedGenerator(object):

    """

    Class behaves as a non-blocking, buffered pipe reader.

    Reads chunks of data (through a thread)

    from a blocking pipe, and attaches these to an array (Deque) of chunks.

    Reading is halted in the thread when max chunks is internally buffered.

    The .next() may operate in blocking or non-blocking fashion by yielding

    '' if no data is ready

    to be sent or by not returning until there is some data to send

    When we get EOF from underlying source pipe we raise the marker to raise

    StopIteration after the last chunk of data is yielded.

    """

    def __init__(self, source, buffer_size=65536, chunk_size=4096,

                 starting_values=None, bottomless=False):

        starting_values = starting_values or []

        if bottomless:

            maxlen = int(buffer_size / chunk_size)

        else:

            maxlen = None

        self.data = collections.deque(starting_values, maxlen)

        self.worker = InputStreamChunker(source, self.data, buffer_size,

                                         chunk_size)

        if starting_values:

            self.worker.data_added.set()

        self.worker.start()

    ####################

    # Generator's methods

    ####################

    def __iter__(self):

        return self

        while not len(self.data) and not self.worker.EOF.is_set():

            self.worker.data_added.clear()

            self.worker.data_added.wait(0.2)

        if len(self.data):

            self.worker.keep_reading.set()

            return bytes(self.data.popleft())

        elif self.worker.EOF.is_set():

            raise StopIteration

    def throw(self, type, value=None, traceback=None):

        if not self.worker.EOF.is_set():

            raise type(value)

    def start(self):

        self.worker.start()

    def stop(self):

        self.worker.stop()

    def close(self):

        try:

            self.worker.stop()

            self.throw(GeneratorExit)

        except (GeneratorExit, StopIteration):

            pass

    ####################

    # Threaded reader's infrastructure.

    ####################

    @property

    def input(self):

        return self.worker.w

    @property

    def data_added_event(self):

        return self.worker.data_added

    @property

    def data_added(self):

        return self.worker.data_added.is_set()

    @property

    def reading_paused(self):

        return not self.worker.keep_reading.is_set()

    @property

    def done_reading_event(self):

        """

        assert isinstance(cmd, list), cmd

        _p = subprocess.Popen(cmd, bufsize=-1,

                              stdin=inputstream,

                              stdout=subprocess.PIPE,

                              stderr=subprocess.PIPE,

                              **kwargs)

        bg_out = BufferedGenerator(_p.stdout, buffer_size, chunk_size,

                                   starting_values)

        bg_err = BufferedGenerator(_p.stderr, 16000, 1, bottomless=True)

        while not bg_out.done_reading and not bg_out.reading_paused:

            # doing this until we reach either end of file, or end of buffer.

            bg_out.data_added_event.wait(1)

            bg_out.data_added_event.clear()

        # at this point it's still ambiguous if we are done reading or just full buffer.

        # Either way, if error (returned by ended process, or implied based on

        # presence of stuff in stderr output) we error out.

        # Else, we are happy.

        returncode = _p.poll()

        if (returncode is not None # process has terminated

            and returncode != 0

        ): # and it failed

            bg_out.stop()

            out = b''.join(bg_out)

            bg_err.stop()

            err = b''.join(bg_err)

            if (err.strip() == b'fatal: The remote end hung up unexpectedly' and

                out.startswith(b'0034shallow ')

            ):

                # hack inspired by https://github.com/schacon/grack/pull/7

                bg_out = iter([out])

                _p = None

            elif err:

                raise EnvironmentError("Subprocess exited due to an error: %s" % err)

            else:

                raise EnvironmentError(

                    "Subprocess exited with non 0 ret code: %s" % returncode)

        self.process = _p

        self.output = bg_out

        self.error = bg_err

        self.inputstream = inputstream

    def __iter__(self):

        return self

        if self.process:

            returncode = self.process.poll()

            if (returncode is not None # process has terminated

                and returncode != 0

            ): # and it failed

                self.output.stop()

                self.error.stop()

                err = ''.join(self.error)

                raise EnvironmentError("Subprocess exited due to an error:\n" + err)

    def throw(self, type, value=None, traceback=None):

        if self.output.length or not self.output.done_reading:

            raise type(value)

    def close(self):

        try:

            self.process.terminate()

        except:

            pass

        try:

            self.output.close()

        except:

            pass

        try:

            self.error.close()

        except:

            pass

        try:

            os.close(self.inputstream)

        except:

            pass

    HgHttpVcsTest,

    GitHttpVcsTest,

])

class Command(object):

    def __init__(self, cwd):

        self.cwd = cwd

    def execute(self, *args, **environ):

        """

        Runs command on the system with given ``args`` using simple space

        join without safe quoting.

        """

        command = ' '.join(args)

        ignoreReturnCode = environ.pop('ignoreReturnCode', False)

        if DEBUG:

            print('*** CMD %s ***' % command)

        testenv = dict(os.environ)

        testenv['LANG'] = 'en_US.UTF-8'

        testenv['LANGUAGE'] = 'en_US:en'

        testenv['HGPLAIN'] = ''

        testenv['HGRCPATH'] = ''

        testenv.update(environ)

        p = Popen(command, shell=True, stdout=PIPE, stderr=PIPE, cwd=self.cwd, env=testenv)

        stdout, stderr = p.communicate()

        if DEBUG:

            if stdout:

                print('stdout:', stdout)

            if stderr:

                print('stderr:', stderr)

        if not ignoreReturnCode:

            assert p.returncode == 0

        return stdout, stderr

def _get_tmp_dir(prefix='vcs_operations-', suffix=''):

    return tempfile.mkdtemp(dir=base.TESTS_TMP_PATH, prefix=prefix, suffix=suffix)

def _add_files(vcs, dest_dir, files_no=3):

    """

    Generate some files, add it to dest_dir repo and push back

    vcs is git or hg and defines what VCS we want to make those files for

    :param vcs:

    :param dest_dir:

    """

    open(os.path.join(dest_dir, added_file), 'a').close()

    Command(dest_dir).execute(vcs, 'add', added_file)

    email = 'me@example.com'

    if os.name == 'nt':

        author_str = 'User <%s>' % email

    else:

        author_str = 'User ǝɯɐᴎ <%s>' % email

    for i in xrange(files_no):

        cmd = """echo "added_line%s" >> %s""" % (i, added_file)

        Command(dest_dir).execute(cmd)

        if vcs == 'hg':

            cmd = """hg commit -m "committed new %s" -u "%s" "%s" """ % (

                i, author_str, added_file

            )

        elif vcs == 'git':

            cmd = """git commit -m "committed new %s" --author "%s" "%s" """ % (

                i, author_str, added_file

            )

        # git commit needs EMAIL on some machines

        Command(dest_dir).execute(cmd, EMAIL=email)

def _add_files_and_push(webserver, vt, dest_dir, clone_url, ignoreReturnCode=False, files_no=3):

    _add_files(vt.repo_type, dest_dir, files_no=files_no)

    # PUSH it back

    stdout = stderr = None

    if vt.repo_type == 'hg':

        stdout, stderr = Command(dest_dir).execute('hg push -f --verbose', clone_url, ignoreReturnCode=ignoreReturnCode)

    elif vt.repo_type == 'git':

        stdout, stderr = Command(dest_dir).execute('git push -f --verbose', clone_url, "master", ignoreReturnCode=ignoreReturnCode)

    return stdout, stderr

def _check_outgoing(vcs, cwd, clone_url):

    if vcs == 'hg':

        # hg removes the password from default URLs, so we have to provide it here via the clone_url

        return Command(cwd).execute('hg -q outgoing', clone_url, ignoreReturnCode=True)

    elif vcs == 'git':

        Command(cwd).execute('git remote update')

        return Command(cwd).execute('git log origin/master..master')

def set_anonymous_access(enable=True):

    user = User.get_default_user()

    user.active = enable

    Session().commit()

    if enable != User.get_default_user().active:

        raise Exception('Cannot set anonymous access')