Changeset - 7dbe020e93fe
Parent rev.
Child rev.
[Not reviewed]
default
0 1 0
Mads Kiilerich - 8 years ago 2017-10-03 00:14:40
mads@kiilerich.com
diffs: avoid extra copy of diff when splitting into files

Instead of allocating memory for an extra copy of the whole raw diff, just use
buffer() to give read-only string views of the relevant sections of the big
diff string given as input.
1 file changed with 8 insertions and 7 deletions:
kallithea/lib/diffs.py
8
7
0 comments (0 inline, 0 general)
kallithea/lib/diffs.py
➞
Show inline comments
 
@@ -117,118 +117,117 @@ def get_gitdiff(filenode_old, filenode_n
 
    if submodules:
 
        return ''
 

			




	
	
	
		
 
    for filenode in (filenode_old, filenode_new):

			




	
	
	
		
 
        if not isinstance(filenode, FileNode):

			




	
	
	
		
 
            raise VCSError("Given object should be FileNode object, not %s"

			




	
	
	
		
 
                % filenode.__class__)

			




	
	
	
		
 

			




	
	
	
		
 
    repo = filenode_new.changeset.repository

			




	
	
	
		
 
    old_raw_id = getattr(filenode_old.changeset, 'raw_id', repo.EMPTY_CHANGESET)

			




	
	
	
		
 
    new_raw_id = getattr(filenode_new.changeset, 'raw_id', repo.EMPTY_CHANGESET)

			




	
	
	
		
 

			




	
	
	
		
 
    vcs_gitdiff = repo.get_diff(old_raw_id, new_raw_id, filenode_new.path,

			




	
	
	
		
 
                                ignore_whitespace, context)

			




	
	
	
		
 
    return vcs_gitdiff

			




	
	
	
		
 

			




	
	
	
		
 

			




	
	
	
		
 
NEW_FILENODE = 1

			




	
	
	
		
 
DEL_FILENODE = 2

			




	
	
	
		
 
MOD_FILENODE = 3

			




	
	
	
		
 
RENAMED_FILENODE = 4

			




	
	
	
		
 
COPIED_FILENODE = 5

			




	
	
	
		
 
CHMOD_FILENODE = 6

			




	
	
	
		
 
BIN_FILENODE = 7

			




	
	
	
		
 

			




	
	
	
		
 

			




	
	
	
		
 
class DiffLimitExceeded(Exception):

			




	
	
	
		
 
    pass

			




	
	
	
		
 

			




	
	
	
		
 

			




	
	
	
		
 
class LimitedDiffContainer(object):

			




	
	
	
		
 

			




	
	
	
		
 
    def __init__(self, diff_limit, cur_diff_size, diff):

			




	
	
	
		
 
        self.diff = diff

			




	
	
	
		
 
        self.diff_limit = diff_limit

			




	
	
	
		
 
        self.cur_diff_size = cur_diff_size

			




	
	
	
		
 

			




	
	
	
		
 
    def __iter__(self):

			




	
	
	
		
 
        for l in self.diff:

			




	
	
	
		
 
            yield l

			




	
	
	
		
 

			




	
	
	
		
 

			




	
	
	
		
 
class DiffProcessor(object):

			




	
	
	
		
 
    """

			




	
	
	
		
 
    Give it a unified or git diff and it returns a list of the files that were

			




	
	
	
		
 
    mentioned in the diff together with a dict of meta information that

			




	
	
	
		
 
    can be used to render it in a HTML template.

			




	
	
	
		
 
    """

			




	
	
	
		
 
    _diff_git_re = re.compile('^diff --git', re.MULTILINE)

			




	
	
	
		
 
    _chunk_re = re.compile(r'^@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))? @@(.*)')

			




	
	
	
		
 
    _newline_marker = re.compile(r'^\\ No newline at end of file')

			




	
	
	
		
 
    _git_header_re = re.compile(r"""

			




	
	
	
		
 
        # has already been split on this:

			




	
	
	
		
 
        # ^diff[ ]--git

			




	
	
	
		
 
        ^diff[ ]--git

			




	
	
	
		
 
            [ ]a/(?P<a_path>.+?)[ ]b/(?P<b_path>.+?)\n

			




	
	
	
		
 
        (?:^old[ ]mode[ ](?P<old_mode>\d+)\n

			




	
	
	
		
 
           ^new[ ]mode[ ](?P<new_mode>\d+)(?:\n|$))?

			




	
	
	
		
 
        (?:^similarity[ ]index[ ](?P<similarity_index>\d+)%\n

			




	
	
	
		
 
           ^rename[ ]from[ ](?P<rename_from>.+)\n

			




	
	
	
		
 
           ^rename[ ]to[ ](?P<rename_to>.+)(?:\n|$))?

			




	
	
	
		
 
        (?:^new[ ]file[ ]mode[ ](?P<new_file_mode>.+)(?:\n|$))?

			




	
	
	
		
 
        (?:^deleted[ ]file[ ]mode[ ](?P<deleted_file_mode>.+)(?:\n|$))?

			




	
	
	
		
 
        (?:^index[ ](?P<a_blob_id>[0-9A-Fa-f]+)

			




	
	
	
		
 
            \.\.(?P<b_blob_id>[0-9A-Fa-f]+)[ ]?(?P<b_mode>.+)?(?:\n|$))?

			




	
	
	
		
 
        (?:^(?P<bin_patch>GIT[ ]binary[ ]patch)(?:\n|$))?

			




	
	
	
		
 
        (?:^---[ ](a/(?P<a_file>.+?)|/dev/null)\t?(?:\n|$))?

			




	
	
	
		
 
        (?:^\+\+\+[ ](b/(?P<b_file>.+?)|/dev/null)\t?(?:\n|$))?

			




	
	
	
		
 
    """, re.VERBOSE | re.MULTILINE)

			




	
	
	
		
 
    _hg_header_re = re.compile(r"""

			




	
	
	
		
 
        # has already been split on this:

			




	
	
	
		
 
        # ^diff[ ]--git

			




	
	
	
		
 
        ^diff[ ]--git

			




	
	
	
		
 
            [ ]a/(?P<a_path>.+?)[ ]b/(?P<b_path>.+?)\n

			




	
	
	
		
 
        (?:^old[ ]mode[ ](?P<old_mode>\d+)\n

			




	
	
	
		
 
           ^new[ ]mode[ ](?P<new_mode>\d+)(?:\n|$))?

			




	
	
	
		
 
        (?:^similarity[ ]index[ ](?P<similarity_index>\d+)%(?:\n|$))?

			




	
	
	
		
 
        (?:^rename[ ]from[ ](?P<rename_from>.+)\n

			




	
	
	
		
 
           ^rename[ ]to[ ](?P<rename_to>.+)(?:\n|$))?

			




	
	
	
		
 
        (?:^copy[ ]from[ ](?P<copy_from>.+)\n

			




	
	
	
		
 
           ^copy[ ]to[ ](?P<copy_to>.+)(?:\n|$))?

			




	
	
	
		
 
        (?:^new[ ]file[ ]mode[ ](?P<new_file_mode>.+)(?:\n|$))?

			




	
	
	
		
 
        (?:^deleted[ ]file[ ]mode[ ](?P<deleted_file_mode>.+)(?:\n|$))?

			




	
	
	
		
 
        (?:^index[ ](?P<a_blob_id>[0-9A-Fa-f]+)

			




	
	
	
		
 
            \.\.(?P<b_blob_id>[0-9A-Fa-f]+)[ ]?(?P<b_mode>.+)?(?:\n|$))?

			




	
	
	
		
 
        (?:^(?P<bin_patch>GIT[ ]binary[ ]patch)(?:\n|$))?

			




	
	
	
		
 
        (?:^---[ ](a/(?P<a_file>.+?)|/dev/null)\t?(?:\n|$))?

			




	
	
	
		
 
        (?:^\+\+\+[ ](b/(?P<b_file>.+?)|/dev/null)\t?(?:\n|$))?

			




	
	
	
		
 
    """, re.VERBOSE | re.MULTILINE)

			




	
	
	
		
 

			




	
	
	
		
 
    # Used for inline highlighter word split, must match the substitutions in _escaper

			




	
	
	
		
 
    _token_re = re.compile(r'()(&amp;|&lt;|&gt;|<u>\t</u>|<u class="cr"></u>| <i></i>|\W+?)')

			




	
	
	
		
 

			




	
	
	
		
 
    _escape_re = re.compile(r'(&)|(<)|(>)|(\t)|(\r)|(?<=.)( \n| $)')

			




	
	
	
		
 

			




	
	
	
		
 
    def __init__(self, diff, vcs='hg', diff_limit=None):

			




	
	
	
		
 
        """

			




	
	
	
		
 
        :param diff:   a text in diff format

			




	
	
	
		
 
        :param vcs: type of version control hg or git

			




	
	
	
		
 
        :param diff_limit: define the size of diff that is considered "big"

			




	
	
	
		
 
            based on that parameter cut off will be triggered, set to None

			




	
	
	
		
 
            to show full diff

			




	
	
	
		
 
        """

			




	
	
	
		
 
        if not isinstance(diff, basestring):

			




	
	
	
		
 
            raise Exception('Diff must be a basestring got %s instead' % type(diff))

			




	
	
	
		
 

			




	
	
	
		
 
        self._diff = diff

			




	
	
	
		
 
        self.adds = 0

			




	
	
	
		
 
        self.removes = 0

			




	
	
	
		
 
        # calculate diff size

			




	
	
	
		
 
        self.diff_size = len(diff)

			




	
	
	
		
 
        self.diff_limit = diff_limit

			




	
	
	
		
 
        self.cur_diff_size = 0

			




	
	
	
		
 
        self.parsed = False

			




	
	
	
		
 
        self.parsed_diff = []

			




	
	
	
		
 
        self.vcs = vcs

			




	
	
	
		
 

			




	
	
	
		
 
    def _escaper(self, string):

			




	
	
	
		
 
        """

			




	
	
	
		
 
        Do HTML escaping/markup and check the diff limit

			




	
	
	
		
 
        """

			




	
	
		
 
@@ -273,99 +272,101 @@ class DiffProcessor(object):

			




	
	
	
		
 
        for tag, i1, i2, j1, j2 in sequence.get_opcodes():

			




	
	
	
		
 
            oldfrag = ''.join(oldwords[i1:i2])

			




	
	
	
		
 
            newfrag = ''.join(newwords[j1:j2])

			




	
	
	
		
 
            if tag != 'equal':

			




	
	
	
		
 
                if oldfrag:

			




	
	
	
		
 
                    oldfrag = '<del>%s</del>' % oldfrag

			




	
	
	
		
 
                if newfrag:

			




	
	
	
		
 
                    newfrag = '<ins>%s</ins>' % newfrag

			




	
	
	
		
 
            oldfragments.append(oldfrag)

			




	
	
	
		
 
            newfragments.append(newfrag)

			




	
	
	
		
 

			




	
	
	
		
 
        old['line'] = "".join(oldfragments)

			




	
	
	
		
 
        new['line'] = "".join(newfragments)

			




	
	
	
		
 

			




	
	
	
		
 
    def _get_header(self, diff_chunk):

			




	
	
	
		
 
        """

			




	
	
	
		
 
        Parses a Git diff for a single file (header and chunks) and returns a tuple with:

			




	
	
	
		
 

			




	
	
	
		
 
        1. A dict with meta info:

			




	
	
	
		
 

			




	
	
	
		
 
            a_path, b_path, similarity_index, rename_from, rename_to,

			




	
	
	
		
 
            old_mode, new_mode, new_file_mode, deleted_file_mode,

			




	
	
	
		
 
            a_blob_id, b_blob_id, b_mode, a_file, b_file

			




	
	
	
		
 

			




	
	
	
		
 
        2. An iterator yielding lines with simple HTML markup.

			




	
	
	
		
 
        """

			




	
	
	
		
 
        match = None

			




	
	
	
		
 
        if self.vcs == 'git':

			




	
	
	
		
 
            match = self._git_header_re.match(diff_chunk)

			




	
	
	
		
 
        elif self.vcs == 'hg':

			




	
	
	
		
 
            match = self._hg_header_re.match(diff_chunk)

			




	
	
	
		
 
        if match is None:

			




	
	
	
		
 
            raise Exception('diff not recognized as valid %s diff' % self.vcs)

			




	
	
	
		
 
        meta_info = match.groupdict()

			




	
	
	
		
 
        rest = diff_chunk[match.end():]

			




	
	
	
		
 
        if rest and not rest.startswith('@') and not rest.startswith('literal ') and not rest.startswith('delta '):

			




	
	
	
		
 
            raise Exception('cannot parse %s diff header: %r followed by %r' % (self.vcs, diff_chunk[:match.end()], rest[:1000]))

			




	
	
	
		
 
        difflines = imap(self._escaper, re.findall(r'.*\n|.+$', rest)) # don't split on \r as str.splitlines do

			




	
	
	
		
 
        return meta_info, difflines

			




	
	
	
		
 

			




	
	
	
		
 
    def _parse_gitdiff(self, inline_diff=True):

			




	
	
	
		
 
        """Parse self._diff and return a list of dicts with meta info and chunks for each file.

			




	
	
	
		
 
        If diff is truncated, wrap it in LimitedDiffContainer.

			




	
	
	
		
 
        Optionally, do an extra pass and to extra markup of one-liner changes.

			




	
	
	
		
 
        """

			




	
	
	
		
 
        _files = [] # list of dicts with meta info and chunks

			




	
	
	
		
 
        diff_container = lambda arg: arg

			




	
	
	
		
 

			




	
	
	
		
 
        # split the diff in chunks of separate --git a/file b/file chunks

			




	
	
	
		
 
        for raw_diff in ('\n' + self._diff).split('\ndiff --git')[1:]:

			




	
	
	
		
 
            head, diff = self._get_header(raw_diff)

			




	
	
	
		
 
        starts = [m.start() for m in self._diff_git_re.finditer(self._diff)]

			




	
	
	
		
 
        starts.append(len(self._diff))

			




	
	
	
		
 

			




	
	
	
		
 
        for start, end in zip(starts, starts[1:]):

			




	
	
	
		
 
            head, diff = self._get_header(buffer(self._diff, start, end - start))

			




	
	
	
		
 

			




	
	
	
		
 
            op = None

			




	
	
	
		
 
            stats = {

			




	
	
	
		
 
                'added': 0,

			




	
	
	
		
 
                'deleted': 0,

			




	
	
	
		
 
                'binary': False,

			




	
	
	
		
 
                'ops': {},

			




	
	
	
		
 
            }

			




	
	
	
		
 

			




	
	
	
		
 
            if head['deleted_file_mode']:

			




	
	
	
		
 
                op = 'D'

			




	
	
	
		
 
                stats['binary'] = True

			




	
	
	
		
 
                stats['ops'][DEL_FILENODE] = 'deleted file'

			




	
	
	
		
 

			




	
	
	
		
 
            elif head['new_file_mode']:

			




	
	
	
		
 
                op = 'A'

			




	
	
	
		
 
                stats['binary'] = True

			




	
	
	
		
 
                stats['ops'][NEW_FILENODE] = 'new file %s' % head['new_file_mode']

			




	
	
	
		
 
            else:  # modify operation, can be cp, rename, chmod

			




	
	
	
		
 
                # CHMOD

			




	
	
	
		
 
                if head['new_mode'] and head['old_mode']:

			




	
	
	
		
 
                    op = 'M'

			




	
	
	
		
 
                    stats['binary'] = True

			




	
	
	
		
 
                    stats['ops'][CHMOD_FILENODE] = ('modified file chmod %s => %s'

			




	
	
	
		
 
                                        % (head['old_mode'], head['new_mode']))

			




	
	
	
		
 
                # RENAME

			




	
	
	
		
 
                if (head['rename_from'] and head['rename_to']

			




	
	
	
		
 
                      and head['rename_from'] != head['rename_to']):

			




	
	
	
		
 
                    op = 'R'

			




	
	
	
		
 
                    stats['binary'] = True

			




	
	
	
		
 
                    stats['ops'][RENAMED_FILENODE] = ('file renamed from %s to %s'

			




	
	
	
		
 
                                    % (head['rename_from'], head['rename_to']))

			




	
	
	
		
 
                # COPY

			




	
	
	
		
 
                if head.get('copy_from') and head.get('copy_to'):

			




	
	
	
		
 
                    op = 'M'

			




	
	
	
		
 
                    stats['binary'] = True

			




	
	
	
		
 
                    stats['ops'][COPIED_FILENODE] = ('file copied from %s to %s'

			




	
	
	
		
 
                                        % (head['copy_from'], head['copy_to']))

			




	
	
	
		
 
                # FALL BACK: detect missed old style add or remove

			




	
	
	
		
 
                if op is None:

			




	
	
	
		
 
                    if not head['a_file'] and head['b_file']:

			




	
	
	
		
 
                        op = 'A'

			




	
	
	
		
 
                        stats['binary'] = True

			




	
	
	
		
 
                        stats['ops'][NEW_FILENODE] = 'new file'

			




	
	
	
		
 

			




	
	
	
		
 
                    elif head['a_file'] and not head['b_file']:

			




	
	
	
		
 
                        op = 'D'

			




	
	
	
		
 
                        stats['binary'] = True

			




        

    



    


    



    



      

      





    
    0 comments (0 inline, 0 general)
    





    


  

  







  


    




    








    
        
    
    
        This site is powered by
            Kallithea 0.7.0,
        which is
        © 2010–2025 by various authors & licensed under GPLv3.