Changeset - 31ebf7010566
[Not reviewed]
beta
0 9 0
Marcin Kuzminski - 14 years ago 2012-04-17 23:00:36
marcin@python-works.com
various fixes for git and mercurial with InMemoryCommit backend and non-ascii files
- hg use tolocal for unicode objects
- git use core.quotepath=false for commands
9 files changed with 49 insertions and 35 deletions:
0 comments (0 inline, 0 general)
rhodecode/controllers/files.py
Show inline comments
 
@@ -297,28 +297,28 @@ class FilesController(BaseRepoController
 

	
 
            if not content:
 
                h.flash(_('No content'), category='warning')
 
                return redirect(url('changeset_home', repo_name=c.repo_name,
 
                                    revision='tip'))
 
            if not filename:
 
                h.flash(_('No filename'), category='warning')
 
                return redirect(url('changeset_home', repo_name=c.repo_name,
 
                                    revision='tip'))
 

	
 
            try:
 
                self.scm_model.create_node(repo=c.rhodecode_repo,
 
                                             repo_name=repo_name, cs=c.cs,
 
                                             user=self.rhodecode_user,
 
                                             author=author, message=message,
 
                                             content=content, f_path=node_path)
 
                                           repo_name=repo_name, cs=c.cs,
 
                                           user=self.rhodecode_user,
 
                                           author=author, message=message,
 
                                           content=content, f_path=node_path)
 
                h.flash(_('Successfully committed to %s' % node_path),
 
                        category='success')
 
            except NodeAlreadyExistsError, e:
 
                h.flash(_(e), category='error')
 
            except Exception:
 
                log.error(traceback.format_exc())
 
                h.flash(_('Error occurred during commit'), category='error')
 
            return redirect(url('changeset_home',
 
                                repo_name=c.repo_name, revision='tip'))
 

	
 
        return render('files/files_add.html')
 

	
rhodecode/lib/vcs/backends/git/changeset.py
Show inline comments
 
@@ -64,25 +64,24 @@ class GitChangeset(BaseChangeset):
 
        """
 
        return self.changed, self.added, self.removed
 

	
 
    @LazyProperty
 
    def branch(self):
 

	
 
        heads = self.repository._heads(reverse=False)
 

	
 
        ref = heads.get(self.raw_id)
 
        if ref:
 
            return safe_unicode(ref)
 

	
 

	
 
    def _fix_path(self, path):
 
        """
 
        Paths are stored without trailing slash so we need to get rid off it if
 
        needed.
 
        """
 
        if path.endswith('/'):
 
            path = path.rstrip('/')
 
        return path
 

	
 
    def _get_id_for_path(self, path):
 

	
 
        # FIXME: Please, spare a couple of minutes and make those codes cleaner;
 
@@ -122,25 +121,24 @@ class GitChangeset(BaseChangeset):
 
                        raise ChangesetError('%s is not a directory' % curdir)
 
                else:
 
                    raise ChangesetError('%s have not been found' % curdir)
 

	
 
                # cache all items from the given traversed tree
 
                for item, stat, id in tree.iteritems():
 
                    if curdir:
 
                        name = '/'.join((curdir, item))
 
                    else:
 
                        name = item
 
                    self._paths[name] = id
 
                    self._stat_modes[name] = stat
 

	
 
            if not path in self._paths:
 
                raise NodeDoesNotExistError("There is no file nor directory "
 
                    "at the given path %r at revision %r"
 
                    % (path, self.short_id))
 
        return self._paths[path]
 

	
 
    def _get_kind(self, path):
 
        id = self._get_id_for_path(path)
 
        obj = self.repository._repo[id]
 
        if isinstance(obj, objects.Blob):
 
            return NodeKind.FILE
 
        elif isinstance(obj, objects.Tree):
 
@@ -384,47 +382,50 @@ class GitChangeset(BaseChangeset):
 
    @LazyProperty
 
    def affected_files(self):
 
        """
 
        Get's a fast accessible file changes for given changeset
 
        """
 

	
 
        return self.added + self.changed
 

	
 
    @LazyProperty
 
    def _diff_name_status(self):
 
        output = []
 
        for parent in self.parents:
 
            cmd = 'diff --name-status %s %s' % (parent.raw_id, self.raw_id)
 
            cmd = 'diff --name-status %s %s --encoding=utf8' % (parent.raw_id, self.raw_id)
 
            so, se = self.repository.run_git_command(cmd)
 
            output.append(so.strip())
 
        return '\n'.join(output)
 

	
 
    def _get_paths_for_status(self, status):
 
        """
 
        Returns sorted list of paths for given ``status``.
 

	
 
        :param status: one of: *added*, *modified* or *deleted*
 
        """
 
        paths = set()
 
        char = status[0].upper()
 
        for line in self._diff_name_status.splitlines():
 
            if not line:
 
                continue
 

	
 
            if line.startswith(char):
 
                splitted = line.split(char,1)
 
                splitted = line.split(char, 1)
 
                if not len(splitted) == 2:
 
                    raise VCSError("Couldn't parse diff result:\n%s\n\n and "
 
                        "particularly that line: %s" % (self._diff_name_status,
 
                        line))
 
                paths.add(splitted[1].strip())
 
                _path = splitted[1].strip()
 
                paths.add(_path)
 

	
 
        return sorted(paths)
 

	
 
    @LazyProperty
 
    def added(self):
 
        """
 
        Returns list of added ``FileNode`` objects.
 
        """
 
        if not self.parents:
 
            return list(self._get_file_nodes())
 
        return [self.get_node(path) for path in self._get_paths_for_status('added')]
 

	
 
    @LazyProperty
rhodecode/lib/vcs/backends/git/inmemory.py
Show inline comments
 
import time
 
import datetime
 
import posixpath
 
from dulwich import objects
 
from dulwich.repo import Repo
 
from rhodecode.lib.vcs.backends.base import BaseInMemoryChangeset
 
from rhodecode.lib.vcs.exceptions import RepositoryError
 
from rhodecode.lib.vcs.utils import safe_str
 

	
 

	
 
class GitInMemoryChangeset(BaseInMemoryChangeset):
 

	
 
    def commit(self, message, author, parents=None, branch=None, date=None,
 
            **kwargs):
 
               **kwargs):
 
        """
 
        Performs in-memory commit (doesn't check workdir in any way) and
 
        returns newly created ``Changeset``. Updates repository's
 
        ``revisions``.
 

	
 
        :param message: message of the commit
 
        :param author: full username, i.e. "Joe Doe <joe.doe@example.com>"
 
        :param parents: single parent or sequence of parents from which commit
 
          would be derieved
 
        :param date: ``datetime.datetime`` instance. Defaults to
 
          ``datetime.datetime.now()``.
 
        :param branch: branch name, as string. If none given, default backend's
 
@@ -111,27 +112,27 @@ class GitInMemoryChangeset(BaseInMemoryC
 
                del tree[path]
 
                if tree:
 
                    # This tree still has elements - don't remove it or any
 
                    # of it's parents
 
                    break
 

	
 
        object_store.add_object(commit_tree)
 

	
 
        # Create commit
 
        commit = objects.Commit()
 
        commit.tree = commit_tree.id
 
        commit.parents = [p._commit.id for p in self.parents if p]
 
        commit.author = commit.committer = author
 
        commit.author = commit.committer = safe_str(author)
 
        commit.encoding = ENCODING
 
        commit.message = message + ' '
 
        commit.message = safe_str(message) + ' '
 

	
 
        # Compute date
 
        if date is None:
 
            date = time.time()
 
        elif isinstance(date, datetime.datetime):
 
            date = time.mktime(date.timetuple())
 

	
 
        author_time = kwargs.pop('author_time', date)
 
        commit.commit_time = int(date)
 
        commit.author_time = int(author_time)
 
        tz = time.timezone
 
        author_tz = kwargs.pop('author_timezone', tz)
rhodecode/lib/vcs/backends/git/repository.py
Show inline comments
 
@@ -70,29 +70,31 @@ class GitRepository(BaseRepository):
 
        """
 
        Runs given ``cmd`` as git command and returns tuple
 
        (returncode, stdout, stderr).
 

	
 
        .. note::
 
           This method exists only until log/blame functionality is implemented
 
           at Dulwich (see https://bugs.launchpad.net/bugs/645142). Parsing
 
           os command's output is road to hell...
 

	
 
        :param cmd: git command to be executed
 
        """
 

	
 
        #cmd = '(cd %s && git %s)' % (self.path, cmd)
 
        _copts = ['-c', 'core.quotepath=false', ]
 

	
 
        if isinstance(cmd, basestring):
 
            cmd = 'GIT_CONFIG_NOGLOBAL=1 git %s' % cmd
 
        else:
 
            cmd = ['GIT_CONFIG_NOGLOBAL=1', 'git'] + cmd
 
            cmd = [cmd]
 

	
 
        cmd = ['GIT_CONFIG_NOGLOBAL=1', 'git'] + _copts + cmd
 

	
 
        try:
 
            opts = dict(
 
                shell=isinstance(cmd, basestring),
 
                stdout=PIPE,
 
                stderr=PIPE)
 
            if os.path.isdir(self.path):
 
                opts['cwd'] = self.path
 
            p = Popen(cmd, **opts)
 
        except OSError, err:
 
            raise RepositoryError("Couldn't run git command (%s).\n"
 
                "Original error was:%s" % (cmd, err))
 
        so, se = p.communicate()
rhodecode/lib/vcs/backends/hg/inmemory.py
Show inline comments
 
import datetime
 
import errno
 

	
 
from rhodecode.lib.vcs.backends.base import BaseInMemoryChangeset
 
from rhodecode.lib.vcs.exceptions import RepositoryError
 

	
 
from ...utils.hgcompat import memfilectx, memctx, hex
 
from ...utils.hgcompat import memfilectx, memctx, hex, tolocal
 

	
 

	
 
class MercurialInMemoryChangeset(BaseInMemoryChangeset):
 

	
 
    def commit(self, message, author, parents=None, branch=None, date=None,
 
            **kwargs):
 
        """
 
        Performs in-memory commit (doesn't check workdir in any way) and
 
        returns newly created ``Changeset``. Updates repository's
 
        ``revisions``.
 

	
 
        :param message: message of the commit
 
@@ -21,27 +21,27 @@ class MercurialInMemoryChangeset(BaseInM
 
        :param parents: single parent or sequence of parents from which commit
 
          would be derieved
 
        :param date: ``datetime.datetime`` instance. Defaults to
 
          ``datetime.datetime.now()``.
 
        :param branch: branch name, as string. If none given, default backend's
 
          branch would be used.
 

	
 
        :raises ``CommitError``: if any error occurs while committing
 
        """
 
        self.check_integrity(parents)
 

	
 
        from .repository import MercurialRepository
 
        if not isinstance(message, str) or not isinstance(author, str):
 
        if not isinstance(message, unicode) or not isinstance(author, unicode):
 
            raise RepositoryError('Given message and author needs to be '
 
                                  'an <str> instance')
 
                                  'an <unicode> instance')
 

	
 
        if branch is None:
 
            branch = MercurialRepository.DEFAULT_BRANCH_NAME
 
        kwargs['branch'] = branch
 

	
 
        def filectxfn(_repo, memctx, path):
 
            """
 
            Marks given path as added/changed/removed in a given _repo. This is
 
            for internal mercurial commit function.
 
            """
 

	
 
            # check if this path is removed
 
@@ -61,46 +61,48 @@ class MercurialInMemoryChangeset(BaseInM
 

	
 
            # or changed
 
            for node in self.changed:
 
                if node.path == path:
 
                    return memfilectx(path=node.path,
 
                        data=(node.content.encode('utf8')
 
                              if not node.is_binary else node.content),
 
                        islink=False,
 
                        isexec=node.is_executable,
 
                        copied=False)
 

	
 
            raise RepositoryError("Given path haven't been marked as added,"
 
                "changed or removed (%s)" % path)
 
                                  "changed or removed (%s)" % path)
 

	
 
        parents = [None, None]
 
        for i, parent in enumerate(self.parents):
 
            if parent is not None:
 
                parents[i] = parent._ctx.node()
 

	
 
        if date and isinstance(date, datetime.datetime):
 
            date = date.ctime()
 

	
 
        commit_ctx = memctx(repo=self.repository._repo,
 
            parents=parents,
 
            text='',
 
            files=self.get_paths(),
 
            filectxfn=filectxfn,
 
            user=author,
 
            date=date,
 
            extra=kwargs)
 

	
 
        loc = lambda u: tolocal(u.encode('utf-8'))
 

	
 
        # injecting given _repo params
 
        commit_ctx._text = message
 
        commit_ctx._user = author
 
        commit_ctx._text = loc(message)
 
        commit_ctx._user = loc(author)
 
        commit_ctx._date = date
 

	
 
        # TODO: Catch exceptions!
 
        n = self.repository._repo.commitctx(commit_ctx)
 
        # Returns mercurial node
 
        self._commit_ctx = commit_ctx  # For reference
 
        # Update vcs repository object & recreate mercurial _repo
 
        # new_ctx = self.repository._repo[node]
 
        # new_tip = self.repository.get_changeset(new_ctx.hex())
 
        new_id = hex(n)
 
        self.repository.revisions.append(new_id)
 
        self._repo = self.repository._get_repo(create=False)
rhodecode/lib/vcs/nodes.py
Show inline comments
 
@@ -111,24 +111,28 @@ class Node(object):
 
            raise NodeError("Root node cannot be FILE kind")
 

	
 
    @LazyProperty
 
    def parent(self):
 
        parent_path = self.get_parent_path()
 
        if parent_path:
 
            if self.changeset:
 
                return self.changeset.get_node(parent_path)
 
            return DirNode(parent_path)
 
        return None
 

	
 
    @LazyProperty
 
    def unicode_path(self):
 
        return safe_unicode(self.path)
 

	
 
    @LazyProperty
 
    def name(self):
 
        """
 
        Returns name of the node so if its path
 
        then only last part is returned.
 
        """
 
        return safe_unicode(self.path.rstrip('/').split('/')[-1])
 

	
 
    def _get_kind(self):
 
        return self._kind
 

	
 
    def _set_kind(self, kind):
 
        if hasattr(self, '_kind'):
rhodecode/lib/vcs/utils/hgcompat.py
Show inline comments
 
"""Mercurial libs compatibility
 
"""
 
Mercurial libs compatibility
 
"""
 

	
 
"""
 
from mercurial import archival, merge as hg_merge, patch, ui
 
from mercurial.commands import clone, nullid, pull
 
from mercurial.context import memctx, memfilectx
 
from mercurial.error import RepoError, RepoLookupError, Abort
 
from mercurial.hgweb.common import get_contact
 
from mercurial.localrepo import localrepository
 
from mercurial.match import match
 
from mercurial.mdiff import diffopts
 
from mercurial.node import hex
 
from mercurial.encoding import tolocal
 
\ No newline at end of file
rhodecode/model/scm.py
Show inline comments
 
@@ -26,25 +26,25 @@ import os
 
import time
 
import traceback
 
import logging
 
import cStringIO
 

	
 
from rhodecode.lib.vcs import get_backend
 
from rhodecode.lib.vcs.exceptions import RepositoryError
 
from rhodecode.lib.vcs.utils.lazy import LazyProperty
 
from rhodecode.lib.vcs.nodes import FileNode
 

	
 
from rhodecode import BACKENDS
 
from rhodecode.lib import helpers as h
 
from rhodecode.lib.utils2 import safe_str
 
from rhodecode.lib.utils2 import safe_str, safe_unicode
 
from rhodecode.lib.auth import HasRepoPermissionAny, HasReposGroupPermissionAny
 
from rhodecode.lib.utils import get_repos as get_filesystem_repos, make_ui, \
 
    action_logger, EmptyChangeset, REMOVED_REPO_PAT
 
from rhodecode.model import BaseModel
 
from rhodecode.model.db import Repository, RhodeCodeUi, CacheInvalidation, \
 
    UserFollowing, UserLog, User, RepoGroup
 

	
 
log = logging.getLogger(__name__)
 

	
 

	
 
class UserTemp(object):
 
    def __init__(self, user_id):
 
@@ -360,32 +360,34 @@ class ScmModel(BaseModel):
 

	
 
    def commit_change(self, repo, repo_name, cs, user, author, message,
 
                      content, f_path):
 

	
 
        if repo.alias == 'hg':
 
            from rhodecode.lib.vcs.backends.hg import MercurialInMemoryChangeset as IMC
 
        elif repo.alias == 'git':
 
            from rhodecode.lib.vcs.backends.git import GitInMemoryChangeset as IMC
 

	
 
        # decoding here will force that we have proper encoded values
 
        # in any other case this will throw exceptions and deny commit
 
        content = safe_str(content)
 
        message = safe_str(message)
 
        path = safe_str(f_path)
 
        author = safe_str(author)
 
        # message and author needs to be unicode
 
        # proper backend should then translate that into required type
 
        message = safe_unicode(message)
 
        author = safe_unicode(author)
 
        m = IMC(repo)
 
        m.change(FileNode(path, content))
 
        tip = m.commit(message=message,
 
                 author=author,
 
                 parents=[cs], branch=cs.branch)
 
                       author=author,
 
                       parents=[cs], branch=cs.branch)
 

	
 
        new_cs = tip.short_id
 
        action = 'push_local:%s' % new_cs
 

	
 
        action_logger(user, action, repo_name)
 

	
 
        self.mark_for_invalidation(repo_name)
 

	
 
    def create_node(self, repo, repo_name, cs, user, author, message, content,
 
                      f_path):
 
        if repo.alias == 'hg':
 
            from rhodecode.lib.vcs.backends.hg import MercurialInMemoryChangeset as IMC
 
@@ -394,39 +396,39 @@ class ScmModel(BaseModel):
 
        # decoding here will force that we have proper encoded values
 
        # in any other case this will throw exceptions and deny commit
 

	
 
        if isinstance(content, (basestring,)):
 
            content = safe_str(content)
 
        elif isinstance(content, (file, cStringIO.OutputType,)):
 
            content = content.read()
 
        else:
 
            raise Exception('Content is of unrecognized type %s' % (
 
                type(content)
 
            ))
 

	
 
        message = safe_str(message)
 
        message = safe_unicode(message)
 
        author = safe_unicode(author)
 
        path = safe_str(f_path)
 
        author = safe_str(author)
 
        m = IMC(repo)
 

	
 
        if isinstance(cs, EmptyChangeset):
 
            # Emptychangeset means we we're editing empty repository
 
            # EmptyChangeset means we we're editing empty repository
 
            parents = None
 
        else:
 
            parents = [cs]
 

	
 
        m.add(FileNode(path, content=content))
 
        tip = m.commit(message=message,
 
                 author=author,
 
                 parents=parents, branch=cs.branch)
 
                       author=author,
 
                       parents=parents, branch=cs.branch)
 
        new_cs = tip.short_id
 
        action = 'push_local:%s' % new_cs
 

	
 
        action_logger(user, action, repo_name)
 

	
 
        self.mark_for_invalidation(repo_name)
 

	
 
    def get_nodes(self, repo_name, revision, root_path='/', flat=True):
 
        """
 
        recursive walk in root dir and return a set of all path in that dir
 
        based on repository walk function
 

	
rhodecode/templates/files/files_edit.html
Show inline comments
 
@@ -47,25 +47,25 @@
 
                    <div class="left item last">${c.file.mimetype}</div>
 
                    <div class="buttons">
 
                      ${h.link_to(_('show annotation'),h.url('files_annotate_home',repo_name=c.repo_name,revision=c.cs.raw_id,f_path=c.f_path),class_="ui-btn")}
 
                      ${h.link_to(_('show as raw'),h.url('files_raw_home',repo_name=c.repo_name,revision=c.cs.raw_id,f_path=c.f_path),class_="ui-btn")}
 
                      ${h.link_to(_('download as raw'),h.url('files_rawfile_home',repo_name=c.repo_name,revision=c.cs.raw_id,f_path=c.f_path),class_="ui-btn")}
 
                      % if h.HasRepoPermissionAny('repository.write','repository.admin')(c.repo_name):
 
                       % if not c.file.is_binary:
 
                        ${h.link_to(_('source'),h.url('files_home',repo_name=c.repo_name,revision=c.cs.raw_id,f_path=c.f_path),class_="ui-btn")}
 
                       % endif
 
                      % endif
 
                    </div>
 
                </div>
 
                <div class="commit">${_('Editing file')}: ${c.file.path}</div>
 
                <div class="commit">${_('Editing file')}: ${c.file.unicode_path}</div>
 
            </div>
 
			    <pre id="editor_pre"></pre>
 
				<textarea id="editor" name="content" style="display:none">${h.escape(c.file.content)|n}</textarea>
 
				<div style="padding: 10px;color:#666666">${_('commit message')}</div>
 
				<textarea id="commit" name="message" style="height: 60px;width: 99%;margin-left:4px"></textarea>
 
			</div>
 
			<div style="text-align: left;padding-top: 5px">
 
            ${h.submit('commit',_('Commit changes'),class_="ui-btn")}
 
            ${h.reset('reset',_('Reset'),class_="ui-btn")}
 
			</div>
 
			${h.end_form()}
 
			<script type="text/javascript">
0 comments (0 inline, 0 general)