kallithea Changeset - 1112e440b921

Changeset - 1112e440b921

Parent rev.

Child rev.

[Not reviewed]

default

0 13 0

Mads Kiilerich - 6 years ago 2019-12-28 19:59:15
mads@kiilerich.com

Grafted from: 736eece1b4ed

py3: add safe_str where we really need it to get a str - probably from bytes

13 files changed with 45 insertions and 44 deletions:

kallithea/config/app_cfg.py

kallithea/controllers/home.py

kallithea/lib/base.py

kallithea/lib/diffs.py

kallithea/lib/hooks.py

kallithea/lib/utils.py

kallithea/lib/vcs/backends/git/changeset.py

kallithea/lib/vcs/backends/git/repository.py

kallithea/lib/vcs/backends/git/workdir.py

kallithea/lib/vcs/backends/hg/changeset.py

kallithea/lib/vcs/backends/hg/inmemory.py

kallithea/lib/vcs/backends/hg/repository.py

kallithea/lib/vcs/backends/hg/workdir.py

0 comments (0 inline, 0 general)

kallithea/config/app_cfg.py

➞

Show inline comments

 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
+#
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 Global configuration file for TurboGears2 specific settings in Kallithea.
 This file complements the .ini file.
 """
 import logging
 import os
 import platform
 import sys
 import alembic.config
 import mercurial
 import tg
 from alembic.migration import MigrationContext
 from alembic.script.base import ScriptDirectory
 from sqlalchemy import create_engine
 from tg.configuration import AppConfig
 from tg.support.converters import asbool
 import kallithea.lib.locale
 import kallithea.model.base
 import kallithea.model.meta
 from kallithea.lib.middleware.https_fixup import HttpsFixup
 from kallithea.lib.middleware.permanent_repo_url import PermanentRepoUrl
 from kallithea.lib.middleware.simplegit import SimpleGit
 from kallithea.lib.middleware.simplehg import SimpleHg
 from kallithea.lib.middleware.wrapper import RequestWrapper
 from kallithea.lib.utils import check_git_version, load_rcextensions, make_ui, set_app_settings, set_indexer_config, set_vcs_config
 from kallithea.lib.utils2 import str2bool
+from kallithea.lib.utils2 import safe_str, str2bool
 log = logging.getLogger(__name__)
 class KallitheaAppConfig(AppConfig):
     # Note: AppConfig has a misleading name, as it's not the application
     # configuration, but the application configurator. The AppConfig values are
     # used as a template to create the actual configuration, which might
     # overwrite or extend the one provided by the configurator template.
     # To make it clear, AppConfig creates the config and sets into it the same
     # values that AppConfig itself has. Then the values from the config file and
     # gearbox options are loaded and merged into the configuration. Then an
     # after_init_config(conf) method of AppConfig is called for any change that
     # might depend on options provided by configuration files.
     def __init__(self):
         super(KallitheaAppConfig, self).__init__()
         self['package'] = kallithea
         self['prefer_toscawidgets2'] = False
         self['use_toscawidgets'] = False
         self['renderers'] = []
         # Enable json in expose
         self['renderers'].append('json')
         # Configure template rendering
         self['renderers'].append('mako')
         self['default_renderer'] = 'mako'
         self['use_dotted_templatenames'] = False
         # Configure Sessions, store data as JSON to avoid pickle security issues
         self['session.enabled'] = True
         self['session.data_serializer'] = 'json'
         # Configure the base SQLALchemy Setup
         self['use_sqlalchemy'] = True
         self['model'] = kallithea.model.base
         self['DBSession'] = kallithea.model.meta.Session
         # Configure App without an authentication backend.
         self['auth_backend'] = None
         # Use custom error page for these errors. By default, Turbogears2 does not add
@@ @@ -118,96 +118,96 @@ except ImportError: @@
 else:
     base_config['renderers'].append('kajiki')
     enable_debugbar(base_config)
 def setup_configuration(app):
     config = app.config
     if not kallithea.lib.locale.current_locale_is_valid():
         log.error("Terminating ...")
         sys.exit(1)
     # Mercurial sets encoding at module import time, so we have to monkey patch it
     hgencoding = config.get('hgencoding')
     if hgencoding:
         mercurial.encoding.encoding = hgencoding
     if config.get('ignore_alembic_revision', False):
         log.warn('database alembic revision checking is disabled')
     else:
         dbconf = config['sqlalchemy.url']
         alembic_cfg = alembic.config.Config()
         alembic_cfg.set_main_option('script_location', 'kallithea:alembic')
         alembic_cfg.set_main_option('sqlalchemy.url', dbconf)
         script_dir = ScriptDirectory.from_config(alembic_cfg)
         available_heads = sorted(script_dir.get_heads())
         engine = create_engine(dbconf)
         with engine.connect() as conn:
             context = MigrationContext.configure(conn)
             current_heads = sorted(str(s) for s in context.get_current_heads())
         if current_heads != available_heads:
             log.error('Failed to run Kallithea:\n\n'
                       'The database version does not match the Kallithea version.\n'
                       'Please read the documentation on how to upgrade or downgrade the database.\n'
                       'Current database version id(s): %s\n'
                       'Expected database version id(s): %s\n'
                       'If you are a developer and you know what you are doing, you can add `ignore_alembic_revision = True` '
                       'to your .ini file to skip the check.\n' % (' '.join(current_heads), ' '.join(available_heads)))
             sys.exit(1)
     # store some globals into kallithea
     kallithea.CELERY_ON = str2bool(config.get('use_celery'))
     kallithea.CELERY_EAGER = str2bool(config.get('celery.always.eager'))
     kallithea.CONFIG = config
     load_rcextensions(root_path=config['here'])
     repos_path = make_ui().configitems(b'paths')[0][1]
+    repos_path = safe_str(make_ui().configitems(b'paths')[0][1])
     config['base_path'] = repos_path
     set_app_settings(config)
     instance_id = kallithea.CONFIG.get('instance_id', '*')
     if instance_id == '*':
         instance_id = '%s-%s' % (platform.uname()[1], os.getpid())
         kallithea.CONFIG['instance_id'] = instance_id
     # update kallithea.CONFIG with the meanwhile changed 'config'
     kallithea.CONFIG.update(config)
     # configure vcs and indexer libraries (they are supposed to be independent
     # as much as possible and thus avoid importing tg.config or
     # kallithea.CONFIG).
     set_vcs_config(kallithea.CONFIG)
     set_indexer_config(kallithea.CONFIG)
     check_git_version()
     kallithea.model.meta.Session.remove()
 tg.hooks.register('configure_new_app', setup_configuration)
 def setup_application(app):
     config = app.config
     # we want our low level middleware to get to the request ASAP. We don't
     # need any stack middleware in them - especially no StatusCodeRedirect buffering
     app = SimpleHg(app, config)
     app = SimpleGit(app, config)
     # Enable https redirects based on HTTP_X_URL_SCHEME set by proxy
     if any(asbool(config.get(x)) for x in ['https_fixup', 'force_https', 'use_htsts']):
         app = HttpsFixup(app, config)
     app = PermanentRepoUrl(app, config)
     # Optional and undocumented wrapper - gives more verbose request/response logging, but has a slight overhead
     if str2bool(config.get('use_wsgi_wrapper')):
         app = RequestWrapper(app, config)
     return app
 tg.hooks.register('before_config', setup_application)

kallithea/controllers/home.py

➞

Show inline comments

 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
+#
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.controllers.home
 ~~~~~~~~~~~~~~~~~~~~~~~~~~
 Home controller for Kallithea
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Feb 18, 2010
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 import logging
 from sqlalchemy import or_
 from tg import request
 from tg import tmpl_context as c
 from tg.i18n import ugettext as _
 from webob.exc import HTTPBadRequest
 from kallithea.lib import helpers as h
 from kallithea.lib.auth import HasRepoPermissionLevelDecorator, LoginRequired
 from kallithea.lib.base import BaseController, jsonify, render
 from kallithea.lib.utils2 import safe_str
 from kallithea.model.db import RepoGroup, Repository, User, UserGroup
 from kallithea.model.repo import RepoModel
 from kallithea.model.scm import UserGroupList
 log = logging.getLogger(__name__)
 class HomeController(BaseController):
     def about(self):
         return render('/about.html')
     @LoginRequired(allow_default_user=True)
     def index(self):
         c.group = None
         repo_groups_list = self.scm_model.get_repo_groups()
         repos_list = Repository.query(sorted=True).filter_by(group=None).all()
         c.data = RepoModel().get_repos_as_dict(repos_list,
                                                repo_groups_list=repo_groups_list,
                                                short_name=True)
         return render('/index.html')
     @LoginRequired(allow_default_user=True)
     @jsonify
     def repo_switcher_data(self):
         if request.is_xhr:
             all_repos = Repository.query(sorted=True).all()
             repo_iter = self.scm_model.get_repos(all_repos)
             all_groups = RepoGroup.query(sorted=True).all()
             repo_groups_iter = self.scm_model.get_repo_groups(all_groups)
             res = [{
                     'text': _('Groups'),
                     'children': [
                        {'id': obj.group_name,
                         'text': obj.group_name,
                         'type': 'group',
                         'obj': {}}
                        for obj in repo_groups_iter
                     ],
                    },
+                   {
                     'text': _('Repositories'),
                     'children': [
                        {'id': obj.repo_name,
                         'text': obj.repo_name,
                         'type': 'repo',
                         'obj': obj.get_dict()}
                        for obj in repo_iter
                     ],
                    }]
             for res_dict in res:
                 for child in (res_dict['children']):
                     child['obj'].pop('_changeset_cache', None)  # bytes cannot be encoded in json ... but this value isn't relevant on client side at all ...
             data = {
                 'more': False,
                 'results': res,
+            }
             return data
         else:
             raise HTTPBadRequest()
     @LoginRequired(allow_default_user=True)
     @HasRepoPermissionLevelDecorator('read')
     @jsonify
     def repo_refs_data(self, repo_name):
         repo = Repository.get_by_repo_name(repo_name).scm_instance
         res = []
         _branches = repo.branches.items()
         if _branches:
             res.append({
                 'text': _('Branch'),
                 'children': [{'id': rev, 'text': name, 'type': 'branch'} for name, rev in _branches]
+                'children': [{'id': safe_str(rev), 'text': safe_str(name), 'type': 'branch'} for name, rev in _branches]
             })
         _closed_branches = repo.closed_branches.items()
         if _closed_branches:
             res.append({
                 'text': _('Closed Branches'),
                 'children': [{'id': rev, 'text': name, 'type': 'closed-branch'} for name, rev in _closed_branches]
+                'children': [{'id': safe_str(rev), 'text': safe_str(name), 'type': 'closed-branch'} for name, rev in _closed_branches]
             })
         _tags = repo.tags.items()
         if _tags:
             res.append({
                 'text': _('Tag'),
                 'children': [{'id': rev, 'text': name, 'type': 'tag'} for name, rev in _tags]
+                'children': [{'id': safe_str(rev), 'text': safe_str(name), 'type': 'tag'} for name, rev in _tags]
             })
         _bookmarks = repo.bookmarks.items()
         if _bookmarks:
             res.append({
                 'text': _('Bookmark'),
                 'children': [{'id': rev, 'text': name, 'type': 'book'} for name, rev in _bookmarks]
+                'children': [{'id': safe_str(rev), 'text': safe_str(name), 'type': 'book'} for name, rev in _bookmarks]
             })
         data = {
             'more': False,
             'results': res
+        }
         return data
     @LoginRequired()
     @jsonify
     def users_and_groups_data(self):
         """
         Returns 'results' with a list of users and user groups.
         You can either use the 'key' GET parameter to get a user by providing
         the exact user key or you can use the 'query' parameter to
         search for users by user key, first name and last name.
         'types' defaults to just 'users' but can be set to 'users,groups' to
         get both users and groups.
         No more than 500 results (of each kind) will be returned.
         """
         types = request.GET.get('types', 'users').split(',')
         key = request.GET.get('key', '')
         query = request.GET.get('query', '')
         results = []
         if 'users' in types:
             user_list = []
             if key:
                 u = User.get_by_username(key)
                 if u:
                     user_list = [u]
             elif query:
                 user_list = User.query() \
                     .filter(User.is_default_user == False) \
                     .filter(User.active == True) \
                     .filter(or_(
                         User.username.ilike("%%" + query + "%%"),
                         User.name.ilike("%%" + query + "%%"),
                         User.lastname.ilike("%%" + query + "%%"),
                     )) \
                     .order_by(User.username) \
                     .limit(500) \
                     .all()
             for u in user_list:
                 results.append({
                     'type': 'user',
                     'id': u.user_id,
                     'nname': u.username,
                     'fname': u.name,

kallithea/lib/base.py

➞

Show inline comments

@@ @@ -127,97 +127,97 @@ def log_in_user(user, remember, is_exter @@
     session.invalidate()
     session['authuser'] = cookie = auth_user.to_cookie()
     # If they want to be remembered, update the cookie.
     # NOTE: Assumes that beaker defaults to browser session cookie.
     if remember:
         t = datetime.datetime.now() + datetime.timedelta(days=365)
         session._set_cookie_expires(t)
     session.save()
     log.info('user %s is now authenticated and stored in '
              'session, session attrs %s', user.username, cookie)
     # dumps session attrs back to cookie
     session._update_cookie_out()
     return auth_user
 class BasicAuth(paste.auth.basic.AuthBasicAuthenticator):
     def __init__(self, realm, authfunc, auth_http_code=None):
         self.realm = realm
         self.authfunc = authfunc
         self._rc_auth_http_code = auth_http_code
     def build_authentication(self, environ):
         head = paste.httpheaders.WWW_AUTHENTICATE.tuples('Basic realm="%s"' % self.realm)
         # Consume the whole body before sending a response
         try:
             request_body_size = int(environ.get('CONTENT_LENGTH', 0))
         except (ValueError):
             request_body_size = 0
         environ['wsgi.input'].read(request_body_size)
         if self._rc_auth_http_code and self._rc_auth_http_code == '403':
             # return 403 if alternative http return code is specified in
             # Kallithea config
             return paste.httpexceptions.HTTPForbidden(headers=head)
         return paste.httpexceptions.HTTPUnauthorized(headers=head)
     def authenticate(self, environ):
         authorization = paste.httpheaders.AUTHORIZATION(environ)
         if not authorization:
             return self.build_authentication(environ)
         (authmeth, auth) = authorization.split(' ', 1)
         if 'basic' != authmeth.lower():
             return self.build_authentication(environ)
         auth = base64.b64decode(auth.strip())
+        auth = safe_str(base64.b64decode(auth.strip()))
         _parts = auth.split(':', 1)
         if len(_parts) == 2:
             username, password = _parts
             if self.authfunc(username, password, environ) is not None:
                 return username
         return self.build_authentication(environ)
     __call__ = authenticate
 class BaseVCSController(object):
     """Base controller for handling Mercurial/Git protocol requests
     (coming from a VCS client, and not a browser).
     """
     scm_alias = None # 'hg' / 'git'
     def __init__(self, application, config):
         self.application = application
         self.config = config
         # base path of repo locations
         self.basepath = self.config['base_path']
         # authenticate this VCS request using the authentication modules
         self.authenticate = BasicAuth('', auth_modules.authenticate,
                                       config.get('auth_ret_code'))
     @classmethod
     def parse_request(cls, environ):
         """If request is parsed as a request for this VCS, return a namespace with the parsed request.
         If the request is unknown, return None.
         """
         raise NotImplementedError()
     def _authorize(self, environ, action, repo_name, ip_addr):
         """Authenticate and authorize user.
         Since we're dealing with a VCS client and not a browser, we only
         support HTTP basic authentication, either directly via raw header
         inspection, or by using container authentication to delegate the
         authentication to the web server.
         Returns (user, None) on successful authentication and authorization.
         Returns (None, wsgi_app) to send the wsgi_app response to the client.
         """
         # Use anonymous access if allowed for action on repo.
         default_user = User.get_default_user(cache=True)
         default_authuser = AuthUser.make(dbuser=default_user, ip_addr=ip_addr)
         if default_authuser is None:

kallithea/lib/diffs.py

➞

Show inline comments

@@ @@ -493,100 +493,100 @@ _git_header_re = re.compile(br""" @@
         \.\.(?P<b_blob_id>[0-9A-Fa-f]+)[ ]?(?P<b_mode>.+)?(?:\n|$))?
     (?:^(?P<bin_patch>GIT[ ]binary[ ]patch)(?:\n|$))?
     (?:^---[ ](a/(?P<a_file>.+?)|/dev/null)\t?(?:\n|$))?
     (?:^\+\+\+[ ](b/(?P<b_file>.+?)|/dev/null)\t?(?:\n|$))?
 """, re.VERBOSE | re.MULTILINE)
 _hg_header_re = re.compile(br"""
     ^diff[ ]--git[ ]a/(?P<a_path>.+?)[ ]b/(?P<b_path>.+?)\n
     (?:^old[ ]mode[ ](?P<old_mode>\d+)\n
        ^new[ ]mode[ ](?P<new_mode>\d+)(?:\n|$))?
     (?:^similarity[ ]index[ ](?P<similarity_index>\d+)%(?:\n|$))?
     (?:^rename[ ]from[ ](?P<rename_from>.+)\n
        ^rename[ ]to[ ](?P<rename_to>.+)(?:\n|$))?
     (?:^copy[ ]from[ ](?P<copy_from>.+)\n
        ^copy[ ]to[ ](?P<copy_to>.+)(?:\n|$))?
     (?:^new[ ]file[ ]mode[ ](?P<new_file_mode>.+)(?:\n|$))?
     (?:^deleted[ ]file[ ]mode[ ](?P<deleted_file_mode>.+)(?:\n|$))?
     (?:^index[ ](?P<a_blob_id>[0-9A-Fa-f]+)
         \.\.(?P<b_blob_id>[0-9A-Fa-f]+)[ ]?(?P<b_mode>.+)?(?:\n|$))?
     (?:^(?P<bin_patch>GIT[ ]binary[ ]patch)(?:\n|$))?
     (?:^---[ ](a/(?P<a_file>.+?)|/dev/null)\t?(?:\n|$))?
     (?:^\+\+\+[ ](b/(?P<b_file>.+?)|/dev/null)\t?(?:\n|$))?
 """, re.VERBOSE | re.MULTILINE)
 _header_next_check = re.compile(br'''(?!@)(?!literal )(?!delta )''')
 def _get_header(vcs, diff_chunk):
     """
     Parses a Git diff for a single file (header and chunks) and returns a tuple with:
 . A dict with meta info:
         a_path, b_path, similarity_index, rename_from, rename_to,
         old_mode, new_mode, new_file_mode, deleted_file_mode,
         a_blob_id, b_blob_id, b_mode, a_file, b_file
 . An iterator yielding lines with simple HTML markup.
     """
     match = None
     if vcs == 'git':
         match = _git_header_re.match(diff_chunk)
     elif vcs == 'hg':
         match = _hg_header_re.match(diff_chunk)
     if match is None:
         raise Exception('diff not recognized as valid %s diff' % vcs)
     meta_info = match.groupdict()
+    meta_info = {k: None if v is None else safe_str(v) for k, v in match.groupdict().items()}
     rest = diff_chunk[match.end():]
     if rest and _header_next_check.match(rest):
         raise Exception('cannot parse %s diff header: %r followed by %r' % (vcs, diff_chunk[:match.end()], rest[:1000]))
+        raise Exception('cannot parse %s diff header: %r followed by %r' % (vcs, safe_str(bytes(diff_chunk[:match.end()])), safe_str(bytes(rest[:1000]))))
     diff_lines = (_escaper(m.group(0)) for m in re.finditer(br'.*\n|.+$', rest)) # don't split on \r as str.splitlines do
     return meta_info, diff_lines
 _chunk_re = re.compile(r'^@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))? @@(.*)')
 _newline_marker = re.compile(r'^\\ No newline at end of file')
 def _parse_lines(diff_lines):
     """
     Given an iterator of diff body lines, parse them and return a dict per
     line and added/removed totals.
     """
     added = deleted = 0
     old_line = old_end = new_line = new_end = None
     chunks = []
     try:
         line = next(diff_lines)
         while True:
             lines = []
             chunks.append(lines)
             match = _chunk_re.match(line)
             if not match:
                 raise Exception('error parsing diff @@ line %r' % line)
             gr = match.groups()
             (old_line, old_end,
              new_line, new_end) = [int(x or 1) for x in gr[:-1]]
             old_line -= 1
             new_line -= 1
             context = len(gr) == 5
             old_end += old_line
             new_end += new_line
             if context:
                 # skip context only if it's first line
                 if int(gr[0]) > 1:
                     lines.append({
                         'old_lineno': '...',
                         'new_lineno': '...',
                         'action':     'context',
                         'line':       line,
                     })

kallithea/lib/hooks.py

➞

Show inline comments

 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
+#
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.lib.hooks
 ~~~~~~~~~~~~~~~~~~~
 Hooks run by Kallithea
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Aug 6, 2010
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 import os
 import sys
 import time
 import mercurial.scmutil
 from kallithea.lib import helpers as h
 from kallithea.lib.exceptions import UserCreationError
 from kallithea.lib.utils import action_logger, make_ui
 from kallithea.lib.utils2 import HookEnvironmentError, ascii_str, get_hook_environment, safe_bytes
+from kallithea.lib.utils2 import HookEnvironmentError, ascii_str, get_hook_environment, safe_bytes, safe_str
 from kallithea.lib.vcs.backends.base import EmptyChangeset
 from kallithea.model.db import Repository, User
 def _get_scm_size(alias, root_path):
     if not alias.startswith('.'):
         alias += '.'
     size_scm, size_root = 0, 0
     for path, dirs, files in os.walk(root_path):
         if path.find(alias) != -1:
             for f in files:
                 try:
                     size_scm += os.path.getsize(os.path.join(path, f))
                 except OSError:
                     pass
         else:
             for f in files:
                 try:
                     size_root += os.path.getsize(os.path.join(path, f))
                 except OSError:
                     pass
     size_scm_f = h.format_byte_size(size_scm)
     size_root_f = h.format_byte_size(size_root)
     size_total_f = h.format_byte_size(size_root + size_scm)
     return size_scm_f, size_root_f, size_total_f
 def repo_size(ui, repo, hooktype=None, **kwargs):
     """Show size of Mercurial repository, to be called after push."""
     size_hg_f, size_root_f, size_total_f = _get_scm_size('.hg', repo.root)
+    size_hg_f, size_root_f, size_total_f = _get_scm_size('.hg', safe_str(repo.root))
     last_cs = repo[len(repo) - 1]
     msg = ('Repository size .hg: %s Checkout: %s Total: %s\n'
            'Last revision is now r%s:%s\n') % (
         size_hg_f, size_root_f, size_total_f, last_cs.rev(), ascii_str(last_cs.hex())[:12]
+    )
     ui.status(safe_bytes(msg))
 def log_pull_action(ui, repo, **kwargs):
     """Logs user last pull action
     Called as Mercurial hook outgoing.pull_logger or from Kallithea before invoking Git.
     Does *not* use the action from the hook environment but is always 'pull'.
     """
     ex = get_hook_environment()
     user = User.get_by_username(ex.username)
     action = 'pull'
     action_logger(user, action, ex.repository, ex.ip, commit=True)
     # extension hook call
     from kallithea import EXTENSIONS
     callback = getattr(EXTENSIONS, 'PULL_HOOK', None)
     if callable(callback):
         kw = {}
         kw.update(ex)
         callback(**kw)
     return 0
 def log_push_action(ui, repo, node, node_last, **kwargs):
     """
     Entry point for Mercurial hook changegroup.push_logger.
     The pushed changesets is given by the revset 'node:node_last'.
     Note: This hook is not only logging, but also the side effect invalidating
     cahes! The function should perhaps be renamed.
     """
     revs = [ascii_str(repo[r].hex()) for r in mercurial.scmutil.revrange(repo, [b'%s:%s' % (node, node_last)])]
     process_pushed_raw_ids(revs)
     return 0
 def process_pushed_raw_ids(revs):

kallithea/lib/utils.py

➞

Show inline comments

 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
+#
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.lib.utils
 ~~~~~~~~~~~~~~~~~~~
 Utilities library for Kallithea
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Apr 18, 2010
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 import datetime
 import logging
 import os
 import re
 import sys
 import traceback
 from distutils.version import StrictVersion
 import beaker.cache
 import mercurial.config
 import mercurial.ui
 from tg.i18n import ugettext as _
 import kallithea.config.conf
 from kallithea.lib.exceptions import HgsubversionImportError
 from kallithea.lib.utils2 import ascii_bytes, aslist, get_current_authuser, safe_bytes
+from kallithea.lib.utils2 import ascii_bytes, aslist, get_current_authuser, safe_bytes, safe_str
 from kallithea.lib.vcs.backends.git.repository import GitRepository
 from kallithea.lib.vcs.backends.hg.repository import MercurialRepository
 from kallithea.lib.vcs.conf import settings
 from kallithea.lib.vcs.exceptions import RepositoryError, VCSError
 from kallithea.lib.vcs.utils.fakemod import create_module
 from kallithea.lib.vcs.utils.helpers import get_scm
 from kallithea.model import meta
 from kallithea.model.db import RepoGroup, Repository, Setting, Ui, User, UserGroup, UserLog
 log = logging.getLogger(__name__)
 REMOVED_REPO_PAT = re.compile(r'rm__\d{8}_\d{6}_\d{6}_.*')
 #==============================================================================
 # PERM DECORATOR HELPERS FOR EXTRACTING NAMES FOR PERM CHECKS
 #==============================================================================
 def get_repo_slug(request):
     _repo = request.environ['pylons.routes_dict'].get('repo_name')
     if _repo:
         _repo = _repo.rstrip('/')
     return _repo
 def get_repo_group_slug(request):
     _group = request.environ['pylons.routes_dict'].get('group_name')
     if _group:
         _group = _group.rstrip('/')
     return _group
 def get_user_group_slug(request):
     _group = request.environ['pylons.routes_dict'].get('id')
     _group = UserGroup.get(_group)
     if _group:
         return _group.users_group_name
     return None
 def _get_permanent_id(s):
     """Helper for decoding stable URLs with repo ID. For a string like '_123'
     return 123.
     """
     by_id_match = re.match(r'^_(\d+)$', s)
     if by_id_match is None:
         return None
     return int(by_id_match.group(1))
@@ @@ -541,103 +541,103 @@ def load_rcextensions(root_path): @@
         # Additional mappings that are not present in the pygments lexers
         kallithea.config.conf.LANGUAGES_EXTENSIONS_MAP.update(getattr(EXT, 'EXTRA_MAPPINGS', {}))
         # OVERRIDE OUR EXTENSIONS FROM RC-EXTENSIONS (if present)
         if getattr(EXT, 'INDEX_EXTENSIONS', []):
             log.debug('settings custom INDEX_EXTENSIONS')
             kallithea.config.conf.INDEX_EXTENSIONS = getattr(EXT, 'INDEX_EXTENSIONS', [])
         # ADDITIONAL MAPPINGS
         log.debug('adding extra into INDEX_EXTENSIONS')
         kallithea.config.conf.INDEX_EXTENSIONS.extend(getattr(EXT, 'EXTRA_INDEX_EXTENSIONS', []))
         # auto check if the module is not missing any data, set to default if is
         # this will help autoupdate new feature of rcext module
         #from kallithea.config import rcextensions
         #for k in dir(rcextensions):
         #    if not k.startswith('_') and not hasattr(EXT, k):
         #        setattr(EXT, k, getattr(rcextensions, k))
 #==============================================================================
 # MISC
 #==============================================================================
 git_req_ver = StrictVersion('1.7.4')
 def check_git_version():
     """
     Checks what version of git is installed on the system, and raise a system exit
     if it's too old for Kallithea to work properly.
     """
     if 'git' not in kallithea.BACKENDS:
         return None
     if not settings.GIT_EXECUTABLE_PATH:
         log.warning('No git executable configured - check "git_path" in the ini file.')
         return None
     try:
         stdout, stderr = GitRepository._run_git_command(['--version'])
     except RepositoryError as e:
         # message will already have been logged as error
         log.warning('No working git executable found - check "git_path" in the ini file.')
         return None
     if stderr:
         log.warning('Error/stderr from "%s --version":\n%s', settings.GIT_EXECUTABLE_PATH, stderr)
+        log.warning('Error/stderr from "%s --version":\n%s', settings.GIT_EXECUTABLE_PATH, safe_str(stderr))
     if not stdout:
         log.warning('No working git executable found - check "git_path" in the ini file.')
         return None
     output = stdout.strip()
+    output = safe_str(stdout).strip()
     m = re.search(r"\d+.\d+.\d+", output)
     if m:
         ver = StrictVersion(m.group(0))
         log.debug('Git executable: "%s", version %s (parsed from: "%s")',
                   settings.GIT_EXECUTABLE_PATH, ver, output)
         if ver < git_req_ver:
             log.error('Kallithea detected %s version %s, which is too old '
                       'for the system to function properly. '
                       'Please upgrade to version %s or later. '
                       'If you strictly need Mercurial repositories, you can '
                       'clear the "git_path" setting in the ini file.',
                       settings.GIT_EXECUTABLE_PATH, ver, git_req_ver)
             log.error("Terminating ...")
             sys.exit(1)
     else:
         ver = StrictVersion('0.0.0')
         log.warning('Error finding version number in "%s --version" stdout:\n%s',
                     settings.GIT_EXECUTABLE_PATH, output)
     return ver
 #===============================================================================
 # CACHE RELATED METHODS
 #===============================================================================
 def conditional_cache(region, prefix, condition, func):
     """
     Conditional caching function use like::
         def _c(arg):
             #heavy computation function
             return data
         # depending from condition the compute is wrapped in cache or not
         compute = conditional_cache('short_term', 'cache_desc', condition=True, func=func)
         return compute(arg)
     :param region: name of cache region
     :param prefix: cache region prefix
     :param condition: condition for cache to be triggered, and return data cached
     :param func: wrapped heavy function to compute
     """
     wrapped = func
     if condition:
         log.debug('conditional_cache: True, wrapping call of '
                   'func: %s into %s region cache' % (region, func))

kallithea/lib/vcs/backends/git/changeset.py

➞

Show inline comments

@@ @@ -51,152 +51,151 @@ class GitChangeset(BaseChangeset): @@
     def message(self):
         return safe_str(self._commit.message)
     @LazyProperty
     def committer(self):
         return safe_str(getattr(self._commit, self._committer_property))
     @LazyProperty
     def author(self):
         return safe_str(getattr(self._commit, self._author_property))
     @LazyProperty
     def date(self):
         return date_fromtimestamp(getattr(self._commit, self._date_property),
                                   getattr(self._commit, self._date_tz_property))
     @LazyProperty
     def _timestamp(self):
         return getattr(self._commit, self._date_property)
     @LazyProperty
     def status(self):
         """
         Returns modified, added, removed, deleted files for current changeset
         """
         return self.changed, self.added, self.removed
     @LazyProperty
     def tags(self):
         _tags = []
         for tname, tsha in self.repository.tags.items():
             if tsha == self.raw_id:
                 _tags.append(tname)
         return _tags
     @LazyProperty
     def branch(self):
         # Note: This function will return one branch name for the changeset -
         # that might not make sense in Git where branches() is a better match
         # for the basic model
         heads = self.repository._heads(reverse=False)
         ref = heads.get(self._commit.id)
         if ref:
             return safe_str(ref)
     @LazyProperty
     def branches(self):
         heads = self.repository._heads(reverse=True)
         return [b for b in heads if heads[b] == self._commit.id] # FIXME: Inefficient ... and returning None!
+        return [safe_str(b) for b in heads if heads[b] == self._commit.id] # FIXME: Inefficient ... and returning None!
     def _fix_path(self, path):
         """
         Paths are stored without trailing slash so we need to get rid off it if
         needed.
         """
         if path.endswith('/'):
             path = path.rstrip('/')
         return path
     def _get_id_for_path(self, path):
         # FIXME: Please, spare a couple of minutes and make those codes cleaner;
         if path not in self._paths:
             path = path.strip('/')
             # set root tree
             tree = self.repository._repo[self._tree_id]
             if path == '':
                 self._paths[''] = tree.id
                 return tree.id
             splitted = path.split('/')
             dirs, name = splitted[:-1], splitted[-1]
             curdir = ''
             # initially extract things from root dir
             for item, stat, id in tree.items():
                 name = safe_str(item)
                 if curdir:
                     name = '/'.join((curdir, item))
                 else:
                     name = item
                     name = '/'.join((curdir, name))
                 self._paths[name] = id
                 self._stat_modes[name] = stat
             for dir in dirs:
                 if curdir:
                     curdir = '/'.join((curdir, dir))
                 else:
                     curdir = dir
                 dir_id = None
                 for item, stat, id in tree.items():
                     if dir == item:
                     name = safe_str(item)
                     if dir == name:
                         dir_id = id
                 if dir_id:
                     # Update tree
                     tree = self.repository._repo[dir_id]
                     if not isinstance(tree, objects.Tree):
                         raise ChangesetError('%s is not a directory' % curdir)
                 else:
                     raise ChangesetError('%s have not been found' % curdir)
                 # cache all items from the given traversed tree
                 for item, stat, id in tree.items():
                     name = safe_str(item)
                     if curdir:
                         name = '/'.join((curdir, item))
                     else:
                         name = item
                         name = '/'.join((curdir, name))
                     self._paths[name] = id
                     self._stat_modes[name] = stat
             if path not in self._paths:
                 raise NodeDoesNotExistError("There is no file nor directory "
                     "at the given path '%s' at revision %s"
                     % (path, self.short_id))
         return self._paths[path]
     def _get_kind(self, path):
         obj = self.repository._repo[self._get_id_for_path(path)]
         if isinstance(obj, objects.Blob):
             return NodeKind.FILE
         elif isinstance(obj, objects.Tree):
             return NodeKind.DIR
     def _get_filectx(self, path):
         path = self._fix_path(path)
         if self._get_kind(path) != NodeKind.FILE:
             raise ChangesetError("File does not exist for revision %s at "
                 " '%s'" % (self.raw_id, path))
         return path
     def _get_file_nodes(self):
         return chain(*(t[2] for t in self.walk()))
     @LazyProperty
     def parents(self):
         """
         Returns list of parents changesets.
         """
         return [self.repository.get_changeset(ascii_str(parent_id))
                 for parent_id in self._commit.parents]
     @LazyProperty
     def children(self):
         """
         Returns list of children changesets.
         """
         rev_filter = settings.GIT_REV_FILTER
         so = self.repository.run_git_command(
             ['rev-list', rev_filter, '--children']
+        )
         return [
             self.repository.get_changeset(cs)
             for parts in (l.split(' ') for l in so.splitlines())
             if parts[0] == self.raw_id
             for cs in parts[1:]
+        ]
@@ @@ -359,194 +358,193 @@ class GitChangeset(BaseChangeset): @@
         elif prefix.startswith('/'):
             raise VCSError("Prefix cannot start with leading slash")
         elif prefix.strip() == '':
             raise VCSError("Prefix cannot be empty")
         if kind == 'zip':
             frmt = 'zip'
         else:
             frmt = 'tar'
         _git_path = settings.GIT_EXECUTABLE_PATH
         cmd = '%s archive --format=%s --prefix=%s/ %s' % (_git_path,
                                                 frmt, prefix, self.raw_id)
         if kind == 'tgz':
             cmd += ' | gzip -9'
         elif kind == 'tbz2':
             cmd += ' | bzip2 -9'
         if stream is None:
             raise VCSError('You need to pass in a valid stream for filling'
                            ' with archival data')
         popen = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True,
                       cwd=self.repository.path)
         buffer_size = 1024 * 8
         chunk = popen.stdout.read(buffer_size)
         while chunk:
             stream.write(chunk)
             chunk = popen.stdout.read(buffer_size)
         # Make sure all descriptors would be read
         popen.communicate()
     def get_nodes(self, path):
         """
         Returns combined ``DirNode`` and ``FileNode`` objects list representing
         state of changeset at the given ``path``. If node at the given ``path``
         is not instance of ``DirNode``, ChangesetError would be raised.
         """
         if self._get_kind(path) != NodeKind.DIR:
             raise ChangesetError("Directory does not exist for revision %s at "
                 " '%s'" % (self.revision, path))
         path = self._fix_path(path)
         id = self._get_id_for_path(path)
         tree = self.repository._repo[id]
         dirnodes = []
         filenodes = []
         als = self.repository.alias
         for name, stat, id in tree.items():
             obj_path = safe_str(name)
             if path != '':
                 obj_path = '/'.join((path, name))
             else:
                 obj_path = name
                 obj_path = '/'.join((path, obj_path))
             if objects.S_ISGITLINK(stat):
                 root_tree = self.repository._repo[self._tree_id]
                 cf = ConfigFile.from_file(BytesIO(self.repository._repo.get_object(root_tree[b'.gitmodules'][1]).data))
                 url = ascii_str(cf.get(('submodule', obj_path), 'url'))
                 dirnodes.append(SubModuleNode(obj_path, url=url, changeset=ascii_str(id),
                                               alias=als))
                 continue
             obj = self.repository._repo.get_object(id)
             if obj_path not in self._stat_modes:
                 self._stat_modes[obj_path] = stat
             if isinstance(obj, objects.Tree):
                 dirnodes.append(DirNode(obj_path, changeset=self))
             elif isinstance(obj, objects.Blob):
                 filenodes.append(FileNode(obj_path, changeset=self, mode=stat))
             else:
                 raise ChangesetError("Requested object should be Tree "
                                      "or Blob, is %r" % type(obj))
         nodes = dirnodes + filenodes
         for node in nodes:
             if node.path not in self.nodes:
                 self.nodes[node.path] = node
         nodes.sort()
         return nodes
     def get_node(self, path):
         """
         Returns ``Node`` object from the given ``path``. If there is no node at
         the given ``path``, ``ChangesetError`` would be raised.
         """
         path = self._fix_path(path)
         if path not in self.nodes:
             try:
                 id_ = self._get_id_for_path(path)
             except ChangesetError:
                 raise NodeDoesNotExistError("Cannot find one of parents' "
                     "directories for a given path: %s" % path)
             _GL = lambda m: m and objects.S_ISGITLINK(m)
             if _GL(self._stat_modes.get(path)):
                 tree = self.repository._repo[self._tree_id]
                 cf = ConfigFile.from_file(BytesIO(self.repository._repo.get_object(tree[b'.gitmodules'][1]).data))
                 url = ascii_str(cf.get(('submodule', path), 'url'))
                 node = SubModuleNode(path, url=url, changeset=ascii_str(id_),
                                      alias=self.repository.alias)
             else:
                 obj = self.repository._repo.get_object(id_)
                 if isinstance(obj, objects.Tree):
                     if path == '':
                         node = RootNode(changeset=self)
                     else:
                         node = DirNode(path, changeset=self)
                     node._tree = obj
                 elif isinstance(obj, objects.Blob):
                     node = FileNode(path, changeset=self)
                     node._blob = obj
                 else:
                     raise NodeDoesNotExistError("There is no file nor directory "
                         "at the given path: '%s' at revision %s"
                         % (path, self.short_id))
             # cache node
             self.nodes[path] = node
         return self.nodes[path]
     @LazyProperty
     def affected_files(self):
         """
         Gets a fast accessible file changes for given changeset
         """
         added, modified, deleted = self._changes_cache
         return list(added.union(modified).union(deleted))
     @LazyProperty
     def _changes_cache(self):
         added = set()
         modified = set()
         deleted = set()
         _r = self.repository._repo
         parents = self.parents
         if not self.parents:
             parents = [EmptyChangeset()]
         for parent in parents:
             if isinstance(parent, EmptyChangeset):
                 oid = None
             else:
                 oid = _r[parent._commit.id].tree
             changes = _r.object_store.tree_changes(oid, _r[self._commit.id].tree)
             for (oldpath, newpath), (_, _), (_, _) in changes:
                 if newpath and oldpath:
                     modified.add(newpath)
+                    modified.add(safe_str(newpath))
                 elif newpath and not oldpath:
                     added.add(newpath)
+                    added.add(safe_str(newpath))
                 elif not newpath and oldpath:
                     deleted.add(oldpath)
+                    deleted.add(safe_str(oldpath))
         return added, modified, deleted
     def _get_paths_for_status(self, status):
         """
         Returns sorted list of paths for given ``status``.
         :param status: one of: *added*, *modified* or *deleted*
         """
         added, modified, deleted = self._changes_cache
         return sorted({
             'added': list(added),
             'modified': list(modified),
             'deleted': list(deleted)}[status]
+        )
     @LazyProperty
     def added(self):
         """
         Returns list of added ``FileNode`` objects.
         """
         if not self.parents:
             return list(self._get_file_nodes())
         return AddedFileNodesGenerator([n for n in
                                 self._get_paths_for_status('added')], self)
     @LazyProperty
     def changed(self):
         """
         Returns list of modified ``FileNode`` objects.
         """
         if not self.parents:
             return []
         return ChangedFileNodesGenerator([n for n in
                                 self._get_paths_for_status('modified')], self)
     @LazyProperty
     def removed(self):
         """
         Returns list of removed ``FileNode`` objects.
         """
         if not self.parents:
             return []
         return RemovedFileNodesGenerator([n for n in
                                 self._get_paths_for_status('deleted')], self)
     extra = {}

kallithea/lib/vcs/backends/git/repository.py

➞

Show inline comments

@@ @@ -314,156 +314,156 @@ class GitRepository(BaseRepository): @@
     def _get_url(self, url):
         """
         Returns normalized url. If schema is not given, would fall to
         filesystem (``file:///``) schema.
         """
         if url != 'default' and '://' not in url:
             url = ':///'.join(('file', url))
         return url
     @LazyProperty
     def name(self):
         return os.path.basename(self.path)
     @LazyProperty
     def last_change(self):
         """
         Returns last change made on this repository as datetime object
         """
         return date_fromtimestamp(self._get_mtime(), makedate()[1])
     def _get_mtime(self):
         try:
             return time.mktime(self.get_changeset().date.timetuple())
         except RepositoryError:
             idx_loc = '' if self.bare else '.git'
             # fallback to filesystem
             in_path = os.path.join(self.path, idx_loc, "index")
             he_path = os.path.join(self.path, idx_loc, "HEAD")
             if os.path.exists(in_path):
                 return os.stat(in_path).st_mtime
             else:
                 return os.stat(he_path).st_mtime
     @LazyProperty
     def description(self):
         return safe_str(self._repo.get_description() or b'unknown')
     @LazyProperty
     def contact(self):
         undefined_contact = u'Unknown'
         return undefined_contact
     @property
     def branches(self):
         if not self.revisions:
             return {}
         sortkey = lambda ctx: ctx[0]
         _branches = [(key, ascii_str(sha))
+        _branches = [(safe_str(key), ascii_str(sha))
                      for key, (sha, type_) in self._parsed_refs.items() if type_ == b'H']
         return OrderedDict(sorted(_branches, key=sortkey, reverse=False))
     @LazyProperty
     def closed_branches(self):
         return {}
     @LazyProperty
     def tags(self):
         return self._get_tags()
     def _get_tags(self):
         if not self.revisions:
             return {}
         sortkey = lambda ctx: ctx[0]
         _tags = [(key, ascii_str(sha))
+        _tags = [(safe_str(key), ascii_str(sha))
                  for key, (sha, type_) in self._parsed_refs.items() if type_ == b'T']
         return OrderedDict(sorted(_tags, key=sortkey, reverse=True))
     def tag(self, name, user, revision=None, message=None, date=None,
             **kwargs):
         """
         Creates and returns a tag for the given ``revision``.
         :param name: name for new tag
         :param user: full username, i.e.: "Joe Doe <joe.doe@example.com>"
         :param revision: changeset id for which new tag would be created
         :param message: message of the tag's commit
         :param date: date of tag's commit
         :raises TagAlreadyExistError: if tag with same name already exists
         """
         if name in self.tags:
             raise TagAlreadyExistError("Tag %s already exists" % name)
         changeset = self.get_changeset(revision)
         message = message or "Added tag %s for commit %s" % (name,
             changeset.raw_id)
         self._repo.refs[b"refs/tags/%s" % safe_bytes(name)] = changeset._commit.id
         self._parsed_refs = self._get_parsed_refs()
         self.tags = self._get_tags()
         return changeset
     def remove_tag(self, name, user, message=None, date=None):
         """
         Removes tag with the given ``name``.
         :param name: name of the tag to be removed
         :param user: full username, i.e.: "Joe Doe <joe.doe@example.com>"
         :param message: message of the tag's removal commit
         :param date: date of tag's removal commit
         :raises TagDoesNotExistError: if tag with given name does not exists
         """
         if name not in self.tags:
             raise TagDoesNotExistError("Tag %s does not exist" % name)
         # self._repo.refs is a DiskRefsContainer, and .path gives the full absolute path of '.git'
         tagpath = os.path.join(self._repo.refs.path, 'refs', 'tags', name)
+        tagpath = os.path.join(safe_str(self._repo.refs.path), 'refs', 'tags', name)
         try:
             os.remove(tagpath)
             self._parsed_refs = self._get_parsed_refs()
             self.tags = self._get_tags()
         except OSError as e:
             raise RepositoryError(e.strerror)
     @LazyProperty
     def bookmarks(self):
         """
         Gets bookmarks for this repository
         """
         return {}
     @LazyProperty
     def _parsed_refs(self):
         return self._get_parsed_refs()
     def _get_parsed_refs(self):
         """Return refs as a dict, like:
         { b'v0.2.0': [b'599ba911aa24d2981225f3966eb659dfae9e9f30', b'T'] }
         """
         _repo = self._repo
         refs = _repo.get_refs()
         keys = [(b'refs/heads/', b'H'),
                 (b'refs/remotes/origin/', b'RH'),
                 (b'refs/tags/', b'T')]
         _refs = {}
         for ref, sha in refs.items():
             for k, type_ in keys:
                 if ref.startswith(k):
                     _key = ref[len(k):]
                     if type_ == b'T':
                         obj = _repo.get_object(sha)
                         if isinstance(obj, Tag):
                             sha = _repo.get_object(sha).object[1]
                     _refs[_key] = [sha, type_]
                     break
         return _refs
     def _heads(self, reverse=False):
         refs = self._repo.get_refs()
         heads = {}
         for key, val in refs.items():
             for ref_key in [b'refs/heads/', b'refs/remotes/origin/']:
                 if key.startswith(ref_key):
                     n = key[len(ref_key):]
@@ @@ -667,70 +667,71 @@ class GitRepository(BaseRepository): @@
         for line in (x for x in so.splitlines()):
             sha, ref = line.split('\t')
             cmd.append('+%s:%s' % (ref, ref))
         self.run_git_command(cmd)
     def _update_server_info(self):
         """
         runs gits update-server-info command in this repo instance
         """
         from dulwich.server import update_server_info
         try:
             update_server_info(self._repo)
         except OSError as e:
             if e.errno not in [errno.ENOENT, errno.EROFS]:
                 raise
             # Workaround for dulwich crashing on for example its own dulwich/tests/data/repos/simple_merge.git/info/refs.lock
             log.error('Ignoring %s running update-server-info: %s', type(e).__name__, e)
     @LazyProperty
     def workdir(self):
         """
         Returns ``Workdir`` instance for this repository.
         """
         return GitWorkdir(self)
     def get_config_value(self, section, name, config_file=None):
         """
         Returns configuration value for a given [``section``] and ``name``.
         :param section: Section we want to retrieve value from
         :param name: Name of configuration we want to retrieve
         :param config_file: A path to file which should be used to retrieve
           configuration from (might also be a list of file paths)
         """
         if config_file is None:
             config_file = []
         elif isinstance(config_file, str):
             config_file = [config_file]
         def gen_configs():
             for path in config_file + self._config_files:
                 try:
                     yield ConfigFile.from_path(path)
                 except (IOError, OSError, ValueError):
                     continue
         for config in gen_configs():
             try:
-                return config.get(section, name)
+                value = config.get(section, name)
             except KeyError:
                 continue
             return None if value is None else safe_str(value)
         return None
     def get_user_name(self, config_file=None):
         """
         Returns user's name from global configuration file.
         :param config_file: A path to file which should be used to retrieve
           configuration from (might also be a list of file paths)
         """
         return self.get_config_value('user', 'name', config_file)
     def get_user_email(self, config_file=None):
         """
         Returns user's email from global configuration file.
         :param config_file: A path to file which should be used to retrieve
           configuration from (might also be a list of file paths)
         """
         return self.get_config_value('user', 'email', config_file)

kallithea/lib/vcs/backends/git/workdir.py

➞

Show inline comments

 import re
 from kallithea.lib.utils2 import ascii_str
+from kallithea.lib.utils2 import ascii_str, safe_str
 from kallithea.lib.vcs.backends.base import BaseWorkdir
 from kallithea.lib.vcs.exceptions import BranchDoesNotExistError, RepositoryError
 class GitWorkdir(BaseWorkdir):
     def get_branch(self):
         headpath = self.repository._repo.refs.refpath(b'HEAD')
         try:
             content = open(headpath).read()
+            content = safe_str(open(headpath, 'rb').read())
             match = re.match(r'^ref: refs/heads/(?P<branch>.+)\n$', content)
             if match:
                 return match.groupdict()['branch']
             else:
                 raise RepositoryError("Couldn't compute workdir's branch")
         except IOError:
             # Try naive way...
             raise RepositoryError("Couldn't compute workdir's branch")
     def get_changeset(self):
         wk_dir_id = ascii_str(self.repository._repo.refs.as_dict().get(b'HEAD'))
         return self.repository.get_changeset(wk_dir_id)
     def checkout_branch(self, branch=None):
         if branch is None:
             branch = self.repository.DEFAULT_BRANCH_NAME
         if branch not in self.repository.branches:
             raise BranchDoesNotExistError
         self.repository.run_git_command(['checkout', branch])

kallithea/lib/vcs/backends/hg/changeset.py

➞

Show inline comments

@@ @@ -76,97 +76,97 @@ class MercurialChangeset(BaseChangeset): @@
     @LazyProperty
     def successors(self):
         successors = mercurial.obsutil.successorssets(self._ctx._repo, self._ctx.node(), closest=True)
         if successors:
             # flatten the list here handles both divergent (len > 1)
             # and the usual case (len = 1)
             successors = [mercurial.node.hex(n)[:12] for sub in successors for n in sub if n != self._ctx.node()]
         return successors
     @LazyProperty
     def predecessors(self):
         return [mercurial.node.hex(n)[:12] for n in mercurial.obsutil.closestpredecessors(self._ctx._repo, self._ctx.node())]
     @LazyProperty
     def bookmarks(self):
         return [safe_str(bookmark) for bookmark in self._ctx.bookmarks()]
     @LazyProperty
     def message(self):
         return safe_str(self._ctx.description())
     @LazyProperty
     def committer(self):
         return safe_str(self.author)
     @LazyProperty
     def author(self):
         return safe_str(self._ctx.user())
     @LazyProperty
     def date(self):
         return date_fromtimestamp(*self._ctx.date())
     @LazyProperty
     def _timestamp(self):
         return self._ctx.date()[0]
     @LazyProperty
     def status(self):
         """
         Returns modified, added, removed, deleted files for current changeset
         """
         return self.repository._repo.status(self._ctx.p1().node(),
                                             self._ctx.node())
     @LazyProperty
     def _file_paths(self):
         return list(self._ctx)
+        return list(safe_str(f) for f in self._ctx)
     @LazyProperty
     def _dir_paths(self):
         p = list(set(get_dirs_for_path(*self._file_paths)))
         p.insert(0, '')
         return p
     @LazyProperty
     def _paths(self):
         return self._dir_paths + self._file_paths
     @LazyProperty
     def short_id(self):
         return self.raw_id[:12]
     @LazyProperty
     def parents(self):
         """
         Returns list of parents changesets.
         """
         return [self.repository.get_changeset(parent.rev())
                 for parent in self._ctx.parents() if parent.rev() >= 0]
     @LazyProperty
     def children(self):
         """
         Returns list of children changesets.
         """
         return [self.repository.get_changeset(child.rev())
                 for child in self._ctx.children() if child.rev() >= 0]
     def next(self, branch=None):
         if branch and self.branch != branch:
             raise VCSError('Branch option used on changeset not belonging '
                            'to that branch')
         cs = self
         while True:
             try:
                 next_ = cs.repository.revisions.index(cs.raw_id) + 1
                 next_rev = cs.repository.revisions[next_]
             except IndexError:
                 raise ChangesetDoesNotExistError
             cs = cs.repository.get_changeset(next_rev)
             if not branch or branch == cs.branch:
                 return cs
@@ @@ -341,67 +341,67 @@ class MercurialChangeset(BaseChangeset): @@
         als = self.repository.alias
         for k, vals in self._extract_submodules().items():
             #vals = url,rev,type
             loc = vals[0]
             cs = vals[1]
             dirnodes.append(SubModuleNode(k, url=loc, changeset=cs,
                                           alias=als))
         nodes = dirnodes + filenodes
         for node in nodes:
             self.nodes[node.path] = node
         nodes.sort()
         return nodes
     def get_node(self, path):
         """
         Returns ``Node`` object from the given ``path``. If there is no node at
         the given ``path``, ``ChangesetError`` would be raised.
         """
         path = self._fix_path(path)
         if path not in self.nodes:
             if path in self._file_paths:
                 node = FileNode(path, changeset=self)
             elif path in self._dir_paths or path in self._dir_paths:
                 if path == '':
                     node = RootNode(changeset=self)
                 else:
                     node = DirNode(path, changeset=self)
             else:
                 raise NodeDoesNotExistError("There is no file nor directory "
                     "at the given path: '%s' at revision %s"
                     % (path, self.short_id))
             # cache node
             self.nodes[path] = node
         return self.nodes[path]
     @LazyProperty
     def affected_files(self):
         """
         Gets a fast accessible file changes for given changeset
         """
         return self._ctx.files()
     @property
     def added(self):
         """
         Returns list of added ``FileNode`` objects.
         """
         return AddedFileNodesGenerator([n for n in self.status.added], self)
+        return AddedFileNodesGenerator([safe_str(n) for n in self.status.added], self)
     @property
     def changed(self):
         """
         Returns list of modified ``FileNode`` objects.
         """
         return ChangedFileNodesGenerator([n for n in self.status.modified], self)
+        return ChangedFileNodesGenerator([safe_str(n) for n in self.status.modified], self)
     @property
     def removed(self):
         """
         Returns list of removed ``FileNode`` objects.
         """
         return RemovedFileNodesGenerator([n for n in self.status.removed], self)
+        return RemovedFileNodesGenerator([safe_str(n) for n in self.status.removed], self)
     @LazyProperty
     def extra(self):
         return self._ctx.extra()

kallithea/lib/vcs/backends/hg/inmemory.py

➞

Show inline comments

 import datetime
 import mercurial.context
 import mercurial.node
 from kallithea.lib.vcs.backends.base import BaseInMemoryChangeset
 from kallithea.lib.vcs.exceptions import RepositoryError
 from kallithea.lib.vcs.utils import ascii_str, safe_bytes
+from kallithea.lib.vcs.utils import ascii_str, safe_bytes, safe_str
 class MercurialInMemoryChangeset(BaseInMemoryChangeset):
     def commit(self, message, author, parents=None, branch=None, date=None,
                **kwargs):
         """
         Performs in-memory commit (doesn't check workdir in any way) and
         returns newly created ``Changeset``. Updates repository's
         ``revisions``.
         :param message: message of the commit
         :param author: full username, i.e. "Joe Doe <joe.doe@example.com>"
         :param parents: single parent or sequence of parents from which commit
           would be derived
         :param date: ``datetime.datetime`` instance. Defaults to
           ``datetime.datetime.now()``.
         :param branch: branch name, as string. If none given, default backend's
           branch would be used.
         :raises ``CommitError``: if any error occurs while committing
         """
         self.check_integrity(parents)
         from .repository import MercurialRepository
         if not isinstance(message, unicode) or not isinstance(author, unicode):
             raise RepositoryError('Given message and author needs to be '
                                   'an <unicode> instance got %r & %r instead'
                                   % (type(message), type(author)))
         if branch is None:
             branch = MercurialRepository.DEFAULT_BRANCH_NAME
         kwargs[b'branch'] = safe_bytes(branch)
         def filectxfn(_repo, memctx, bytes_path):
             """
             Callback from Mercurial, returning ctx to commit for the given
             path.
             """
-            path = bytes_path  # will be different for py3
+            path = safe_str(bytes_path)
             # check if this path is removed
             if path in (node.path for node in self.removed):
                 return None
             # check if this path is added
             for node in self.added:
                 if node.path == path:
                     return mercurial.context.memfilectx(_repo, memctx, path=bytes_path,
                         data=node.content,
                         islink=False,
                         isexec=node.is_executable,
                         copysource=False)
             # or changed
             for node in self.changed:
                 if node.path == path:
                     return mercurial.context.memfilectx(_repo, memctx, path=bytes_path,
                         data=node.content,
                         islink=False,
                         isexec=node.is_executable,
                         copysource=False)
             raise RepositoryError("Given path haven't been marked as added, "
                                   "changed or removed (%s)" % path)
         parents = [None, None]
         for i, parent in enumerate(self.parents):
             if parent is not None:
                 parents[i] = parent._ctx.node()
         if date and isinstance(date, datetime.datetime):
             date = safe_bytes(date.strftime('%a, %d %b %Y %H:%M:%S'))
         commit_ctx = mercurial.context.memctx(
             repo=self.repository._repo,
             parents=parents,
             text=b'',
             files=[safe_bytes(x) for x in self.get_paths()],
             filectxfn=filectxfn,
             user=safe_bytes(author),
             date=date,
             extra=kwargs)
         # injecting given _repo params
         commit_ctx._text = safe_bytes(message)
         commit_ctx._user = safe_bytes(author)
         commit_ctx._date = date

kallithea/lib/vcs/backends/hg/repository.py

➞

Show inline comments

@@ @@ -435,97 +435,97 @@ class MercurialRepository(BaseRepository @@
             if isinstance(revision, int):
                 return ascii_str(self._repo[revision].hex())
             return ascii_str(mercurial.scmutil.revsymbol(self._repo, revision).hex())
         except (IndexError, ValueError, mercurial.error.RepoLookupError, TypeError):
             msg = "Revision %r does not exist for %s" % (safe_str(revision), self.name)
             raise ChangesetDoesNotExistError(msg)
         except (LookupError, ):
             msg = "Ambiguous identifier `%s` for %s" % (safe_str(revision), self.name)
             raise ChangesetDoesNotExistError(msg)
     def get_ref_revision(self, ref_type, ref_name):
         """
         Returns revision number for the given reference.
         """
         if ref_type == 'rev' and not ref_name.strip('0'):
             return self.EMPTY_CHANGESET
         # lookup up the exact node id
         _revset_predicates = {
                 'branch': 'branch',
                 'book': 'bookmark',
                 'tag': 'tag',
                 'rev': 'id',
+            }
         # avoid expensive branch(x) iteration over whole repo
         rev_spec = "%%s & %s(%%s)" % _revset_predicates[ref_type]
         try:
             revs = self._repo.revs(rev_spec, ref_name, ref_name)
         except LookupError:
             msg = "Ambiguous identifier %s:%s for %s" % (ref_type, ref_name, self.name)
             raise ChangesetDoesNotExistError(msg)
         except mercurial.error.RepoLookupError:
             msg = "Revision %s:%s does not exist for %s" % (ref_type, ref_name, self.name)
             raise ChangesetDoesNotExistError(msg)
         if revs:
             revision = revs.last()
         else:
             # TODO: just report 'not found'?
             revision = ref_name
         return self._get_revision(revision)
     def _get_archives(self, archive_name='tip'):
         allowed = self.baseui.configlist(b"web", b"allow_archive",
                                          untrusted=True)
         for name, ext in [(b'zip', '.zip'), (b'gz', '.tar.gz'), (b'bz2', '.tar.bz2')]:
             if name in allowed or self._repo.ui.configbool(b"web",
                                                            b"allow" + name,
                                                            untrusted=True):
                 yield {"type": name, "extension": ext, "node": archive_name}
+                yield {"type": safe_str(name), "extension": ext, "node": archive_name}
     def _get_url(self, url):
         """
         Returns normalized url. If schema is not given, fall back to
         filesystem (``file:///``) schema.
         """
         if url != 'default' and '://' not in url:
             url = "file:" + urllib.request.pathname2url(url)
         return url
     def get_changeset(self, revision=None):
         """
         Returns ``MercurialChangeset`` object representing repository's
         changeset at the given ``revision``.
         """
         return MercurialChangeset(repository=self, revision=self._get_revision(revision))
     def get_changesets(self, start=None, end=None, start_date=None,
                        end_date=None, branch_name=None, reverse=False, max_revisions=None):
         """
         Returns iterator of ``MercurialChangeset`` objects from start to end
         (both are inclusive)
         :param start: None, str, int or mercurial lookup format
         :param end:  None, str, int or mercurial lookup format
         :param start_date:
         :param end_date:
         :param branch_name:
         :param reversed: return changesets in reversed order
         """
         start_raw_id = self._get_revision(start)
         start_pos = None if start is None else self.revisions.index(start_raw_id)
         end_raw_id = self._get_revision(end)
         end_pos = None if end is None else self.revisions.index(end_raw_id)
         if start_pos is not None and end_pos is not None and start_pos > end_pos:
             raise RepositoryError("Start revision '%s' cannot be "
                                   "after end revision '%s'" % (start, end))
         if branch_name and branch_name not in self.allbranches:
             msg = "Branch %r not found in %s" % (branch_name, self.name)
             raise BranchDoesNotExistError(msg)
         if end_pos is not None:
             end_pos += 1
         # filter branches
         filter_ = []
         if branch_name:
             filter_.append(b'branch("%s")' % safe_bytes(branch_name))
@@ @@ -544,73 +544,74 @@ class MercurialRepository(BaseRepository @@
         else:
             revisions = self.revisions
         # this is very much a hack to turn this into a list; a better solution
         # would be to get rid of this function entirely and use revsets
         revs = list(revisions)[start_pos:end_pos]
         if reverse:
             revs.reverse()
         return CollectionGenerator(self, revs)
     def pull(self, url):
         """
         Tries to pull changes from external location.
         """
         other = mercurial.hg.peer(self._repo, {}, safe_bytes(self._get_url(url)))
         try:
             mercurial.exchange.pull(self._repo, other, heads=None, force=None)
         except mercurial.error.Abort as err:
             # Propagate error but with vcs's type
             raise RepositoryError(str(err))
     @LazyProperty
     def workdir(self):
         """
         Returns ``Workdir`` instance for this repository.
         """
         return MercurialWorkdir(self)
     def get_config_value(self, section, name=None, config_file=None):
         """
         Returns configuration value for a given [``section``] and ``name``.
         :param section: Section we want to retrieve value from
         :param name: Name of configuration we want to retrieve
         :param config_file: A path to file which should be used to retrieve
           configuration from (might also be a list of file paths)
         """
         if config_file is None:
             config_file = []
         elif isinstance(config_file, str):
             config_file = [config_file]
         config = self._repo.ui
         if config_file:
             config = mercurial.ui.ui()
             for path in config_file:
                 config.readconfig(safe_bytes(path))
         return config.config(safe_bytes(section), safe_bytes(name))
         value = config.config(safe_bytes(section), safe_bytes(name))
         return value if value is None else safe_str(value)
     def get_user_name(self, config_file=None):
         """
         Returns user's name from global configuration file.
         :param config_file: A path to file which should be used to retrieve
           configuration from (might also be a list of file paths)
         """
         username = self.get_config_value('ui', 'username', config_file=config_file)
         if username:
             return author_name(username)
         return None
     def get_user_email(self, config_file=None):
         """
         Returns user's email from global configuration file.
         :param config_file: A path to file which should be used to retrieve
           configuration from (might also be a list of file paths)
         """
         username = self.get_config_value('ui', 'username', config_file=config_file)
         if username:
             return author_email(username)
         return None

kallithea/lib/vcs/backends/hg/workdir.py

➞

Show inline comments

 import mercurial.merge
 from kallithea.lib.vcs.backends.base import BaseWorkdir
 from kallithea.lib.vcs.exceptions import BranchDoesNotExistError
 from kallithea.lib.vcs.utils import ascii_bytes, ascii_str
+from kallithea.lib.vcs.utils import ascii_bytes, ascii_str, safe_str
 class MercurialWorkdir(BaseWorkdir):
     def get_branch(self):
         return self.repository._repo.dirstate.branch()
+        return safe_str(self.repository._repo.dirstate.branch())
     def get_changeset(self):
         wk_dir_id = ascii_str(self.repository._repo[None].parents()[0].hex())
         return self.repository.get_changeset(wk_dir_id)
     def checkout_branch(self, branch=None):
         if branch is None:
             branch = self.repository.DEFAULT_BRANCH_NAME
         if branch not in self.repository.branches:
             raise BranchDoesNotExistError
         raw_id = self.repository.branches[branch]
         mercurial.merge.update(self.repository._repo, ascii_bytes(raw_id), False, False, None)

0 comments (0 inline, 0 general)