kallithea Changeset - e51ad2cd400e

Changeset - e51ad2cd400e

Parent rev.

Child rev.

[Not reviewed]

default

0 32 0

Mads Kiilerich - 6 years ago 2019-12-26 13:16:14
mads@kiilerich.com

Grafted from: 0c7491817eb8

py3: drop the last uses of safe_str - they are no longer relevant when we don't have a separate unicode type

32 files changed with 111 insertions and 154 deletions:

kallithea/bin/kallithea_cli_repo.py

kallithea/controllers/api/__init__.py

kallithea/controllers/compare.py

kallithea/controllers/files.py

kallithea/controllers/login.py

kallithea/controllers/pullrequests.py

kallithea/controllers/search.py

kallithea/lib/auth_modules/auth_container.py

kallithea/lib/auth_modules/auth_ldap.py

kallithea/lib/base.py

kallithea/lib/caching_query.py

kallithea/lib/hooks.py

kallithea/lib/indexers/daemon.py

kallithea/lib/middleware/simplehg.py

kallithea/lib/utils.py

kallithea/lib/utils2.py

kallithea/lib/vcs/backends/git/changeset.py

kallithea/lib/vcs/backends/git/repository.py

kallithea/lib/vcs/backends/git/ssh.py

kallithea/lib/vcs/backends/hg/changeset.py

kallithea/lib/vcs/backends/hg/repository.py

kallithea/lib/vcs/backends/ssh.py

kallithea/lib/vcs/nodes.py

kallithea/lib/vcs/utils/__init__.py

kallithea/model/db.py

kallithea/model/repo.py

kallithea/model/scm.py

kallithea/tests/base.py

kallithea/tests/functional/test_admin_repos.py

kallithea/tests/functional/test_forks.py

kallithea/tests/vcs/test_hg.py

kallithea/tests/vcs/test_vcs.py

0 comments (0 inline, 0 general)

kallithea/bin/kallithea_cli_repo.py

➞

Show inline comments

 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
+#
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 This file was forked by the Kallithea project in July 2014 and later moved.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Feb 9, 2013
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 import datetime
 import os
 import re
 import shutil
 import click
 import kallithea.bin.kallithea_cli_base as cli_base
 from kallithea.lib.utils import REMOVED_REPO_PAT, repo2db_mapper
-from kallithea.lib.utils2 import ask_ok, safe_str
 from kallithea.lib.utils2 import ask_ok
 from kallithea.model.db import Repository, Ui
 from kallithea.model.meta import Session
 from kallithea.model.scm import ScmModel
 @cli_base.register_command(config_file_initialize_app=True)
 @click.option('--remove-missing', is_flag=True,
         help='Remove missing repositories from the Kallithea database.')
 def repo_scan(remove_missing):
     """Scan filesystem for repositories.
     Search the configured repository root for new repositories and add them
     into Kallithea.
     Additionally, report repositories that were previously known to Kallithea
     but are no longer present on the filesystem. If option --remove-missing is
     given, remove the missing repositories from the Kallithea database.
     """
     click.echo('Now scanning root location for new repos ...')
     added, removed = repo2db_mapper(ScmModel().repo_scan(),
                                     remove_obsolete=remove_missing)
     click.echo('Scan completed.')
     if added:
         click.echo('Added: %s' % ', '.join(added))
     if removed:
         click.echo('%s: %s' % ('Removed' if remove_missing else 'Missing',
                           ', '.join(removed)))
 @cli_base.register_command(config_file_initialize_app=True)
 @click.argument('repositories', nargs=-1)
 def repo_update_metadata(repositories):
     """
     Update repository metadata in database from repository content.
     In normal operation, Kallithea will keep caches up-to-date
     automatically. However, if repositories are externally modified, e.g. by
     a direct push via the filesystem rather than via a Kallithea URL,
     Kallithea is not aware of it. In this case, you should manually run this
     command to update the repository cache.
     If no repositories are specified, the caches of all repositories are
     updated.
     """
     if not repositories:
         repo_list = Repository.query().all()
     else:
         repo_names = [n.strip() for n in repositories]
         repo_list = list(Repository.query()
                         .filter(Repository.repo_name.in_(repo_names)))
     for repo in repo_list:
         # update latest revision metadata in database
         repo.update_changeset_cache()
         # invalidate in-memory VCS object cache... will be repopulated on
         # first access
         repo.set_invalidate()
     Session().commit()
     click.echo('Updated database with information about latest change in the following %s repositories:' % (len(repo_list)))
     click.echo('\n'.join(repo.repo_name for repo in repo_list))
 @cli_base.register_command(config_file_initialize_app=True)
 @click.option('--ask/--no-ask', default=True, help='Ask for confirmation or not. Default is --ask.')
 @click.option('--older-than',
         help="""Only purge repositories that have been removed at least the given time ago.
         For example, '--older-than=30d' purges repositories deleted 30 days ago or longer.
         Possible suffixes: d (days), h (hours), m (minutes), s (seconds).""")
 def repo_purge_deleted(ask, older_than):
     """Purge backups of deleted repositories.
     When a repository is deleted via the Kallithea web interface, the actual
     data is still present on the filesystem but set aside using a special name.
     This command allows to delete these files permanently.
     """
     def _parse_older_than(val):
         regex = re.compile(r'((?P<days>\d+?)d)?((?P<hours>\d+?)h)?((?P<minutes>\d+?)m)?((?P<seconds>\d+?)s)?')
         parts = regex.match(val)
         if not parts:
             return
         parts = parts.groupdict()
         time_params = {}
         for name, param in parts.items():
             if param:
                 time_params[name] = int(param)
         return datetime.timedelta(**time_params)
     def _extract_date(name):
         """
         Extract the date part from rm__<date> pattern of removed repos,
         and convert it to datetime object
         :param name:
         """
         date_part = name[4:19]  # 4:19 since we don't parse milliseconds
         return datetime.datetime.strptime(date_part, '%Y%m%d_%H%M%S')
     repos_location = Ui.get_repos_location()
     to_remove = []
-    for dn_, dirs, f in os.walk(safe_str(repos_location)):
     for dn_, dirs, f in os.walk(repos_location):
         alldirs = list(dirs)
         del dirs[:]
         if ('.hg' in alldirs or
             '.git' in alldirs or
             '.svn' in alldirs or
             'objects' in alldirs and ('refs' in alldirs or 'packed-refs' in f)
         ):
             continue
         for loc in alldirs:
             if REMOVED_REPO_PAT.match(loc):
                 to_remove.append([os.path.join(dn_, loc),
                                   _extract_date(loc)])
             else:
                 dirs.append(loc)
         if dirs:
             click.echo('Scanning: %s' % dn_)
     if not to_remove:
         click.echo('There are no deleted repositories.')
         return
     # filter older than (if present)!
     if older_than:
         now = datetime.datetime.now()
         to_remove_filtered = []
         older_than_date = _parse_older_than(older_than)
         for name, date_ in to_remove:
             repo_age = now - date_
             if repo_age > older_than_date:
                 to_remove_filtered.append([name, date_])
         to_remove = to_remove_filtered
         if not to_remove:
             click.echo('There are no deleted repositories older than %s (%s)'
                     % (older_than, older_than_date))
             return
         click.echo('Considering %s deleted repositories older than %s (%s).'
             % (len(to_remove), older_than, older_than_date))
     else:
         click.echo('Considering %s deleted repositories.' % len(to_remove))
     if not ask:
         remove = True
     else:
         remove = ask_ok('The following repositories will be removed completely:\n%s\n'
                 'Do you want to proceed? [y/n] '
                 % '\n'.join(['%s deleted on %s' % (safe_str(x[0]), safe_str(x[1]))
                                      for x in to_remove]))
             'Do you want to proceed? [y/n] ' %
             '\n'.join('%s deleted on %s' % (path, date_) for path, date_ in to_remove))
     if remove:
         for path, date_ in to_remove:
             click.echo('Purging repository %s' % path)
             shutil.rmtree(path)
     else:
         click.echo('Nothing done, exiting...')

kallithea/controllers/api/__init__.py

➞

Show inline comments

 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
+#
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.controllers.api
 ~~~~~~~~~~~~~~~~~~~~~~~~~
 JSON RPC controller
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Aug 20, 2011
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 import inspect
 import itertools
 import logging
 import time
 import traceback
 import types
 from tg import Response, TGController, request, response
 from webob.exc import HTTPError, HTTPException
 from kallithea.lib import ext_json
 from kallithea.lib.auth import AuthUser
 from kallithea.lib.base import _get_ip_addr as _get_ip
 from kallithea.lib.base import get_path_info
-from kallithea.lib.utils2 import ascii_bytes, safe_str
 from kallithea.lib.utils2 import ascii_bytes
 from kallithea.model.db import User
 log = logging.getLogger('JSONRPC')
 class JSONRPCError(BaseException):
     def __init__(self, message):
         self.message = message
         super(JSONRPCError, self).__init__()
     def __str__(self):
-        return safe_str(self.message)
         return self.message
 class JSONRPCErrorResponse(Response, HTTPException):
     """
     Generate a Response object with a JSON-RPC error body
     """
     def __init__(self, message=None, retid=None, code=None):
         HTTPException.__init__(self, message, self)
         Response.__init__(self,
                           json_body=dict(id=retid, result=None, error=message),
                           status=code,
                           content_type='application/json')
 class JSONRPCController(TGController):
     """
      A WSGI-speaking JSON-RPC controller class
      See the specification:
      <http://json-rpc.org/wiki/specification>`.
      Valid controller return values should be json-serializable objects.
      Sub-classes should catch their exceptions and raise JSONRPCError
      if they want to pass meaningful errors to the client.
      """
     def _get_ip_addr(self, environ):
         return _get_ip(environ)
     def _get_method_args(self):
         """
         Return `self._rpc_args` to dispatched controller method
         chosen by __call__
         """
         return self._rpc_args
     def _dispatch(self, state, remainder=None):
         """
         Parse the request body as JSON, look up the method on the
         controller and if it exists, dispatch to it.
         """
         # Since we are here we should respond as JSON
         response.content_type = 'application/json'
         environ = state.request.environ
         start = time.time()
         ip_addr = self._get_ip_addr(environ)
         self._req_id = None
         if 'CONTENT_LENGTH' not in environ:
             log.debug("No Content-Length")
             raise JSONRPCErrorResponse(retid=self._req_id,
                                        message="No Content-Length in request")
         else:
             length = environ['CONTENT_LENGTH'] or 0
             length = int(environ['CONTENT_LENGTH'])
             log.debug('Content-Length: %s', length)
         if length == 0:
             raise JSONRPCErrorResponse(retid=self._req_id,
                                        message="Content-Length is 0")
         raw_body = environ['wsgi.input'].read(length)
         try:
             json_body = ext_json.loads(raw_body)
         except ValueError as e:
             # catch JSON errors Here
             raise JSONRPCErrorResponse(retid=self._req_id,
                                        message="JSON parse error ERR:%s RAW:%r"
                                                 % (e, raw_body))
         # check AUTH based on API key
         try:
             self._req_api_key = json_body['api_key']
             self._req_id = json_body['id']
             self._req_method = json_body['method']
             self._request_params = json_body['args']
             if not isinstance(self._request_params, dict):
                 self._request_params = {}
             log.debug('method: %s, params: %s',
                       self._req_method, self._request_params)
         except KeyError as e:
             raise JSONRPCErrorResponse(retid=self._req_id,
                                        message='Incorrect JSON query missing %s' % e)
         # check if we can find this session using api_key
         try:
             u = User.get_by_api_key(self._req_api_key)
             auth_user = AuthUser.make(dbuser=u, ip_addr=ip_addr)
             if auth_user is None:
                 raise JSONRPCErrorResponse(retid=self._req_id,
                                            message='Invalid API key')
         except Exception as e:
             raise JSONRPCErrorResponse(retid=self._req_id,
                                        message='Invalid API key')
         request.authuser = auth_user
         request.ip_addr = ip_addr
         self._error = None
         try:
             self._func = self._find_method()
         except AttributeError as e:
             raise JSONRPCErrorResponse(retid=self._req_id,
                                        message=str(e))
         # now that we have a method, add self._req_params to
         # self.kargs and dispatch control to WGIController
         argspec = inspect.getfullargspec(self._func)
         arglist = argspec.args[1:]
         argtypes = [type(arg) for arg in argspec.defaults or []]
         default_empty = type(NotImplemented)
         # kw arguments required by this method
         func_kwargs = dict(itertools.zip_longest(reversed(arglist), reversed(argtypes),
                                                   fillvalue=default_empty))
         # This attribute will need to be first param of a method that uses
         # api_key, which is translated to instance of user at that name
         USER_SESSION_ATTR = 'apiuser'
         # get our arglist and check if we provided them as args
         for arg, default in func_kwargs.items():
             if arg == USER_SESSION_ATTR:
                 # USER_SESSION_ATTR is something translated from API key and
                 # this is checked before so we don't need validate it
                 continue
             # skip the required param check if it's default value is
             # NotImplementedType (default_empty)
             if default == default_empty and arg not in self._request_params:
                 raise JSONRPCErrorResponse(
                     retid=self._req_id,
                     message='Missing non optional `%s` arg in JSON DATA' % arg,
+                )
         extra = set(self._request_params).difference(func_kwargs)
         if extra:
             raise JSONRPCErrorResponse(
                 retid=self._req_id,
                 message='Unknown %s arg in JSON DATA' %
                         ', '.join('`%s`' % arg for arg in extra),
+            )
         self._rpc_args = {}
         self._rpc_args.update(self._request_params)
         self._rpc_args['action'] = self._req_method
         self._rpc_args['environ'] = environ
         log.info('IP: %s Request to %s time: %.3fs' % (
             self._get_ip_addr(environ),
             get_path_info(environ), time.time() - start)
+        )
         state.set_action(self._rpc_call, [])
         state.set_params(self._rpc_args)
         return state
     def _rpc_call(self, action, environ, **rpc_args):
         """
         Call the specified RPC Method
         """
         raw_response = ''
         try:
             raw_response = getattr(self, action)(**rpc_args)
             if isinstance(raw_response, HTTPError):
                 self._error = str(raw_response)
         except JSONRPCError as e:
             self._error = unicode(e)
         except Exception as e:
             log.error('Encountered unhandled exception: %s',
                       traceback.format_exc(),)
             json_exc = JSONRPCError('Internal server error')
             self._error = unicode(json_exc)
         if self._error is not None:
             raw_response = None
         response = dict(id=self._req_id, result=raw_response, error=self._error)
         try:
             return ascii_bytes(ext_json.dumps(response))
         except TypeError as e:
             log.error('API FAILED. Error encoding response for %s %s: %s\n%s', action, rpc_args, e, traceback.format_exc())
             return ascii_bytes(ext_json.dumps(
                 dict(
                     id=self._req_id,
                     result=None,
                     error="Error encoding response",

kallithea/controllers/compare.py

➞

Show inline comments

 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
+#
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.controllers.compare
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 compare controller showing differences between two
 repos, branches, bookmarks or tips
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: May 6, 2012
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 import logging
 import re
 import mercurial.unionrepo
 from tg import request
 from tg import tmpl_context as c
 from tg.i18n import ugettext as _
 from webob.exc import HTTPBadRequest, HTTPFound, HTTPNotFound
 from kallithea.config.routing import url
 from kallithea.controllers.changeset import _context_url, _ignorews_url
 from kallithea.lib import diffs
 from kallithea.lib import helpers as h
 from kallithea.lib.auth import HasRepoPermissionLevelDecorator, LoginRequired
 from kallithea.lib.base import BaseRepoController, render
 from kallithea.lib.graphmod import graph_data
-from kallithea.lib.utils2 import ascii_bytes, ascii_str, safe_bytes, safe_int, safe_str
 from kallithea.lib.utils2 import ascii_bytes, ascii_str, safe_bytes, safe_int
 from kallithea.model.db import Repository
 log = logging.getLogger(__name__)
 class CompareController(BaseRepoController):
     def _before(self, *args, **kwargs):
         super(CompareController, self)._before(*args, **kwargs)
         # The base repository has already been retrieved.
         c.a_repo = c.db_repo
         # Retrieve the "changeset" repository (default: same as base).
         other_repo = request.GET.get('other_repo', None)
         if other_repo is None:
             c.cs_repo = c.a_repo
         else:
             c.cs_repo = Repository.get_by_repo_name(other_repo)
             if c.cs_repo is None:
                 msg = _('Could not find other repository %s') % other_repo
                 h.flash(msg, category='error')
                 raise HTTPFound(location=url('compare_home', repo_name=c.a_repo.repo_name))
         # Verify that it's even possible to compare these two repositories.
         if c.a_repo.scm_instance.alias != c.cs_repo.scm_instance.alias:
             msg = _('Cannot compare repositories of different types')
             h.flash(msg, category='error')
             raise HTTPFound(location=url('compare_home', repo_name=c.a_repo.repo_name))
     @staticmethod
     def _get_changesets(alias, org_repo, org_rev, other_repo, other_rev):
         """
         Returns lists of changesets that can be merged from org_repo@org_rev
         to other_repo@other_rev
         ... and the other way
         ... and the ancestors that would be used for merge
         :param org_repo: repo object, that is most likely the original repo we forked from
         :param org_rev: the revision we want our compare to be made
         :param other_repo: repo object, most likely the fork of org_repo. It has
             all changesets that we need to obtain
         :param other_rev: revision we want out compare to be made on other_repo
         """
         ancestors = None
         if org_rev == other_rev:
             org_changesets = []
             other_changesets = []
         elif alias == 'hg':
             # case two independent repos
             if org_repo != other_repo:
                 hgrepo = mercurial.unionrepo.makeunionrepository(other_repo.baseui,
                                                        safe_bytes(other_repo.path),
                                                        safe_bytes(org_repo.path))
                 # all ancestors of other_rev will be in other_repo and
                 # rev numbers from hgrepo can be used in other_repo - org_rev ancestors cannot
             # no remote compare do it on the same repository
             else:
                 hgrepo = other_repo._repo
             ancestors = [ascii_str(hgrepo[ancestor].hex()) for ancestor in
                          hgrepo.revs(b"id(%s) & ::id(%s)", ascii_bytes(other_rev), ascii_bytes(org_rev))]
             if ancestors:
                 log.debug("shortcut found: %s is already an ancestor of %s", other_rev, org_rev)
             else:
                 log.debug("no shortcut found: %s is not an ancestor of %s", other_rev, org_rev)
                 ancestors = [ascii_str(hgrepo[ancestor].hex()) for ancestor in
                              hgrepo.revs(b"heads(::id(%s) & ::id(%s))", ascii_bytes(org_rev), ascii_bytes(other_rev))] # FIXME: expensive!
             other_changesets = [
                 other_repo.get_changeset(rev)
                 for rev in hgrepo.revs(
                     b"ancestors(id(%s)) and not ancestors(id(%s)) and not id(%s)",
                     ascii_bytes(other_rev), ascii_bytes(org_rev), ascii_bytes(org_rev))
+            ]
             org_changesets = [
                 org_repo.get_changeset(ascii_str(hgrepo[rev].hex()))
                 for rev in hgrepo.revs(
                     b"ancestors(id(%s)) and not ancestors(id(%s)) and not id(%s)",
                     ascii_bytes(org_rev), ascii_bytes(other_rev), ascii_bytes(other_rev))
+            ]
         elif alias == 'git':
             if org_repo != other_repo:
                 from dulwich.repo import Repo
                 from dulwich.client import SubprocessGitClient
                 gitrepo = Repo(org_repo.path)
-                SubprocessGitClient(thin_packs=False).fetch(safe_str(other_repo.path), gitrepo)
                 SubprocessGitClient(thin_packs=False).fetch(other_repo.path, gitrepo)
                 gitrepo_remote = Repo(other_repo.path)
-                SubprocessGitClient(thin_packs=False).fetch(safe_str(org_repo.path), gitrepo_remote)
                 SubprocessGitClient(thin_packs=False).fetch(org_repo.path, gitrepo_remote)
                 revs = [
                     ascii_str(x.commit.id)
                     for x in gitrepo_remote.get_walker(include=[ascii_bytes(other_rev)],
                                                        exclude=[ascii_bytes(org_rev)])
+                ]
                 other_changesets = [other_repo.get_changeset(rev) for rev in reversed(revs)]
                 if other_changesets:
                     ancestors = [other_changesets[0].parents[0].raw_id]
                 else:
                     # no changesets from other repo, ancestor is the other_rev
                     ancestors = [other_rev]
                 gitrepo.close()
                 gitrepo_remote.close()
             else:
                 so = org_repo.run_git_command(
                     ['log', '--reverse', '--pretty=format:%H',
                      '-s', '%s..%s' % (org_rev, other_rev)]
+                )
                 other_changesets = [org_repo.get_changeset(cs)
                               for cs in re.findall(r'[0-9a-fA-F]{40}', so)]
                 so = org_repo.run_git_command(
                     ['merge-base', org_rev, other_rev]
+                )
                 ancestors = [re.findall(r'[0-9a-fA-F]{40}', so)[0]]
             org_changesets = []
         else:
             raise Exception('Bad alias only git and hg is allowed')
         return other_changesets, org_changesets, ancestors
     @LoginRequired(allow_default_user=True)
     @HasRepoPermissionLevelDecorator('read')
     def index(self, repo_name):
         c.compare_home = True
         c.a_ref_name = c.cs_ref_name = None
         return render('compare/compare_diff.html')
     @LoginRequired(allow_default_user=True)
     @HasRepoPermissionLevelDecorator('read')
     def compare(self, repo_name, org_ref_type, org_ref_name, other_ref_type, other_ref_name):
         org_ref_name = org_ref_name.strip()
         other_ref_name = other_ref_name.strip()
         # If merge is True:
         #   Show what org would get if merged with other:
         #   List changesets that are ancestors of other but not of org.
         #   New changesets in org is thus ignored.
         #   Diff will be from common ancestor, and merges of org to other will thus be ignored.
         # If merge is False:
         #   Make a raw diff from org to other, no matter if related or not.
         #   Changesets in one and not in the other will be ignored
         merge = bool(request.GET.get('merge'))
         # fulldiff disables cut_off_limit
         fulldiff = request.GET.get('fulldiff')
         # partial uses compare_cs.html template directly
         partial = request.environ.get('HTTP_X_PARTIAL_XHR')
         # is_ajax_preview puts hidden input field with changeset revisions
         c.is_ajax_preview = partial and request.GET.get('is_ajax_preview')
         # swap url for compare_diff page - never partial and never is_ajax_preview
         c.swap_url = h.url('compare_url',
             repo_name=c.cs_repo.repo_name,
             org_ref_type=other_ref_type, org_ref_name=other_ref_name,
             other_repo=c.a_repo.repo_name,
             other_ref_type=org_ref_type, other_ref_name=org_ref_name,
             merge=merge or '')
         # set callbacks for generating markup for icons
         c.ignorews_url = _ignorews_url
         c.context_url = _context_url
         ignore_whitespace = request.GET.get('ignorews') == '1'
         line_context = safe_int(request.GET.get('context'), 3)
         c.a_rev = self._get_ref_rev(c.a_repo, org_ref_type, org_ref_name,
             returnempty=True)
         c.cs_rev = self._get_ref_rev(c.cs_repo, other_ref_type, other_ref_name)
         c.compare_home = False
         c.a_ref_name = org_ref_name
         c.a_ref_type = org_ref_type
         c.cs_ref_name = other_ref_name
         c.cs_ref_type = other_ref_type
         c.cs_ranges, c.cs_ranges_org, c.ancestors = self._get_changesets(
             c.a_repo.scm_instance.alias, c.a_repo.scm_instance, c.a_rev,
             c.cs_repo.scm_instance, c.cs_rev)
         raw_ids = [x.raw_id for x in c.cs_ranges]
         c.cs_comments = c.cs_repo.get_comments(raw_ids)
         c.cs_statuses = c.cs_repo.statuses(raw_ids)
         revs = [ctx.revision for ctx in reversed(c.cs_ranges)]
         c.jsdata = graph_data(c.cs_repo.scm_instance, revs)
         if partial:
             return render('compare/compare_cs.html')
         org_repo = c.a_repo
         other_repo = c.cs_repo
         if merge:
             rev1 = msg = None
             if not c.cs_ranges:
                 msg = _('Cannot show empty diff')
             elif not c.ancestors:
                 msg = _('No ancestor found for merge diff')
             elif len(c.ancestors) == 1:
                 rev1 = c.ancestors[0]
             else:
                 msg = _('Multiple merge ancestors found for merge compare')
             if rev1 is None:
                 h.flash(msg, category='error')
                 log.error(msg)
                 raise HTTPNotFound
             # case we want a simple diff without incoming changesets,
             # previewing what will be merged.
             # Make the diff on the other repo (which is known to have other_rev)
             log.debug('Using ancestor %s as rev1 instead of %s',
                       rev1, c.a_rev)
             org_repo = other_repo
         else: # comparing tips, not necessarily linearly related
             if org_repo != other_repo:
                 # TODO: we could do this by using hg unionrepo
                 log.error('cannot compare across repos %s and %s', org_repo, other_repo)
                 h.flash(_('Cannot compare repositories without using common ancestor'), category='error')
                 raise HTTPBadRequest
             rev1 = c.a_rev
         diff_limit = None if fulldiff else self.cut_off_limit
         log.debug('running diff between %s and %s in %s',
                   rev1, c.cs_rev, org_repo.scm_instance.path)
         raw_diff = diffs.get_diff(org_repo.scm_instance, rev1=rev1, rev2=c.cs_rev,
                                       ignore_whitespace=ignore_whitespace,
                                       context=line_context)
         diff_processor = diffs.DiffProcessor(raw_diff, diff_limit=diff_limit)
         c.limited_diff = diff_processor.limited_diff
         c.file_diff_data = []
         c.lines_added = 0
         c.lines_deleted = 0
         for f in diff_processor.parsed:
             st = f['stats']
             c.lines_added += st['added']
             c.lines_deleted += st['deleted']
             filename = f['filename']
             fid = h.FID('', filename)
             html_diff = diffs.as_html(enable_comments=False, parsed_lines=[f])
             c.file_diff_data.append((fid, None, f['operation'], f['old_filename'], filename, html_diff, st))
         return render('compare/compare_diff.html')

kallithea/controllers/files.py

➞

Show inline comments

 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
+#
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.controllers.files
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~
 Files controller for Kallithea
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Apr 21, 2010
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 import logging
 import os
 import posixpath
 import shutil
 import tempfile
 import traceback
 from collections import OrderedDict
 from tg import request, response
 from tg import tmpl_context as c
 from tg.i18n import ugettext as _
 from webob.exc import HTTPFound, HTTPNotFound
 from kallithea.config.routing import url
 from kallithea.controllers.changeset import _context_url, _ignorews_url, anchor_url, get_ignore_ws, get_line_ctx
 from kallithea.lib import diffs
 from kallithea.lib import helpers as h
 from kallithea.lib.auth import HasRepoPermissionLevelDecorator, LoginRequired
 from kallithea.lib.base import BaseRepoController, jsonify, render
 from kallithea.lib.exceptions import NonRelativePathError
 from kallithea.lib.utils import action_logger
-from kallithea.lib.utils2 import convert_line_endings, detect_mode, safe_int, safe_str, safe_unicode, str2bool
 from kallithea.lib.utils2 import convert_line_endings, detect_mode, safe_int, safe_unicode, str2bool
 from kallithea.lib.vcs.backends.base import EmptyChangeset
 from kallithea.lib.vcs.conf import settings
 from kallithea.lib.vcs.exceptions import (
     ChangesetDoesNotExistError, ChangesetError, EmptyRepositoryError, ImproperArchiveTypeError, NodeAlreadyExistsError, NodeDoesNotExistError, NodeError, RepositoryError, VCSError)
 from kallithea.lib.vcs.nodes import FileNode
 from kallithea.model.db import Repository
 from kallithea.model.repo import RepoModel
 from kallithea.model.scm import ScmModel
 log = logging.getLogger(__name__)
 class FilesController(BaseRepoController):
     def _before(self, *args, **kwargs):
         super(FilesController, self)._before(*args, **kwargs)
     def __get_cs(self, rev, silent_empty=False):
         """
         Safe way to get changeset if error occur it redirects to tip with
         proper message
         :param rev: revision to fetch
         :silent_empty: return None if repository is empty
         """
         try:
             return c.db_repo_scm_instance.get_changeset(rev)
         except EmptyRepositoryError as e:
             if silent_empty:
                 return None
             url_ = url('files_add_home',
                        repo_name=c.repo_name,
                        revision=0, f_path='', anchor='edit')
             add_new = h.link_to(_('Click here to add new file'), url_, class_="alert-link")
             h.flash(_('There are no files yet.') + ' ' + add_new, category='warning')
             raise HTTPNotFound()
         except (ChangesetDoesNotExistError, LookupError):
             msg = _('Such revision does not exist for this repository')
             h.flash(msg, category='error')
             raise HTTPNotFound()
         except RepositoryError as e:
             h.flash(e, category='error')
             raise HTTPNotFound()
     def __get_filenode(self, cs, path):
         """
         Returns file_node or raise HTTP error.
         :param cs: given changeset
         :param path: path to lookup
         """
         try:
             file_node = cs.get_node(path)
             if file_node.is_dir():
                 raise RepositoryError('given path is a directory')
         except ChangesetDoesNotExistError:
             msg = _('Such revision does not exist for this repository')
             h.flash(msg, category='error')
             raise HTTPNotFound()
         except RepositoryError as e:
             h.flash(e, category='error')
             raise HTTPNotFound()
         return file_node
     @LoginRequired(allow_default_user=True)
     @HasRepoPermissionLevelDecorator('read')
     def index(self, repo_name, revision, f_path, annotate=False):
         # redirect to given revision from form if given
         post_revision = request.POST.get('at_rev', None)
         if post_revision:
             cs = self.__get_cs(post_revision) # FIXME - unused!
         c.revision = revision
         c.changeset = self.__get_cs(revision)
         c.branch = request.GET.get('branch', None)
         c.f_path = f_path
         c.annotate = annotate
         cur_rev = c.changeset.revision
         # used in files_source.html:
         c.cut_off_limit = self.cut_off_limit
         c.fulldiff = request.GET.get('fulldiff')
         # prev link
         try:
             prev_rev = c.db_repo_scm_instance.get_changeset(cur_rev).prev(c.branch)
             c.url_prev = url('files_home', repo_name=c.repo_name,
                          revision=prev_rev.raw_id, f_path=f_path)
             if c.branch:
                 c.url_prev += '?branch=%s' % c.branch
         except (ChangesetDoesNotExistError, VCSError):
             c.url_prev = '#'
         # next link
         try:
             next_rev = c.db_repo_scm_instance.get_changeset(cur_rev).next(c.branch)
             c.url_next = url('files_home', repo_name=c.repo_name,
                      revision=next_rev.raw_id, f_path=f_path)
             if c.branch:
                 c.url_next += '?branch=%s' % c.branch
         except (ChangesetDoesNotExistError, VCSError):
             c.url_next = '#'
         # files or dirs
         try:
             c.file = c.changeset.get_node(f_path)
             if c.file.is_submodule():
                 raise HTTPFound(location=c.file.url)
             elif c.file.is_file():
                 c.load_full_history = False
                 # determine if we're on branch head
                 _branches = c.db_repo_scm_instance.branches
                 c.on_branch_head = revision in _branches or revision in _branches.values()
                 _hist = []
                 c.file_history = []
                 if c.load_full_history:
                     c.file_history, _hist = self._get_node_history(c.changeset, f_path)
                 c.authors = []
                 for a in set([x.author for x in _hist]):
                     c.authors.append((h.email(a), h.person(a)))
             else:
                 c.authors = c.file_history = []
         except RepositoryError as e:
             h.flash(e, category='error')
             raise HTTPNotFound()
         if request.environ.get('HTTP_X_PARTIAL_XHR'):
             return render('files/files_ypjax.html')
         # TODO: tags and bookmarks?
         c.revision_options = [(c.changeset.raw_id,
                               _('%s at %s') % (b, h.short_id(c.changeset.raw_id))) for b in c.changeset.branches] + \
             [(n, b) for b, n in c.db_repo_scm_instance.branches.items()]
         if c.db_repo_scm_instance.closed_branches:
             prefix = _('(closed)') + ' '
             c.revision_options += [('-', '-')] + \
                 [(n, prefix + b) for b, n in c.db_repo_scm_instance.closed_branches.items()]
         return render('files/files.html')
     @LoginRequired(allow_default_user=True)
     @HasRepoPermissionLevelDecorator('read')
     @jsonify
     def history(self, repo_name, revision, f_path):
         changeset = self.__get_cs(revision)
         _file = changeset.get_node(f_path)
         if _file.is_file():
             file_history, _hist = self._get_node_history(changeset, f_path)
             res = []
             for obj in file_history:
                 res.append({
                     'text': obj[1],
                     'children': [{'id': o[0], 'text': o[1]} for o in obj[0]]
                 })
             data = {
                 'more': False,
                 'results': res
+            }
             return data
     @LoginRequired(allow_default_user=True)
     @HasRepoPermissionLevelDecorator('read')
     def authors(self, repo_name, revision, f_path):
         changeset = self.__get_cs(revision)
         _file = changeset.get_node(f_path)
         if _file.is_file():
             file_history, _hist = self._get_node_history(changeset, f_path)
             c.authors = []
             for a in set([x.author for x in _hist]):
                 c.authors.append((h.email(a), h.person(a)))
             return render('files/files_history_box.html')
     @LoginRequired(allow_default_user=True)
     @HasRepoPermissionLevelDecorator('read')
     def rawfile(self, repo_name, revision, f_path):
         cs = self.__get_cs(revision)
         file_node = self.__get_filenode(cs, f_path)
         response.content_disposition = 'attachment; filename=%s' % \
             safe_str(f_path.split(Repository.url_sep())[-1])
         response.content_disposition = \
             'attachment; filename=%s' % f_path.split(Repository.url_sep())[-1]
         response.content_type = file_node.mimetype
         return file_node.content
     @LoginRequired(allow_default_user=True)
     @HasRepoPermissionLevelDecorator('read')
     def raw(self, repo_name, revision, f_path):
         cs = self.__get_cs(revision)
         file_node = self.__get_filenode(cs, f_path)
         raw_mimetype_mapping = {
             # map original mimetype to a mimetype used for "show as raw"
             # you can also provide a content-disposition to override the
             # default "attachment" disposition.
             # orig_type: (new_type, new_dispo)
             # show images inline:
             'image/x-icon': ('image/x-icon', 'inline'),
             'image/png': ('image/png', 'inline'),
             'image/gif': ('image/gif', 'inline'),
             'image/jpeg': ('image/jpeg', 'inline'),
             'image/svg+xml': ('image/svg+xml', 'inline'),
+        }
         mimetype = file_node.mimetype
         try:
             mimetype, dispo = raw_mimetype_mapping[mimetype]
         except KeyError:
             # we don't know anything special about this, handle it safely
             if file_node.is_binary:
                 # do same as download raw for binary files
                 mimetype, dispo = 'application/octet-stream', 'attachment'
             else:
                 # do not just use the original mimetype, but force text/plain,
                 # otherwise it would serve text/html and that might be unsafe.
                 # Note: underlying vcs library fakes text/plain mimetype if the
                 # mimetype can not be determined and it thinks it is not
                 # binary.This might lead to erroneous text display in some
                 # cases, but helps in other cases, like with text files
                 # without extension.
                 mimetype, dispo = 'text/plain', 'inline'
         if dispo == 'attachment':
             dispo = 'attachment; filename=%s' % \
                         safe_str(f_path.split(os.sep)[-1])
             dispo = 'attachment; filename=%s' % f_path.split(os.sep)[-1]
         response.content_disposition = dispo
         response.content_type = mimetype
         return file_node.content
     @LoginRequired()
     @HasRepoPermissionLevelDecorator('write')
     def delete(self, repo_name, revision, f_path):
         repo = c.db_repo
         # check if revision is a branch identifier- basically we cannot
         # create multiple heads via file editing
         _branches = repo.scm_instance.branches
         # check if revision is a branch name or branch hash
         if revision not in _branches and revision not in _branches.values():
             h.flash(_('You can only delete files with revision '
                       'being a valid branch'), category='warning')
             raise HTTPFound(location=h.url('files_home',
                                   repo_name=repo_name, revision='tip',
                                   f_path=f_path))
         r_post = request.POST
         c.cs = self.__get_cs(revision)
         c.file = self.__get_filenode(c.cs, f_path)
         c.default_message = _('Deleted file %s via Kallithea') % (f_path)
         c.f_path = f_path
         node_path = f_path
         author = request.authuser.full_contact
         if r_post:
             message = r_post.get('message') or c.default_message
             try:
                 nodes = {
                     node_path: {
                         'content': ''
+                    }
+                }
                 self.scm_model.delete_nodes(
                     user=request.authuser.user_id,
                     ip_addr=request.ip_addr,
                     repo=c.db_repo,
                     message=message,
                     nodes=nodes,
                     parent_cs=c.cs,
                     author=author,
+                )
                 h.flash(_('Successfully deleted file %s') % f_path,
                         category='success')
             except Exception:
                 log.error(traceback.format_exc())
                 h.flash(_('Error occurred during commit'), category='error')
             raise HTTPFound(location=url('changeset_home',
                                 repo_name=c.repo_name, revision='tip'))
         return render('files/files_delete.html')
     @LoginRequired()
     @HasRepoPermissionLevelDecorator('write')
     def edit(self, repo_name, revision, f_path):
         repo = c.db_repo
         # check if revision is a branch identifier- basically we cannot
         # create multiple heads via file editing
         _branches = repo.scm_instance.branches
         # check if revision is a branch name or branch hash
         if revision not in _branches and revision not in _branches.values():
             h.flash(_('You can only edit files with revision '
                       'being a valid branch'), category='warning')
             raise HTTPFound(location=h.url('files_home',
                                   repo_name=repo_name, revision='tip',
                                   f_path=f_path))
         r_post = request.POST
         c.cs = self.__get_cs(revision)
         c.file = self.__get_filenode(c.cs, f_path)
         if c.file.is_binary:
             raise HTTPFound(location=url('files_home', repo_name=c.repo_name,
                             revision=c.cs.raw_id, f_path=f_path))
         c.default_message = _('Edited file %s via Kallithea') % (f_path)
         c.f_path = f_path
         if r_post:
             old_content = safe_unicode(c.file.content)
             sl = old_content.splitlines(1)
             first_line = sl[0] if sl else ''
             # modes:  0 - Unix, 1 - Mac, 2 - DOS
             mode = detect_mode(first_line, 0)
             content = convert_line_endings(r_post.get('content', ''), mode)
             message = r_post.get('message') or c.default_message
             author = request.authuser.full_contact
             if content == old_content:
                 h.flash(_('No changes'), category='warning')
                 raise HTTPFound(location=url('changeset_home', repo_name=c.repo_name,
                                     revision='tip'))
             try:
                 self.scm_model.commit_change(repo=c.db_repo_scm_instance,
                                              repo_name=repo_name, cs=c.cs,
                                              user=request.authuser.user_id,
                                              ip_addr=request.ip_addr,
                                              author=author, message=message,
                                              content=content, f_path=f_path)
                 h.flash(_('Successfully committed to %s') % f_path,
                         category='success')
             except Exception:
                 log.error(traceback.format_exc())
                 h.flash(_('Error occurred during commit'), category='error')
             raise HTTPFound(location=url('changeset_home',
                                 repo_name=c.repo_name, revision='tip'))
         return render('files/files_edit.html')
     @LoginRequired()
     @HasRepoPermissionLevelDecorator('write')
     def add(self, repo_name, revision, f_path):
         repo = c.db_repo
         r_post = request.POST
         c.cs = self.__get_cs(revision, silent_empty=True)
         if c.cs is None:
             c.cs = EmptyChangeset(alias=c.db_repo_scm_instance.alias)
         c.default_message = (_('Added file via Kallithea'))
         c.f_path = f_path
         if r_post:
             unix_mode = 0
             content = convert_line_endings(r_post.get('content', ''), unix_mode)
             message = r_post.get('message') or c.default_message
             filename = r_post.get('filename')
             location = r_post.get('location', '')
             file_obj = r_post.get('upload_file', None)
             if file_obj is not None and hasattr(file_obj, 'filename'):
                 filename = file_obj.filename
                 content = file_obj.file
                 if hasattr(content, 'file'):
                     # non posix systems store real file under file attr
                     content = content.file
             if not content:
                 h.flash(_('No content'), category='warning')
                 raise HTTPFound(location=url('changeset_home', repo_name=c.repo_name,
                                     revision='tip'))
             if not filename:
                 h.flash(_('No filename'), category='warning')
                 raise HTTPFound(location=url('changeset_home', repo_name=c.repo_name,
                                     revision='tip'))
             # strip all crap out of file, just leave the basename
             filename = os.path.basename(filename)
             node_path = posixpath.join(location, filename)
             author = request.authuser.full_contact
             try:
                 nodes = {
                     node_path: {
                         'content': content
+                    }
+                }
                 self.scm_model.create_nodes(
                     user=request.authuser.user_id,
                     ip_addr=request.ip_addr,
                     repo=c.db_repo,
                     message=message,
                     nodes=nodes,
                     parent_cs=c.cs,
                     author=author,
+                )
                 h.flash(_('Successfully committed to %s') % node_path,
                         category='success')
             except NonRelativePathError as e:
                 h.flash(_('Location must be relative path and must not '
                           'contain .. in path'), category='warning')
                 raise HTTPFound(location=url('changeset_home', repo_name=c.repo_name,
                                     revision='tip'))
             except (NodeError, NodeAlreadyExistsError) as e:
                 h.flash(_(e), category='error')
             except Exception:
                 log.error(traceback.format_exc())
                 h.flash(_('Error occurred during commit'), category='error')
             raise HTTPFound(location=url('changeset_home',
                                 repo_name=c.repo_name, revision='tip'))
         return render('files/files_add.html')
     @LoginRequired(allow_default_user=True)
     @HasRepoPermissionLevelDecorator('read')
     def archivefile(self, repo_name, fname):
         fileformat = None
         revision = None
         ext = None
         subrepos = request.GET.get('subrepos') == 'true'
         for a_type, ext_data in settings.ARCHIVE_SPECS.items():
             archive_spec = fname.split(ext_data[1])
             if len(archive_spec) == 2 and archive_spec[1] == '':
                 fileformat = a_type or ext_data[1]
                 revision = archive_spec[0]
                 ext = ext_data[1]
         try:
             dbrepo = RepoModel().get_by_repo_name(repo_name)
             if not dbrepo.enable_downloads:
                 return _('Downloads disabled') # TODO: do something else?
             if c.db_repo_scm_instance.alias == 'hg':
                 # patch and reset hooks section of UI config to not run any
                 # hooks on fetching archives with subrepos
                 for k, v in c.db_repo_scm_instance._repo.ui.configitems('hooks'):
                     c.db_repo_scm_instance._repo.ui.setconfig('hooks', k, None)
             cs = c.db_repo_scm_instance.get_changeset(revision)
             content_type = settings.ARCHIVE_SPECS[fileformat][0]
         except ChangesetDoesNotExistError:
             return _('Unknown revision %s') % revision
         except EmptyRepositoryError:
             return _('Empty repository')
         except (ImproperArchiveTypeError, KeyError):
             return _('Unknown archive type')
         from kallithea import CONFIG
         rev_name = cs.raw_id[:12]
         archive_name = '%s-%s%s' % (safe_str(repo_name.replace('/', '_')),
                                     safe_str(rev_name), ext)
         archive_name = '%s-%s%s' % (repo_name.replace('/', '_'), rev_name, ext)
         archive_path = None
         cached_archive_path = None
         archive_cache_dir = CONFIG.get('archive_cache_dir')
         if archive_cache_dir and not subrepos: # TODO: subrepo caching?
             if not os.path.isdir(archive_cache_dir):
                 os.makedirs(archive_cache_dir)
             cached_archive_path = os.path.join(archive_cache_dir, archive_name)
             if os.path.isfile(cached_archive_path):
                 log.debug('Found cached archive in %s', cached_archive_path)
                 archive_path = cached_archive_path
             else:
                 log.debug('Archive %s is not yet cached', archive_name)
         if archive_path is None:
             # generate new archive
             fd, archive_path = tempfile.mkstemp()
             log.debug('Creating new temp archive in %s', archive_path)
             with os.fdopen(fd, 'wb') as stream:
                 cs.fill_archive(stream=stream, kind=fileformat, subrepos=subrepos)
                 # stream (and thus fd) has been closed by cs.fill_archive
             if cached_archive_path is not None:
                 # we generated the archive - move it to cache
                 log.debug('Storing new archive in %s', cached_archive_path)
                 shutil.move(archive_path, cached_archive_path)
                 archive_path = cached_archive_path
         def get_chunked_archive(archive_path):
             stream = open(archive_path, 'rb')
             while True:
                 data = stream.read(16 * 1024)
                 if not data:
                     break
                 yield data
             stream.close()
             if archive_path != cached_archive_path:
                 log.debug('Destroying temp archive %s', archive_path)
                 os.remove(archive_path)
         action_logger(user=request.authuser,
                       action='user_downloaded_archive:%s' % (archive_name),
                       repo=repo_name, ipaddr=request.ip_addr, commit=True)
         response.content_disposition = str('attachment; filename=%s' % (archive_name))
         response.content_type = str(content_type)
         return get_chunked_archive(archive_path)
     @LoginRequired(allow_default_user=True)
     @HasRepoPermissionLevelDecorator('read')
     def diff(self, repo_name, f_path):
         ignore_whitespace = request.GET.get('ignorews') == '1'
         line_context = safe_int(request.GET.get('context'), 3)
         diff2 = request.GET.get('diff2', '')
         diff1 = request.GET.get('diff1', '') or diff2
         c.action = request.GET.get('diff')
         c.no_changes = diff1 == diff2
         c.f_path = f_path
         c.big_diff = False
         fulldiff = request.GET.get('fulldiff')
         c.anchor_url = anchor_url
         c.ignorews_url = _ignorews_url
         c.context_url = _context_url
         c.changes = OrderedDict()
         c.changes[diff2] = []
         # special case if we want a show rev only, it's impl here
         # to reduce JS and callbacks
         if request.GET.get('show_rev'):
             if str2bool(request.GET.get('annotate', 'False')):
                 _url = url('files_annotate_home', repo_name=c.repo_name,
                            revision=diff1, f_path=c.f_path)
             else:
                 _url = url('files_home', repo_name=c.repo_name,
                            revision=diff1, f_path=c.f_path)
             raise HTTPFound(location=_url)
         try:
             if diff1 not in ['', None, 'None', '0' * 12, '0' * 40]:
                 c.changeset_1 = c.db_repo_scm_instance.get_changeset(diff1)
                 try:
                     node1 = c.changeset_1.get_node(f_path)
                     if node1.is_dir():
                         raise NodeError('%s path is a %s not a file'
                                         % (node1, type(node1)))
                 except NodeDoesNotExistError:
                     c.changeset_1 = EmptyChangeset(cs=diff1,
                                                    revision=c.changeset_1.revision,
                                                    repo=c.db_repo_scm_instance)
                     node1 = FileNode(f_path, '', changeset=c.changeset_1)
             else:
                 c.changeset_1 = EmptyChangeset(repo=c.db_repo_scm_instance)
                 node1 = FileNode(f_path, '', changeset=c.changeset_1)
             if diff2 not in ['', None, 'None', '0' * 12, '0' * 40]:
                 c.changeset_2 = c.db_repo_scm_instance.get_changeset(diff2)
                 try:
                     node2 = c.changeset_2.get_node(f_path)
                     if node2.is_dir():
                         raise NodeError('%s path is a %s not a file'
                                         % (node2, type(node2)))
                 except NodeDoesNotExistError:
                     c.changeset_2 = EmptyChangeset(cs=diff2,
                                                    revision=c.changeset_2.revision,
                                                    repo=c.db_repo_scm_instance)
                     node2 = FileNode(f_path, '', changeset=c.changeset_2)
             else:
                 c.changeset_2 = EmptyChangeset(repo=c.db_repo_scm_instance)
                 node2 = FileNode(f_path, '', changeset=c.changeset_2)
         except (RepositoryError, NodeError):
             log.error(traceback.format_exc())
             raise HTTPFound(location=url('files_home', repo_name=c.repo_name,
                                 f_path=f_path))
         if c.action == 'download':
             raw_diff = diffs.get_gitdiff(node1, node2,
                                       ignore_whitespace=ignore_whitespace,
                                       context=line_context)
             diff_name = '%s_vs_%s.diff' % (diff1, diff2)
             response.content_type = 'text/plain'
             response.content_disposition = (
                 'attachment; filename=%s' % diff_name
+            )
             return raw_diff
         elif c.action == 'raw':
             raw_diff = diffs.get_gitdiff(node1, node2,
                                       ignore_whitespace=ignore_whitespace,
                                       context=line_context)
             response.content_type = 'text/plain'
             return raw_diff
         else:
             fid = h.FID(diff2, node2.path)
             line_context_lcl = get_line_ctx(fid, request.GET)
             ign_whitespace_lcl = get_ignore_ws(fid, request.GET)
             diff_limit = None if fulldiff else self.cut_off_limit
             c.a_rev, c.cs_rev, a_path, diff, st, op = diffs.wrapped_diff(filenode_old=node1,
                                          filenode_new=node2,
                                          diff_limit=diff_limit,
                                          ignore_whitespace=ign_whitespace_lcl,
                                          line_context=line_context_lcl,
                                          enable_comments=False)
             c.file_diff_data = [(fid, fid, op, a_path, node2.path, diff, st)]
             return render('files/file_diff.html')
     @LoginRequired(allow_default_user=True)
     @HasRepoPermissionLevelDecorator('read')
     def diff_2way(self, repo_name, f_path):
         diff1 = request.GET.get('diff1', '')
         diff2 = request.GET.get('diff2', '')
         try:
             if diff1 not in ['', None, 'None', '0' * 12, '0' * 40]:
                 c.changeset_1 = c.db_repo_scm_instance.get_changeset(diff1)
                 try:
                     node1 = c.changeset_1.get_node(f_path)
                     if node1.is_dir():
                         raise NodeError('%s path is a %s not a file'
                                         % (node1, type(node1)))
                 except NodeDoesNotExistError:
                     c.changeset_1 = EmptyChangeset(cs=diff1,
                                                    revision=c.changeset_1.revision,
                                                    repo=c.db_repo_scm_instance)
                     node1 = FileNode(f_path, '', changeset=c.changeset_1)
             else:
                 c.changeset_1 = EmptyChangeset(repo=c.db_repo_scm_instance)
                 node1 = FileNode(f_path, '', changeset=c.changeset_1)
             if diff2 not in ['', None, 'None', '0' * 12, '0' * 40]:
                 c.changeset_2 = c.db_repo_scm_instance.get_changeset(diff2)
                 try:
                     node2 = c.changeset_2.get_node(f_path)
                     if node2.is_dir():
                         raise NodeError('%s path is a %s not a file'
                                         % (node2, type(node2)))
                 except NodeDoesNotExistError:
                     c.changeset_2 = EmptyChangeset(cs=diff2,
                                                    revision=c.changeset_2.revision,
                                                    repo=c.db_repo_scm_instance)
                     node2 = FileNode(f_path, '', changeset=c.changeset_2)
             else:
                 c.changeset_2 = EmptyChangeset(repo=c.db_repo_scm_instance)
                 node2 = FileNode(f_path, '', changeset=c.changeset_2)
         except ChangesetDoesNotExistError as e:
             msg = _('Such revision does not exist for this repository')
             h.flash(msg, category='error')
             raise HTTPNotFound()
         c.node1 = node1
         c.node2 = node2
         c.cs1 = c.changeset_1

kallithea/controllers/login.py

➞

Show inline comments

 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
+#
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.controllers.login
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~
 Login controller for Kallithea
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Apr 22, 2010
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 import logging
 import re
 import formencode
 from formencode import htmlfill
 from tg import request, session
 from tg import tmpl_context as c
 from tg.i18n import ugettext as _
 from webob.exc import HTTPBadRequest, HTTPFound
 import kallithea.lib.helpers as h
 from kallithea.config.routing import url
 from kallithea.lib.auth import AuthUser, HasPermissionAnyDecorator
 from kallithea.lib.base import BaseController, log_in_user, render
 from kallithea.lib.exceptions import UserCreationError
 from kallithea.lib.utils2 import safe_str
 from kallithea.model.db import Setting, User
 from kallithea.model.forms import LoginForm, PasswordResetConfirmationForm, PasswordResetRequestForm, RegisterForm
 from kallithea.model.meta import Session
 from kallithea.model.user import UserModel
 log = logging.getLogger(__name__)
 class LoginController(BaseController):
     def _validate_came_from(self, came_from,
             _re=re.compile(r"/(?!/)[-!#$%&'()*+,./:;=?@_~0-9A-Za-z]*$")):
         """Return True if came_from is valid and can and should be used.
         Determines if a URI reference is valid and relative to the origin;
         or in RFC 3986 terms, whether it matches this production:
           origin-relative-ref = path-absolute [ "?" query ] [ "#" fragment ]
         with the exception that '%' escapes are not validated and '#' is
         allowed inside the fragment part.
         """
         return _re.match(came_from) is not None
     def index(self):
-        c.came_from = safe_str(request.GET.get('came_from', ''))
         c.came_from = request.GET.get('came_from', '')
         if c.came_from:
             if not self._validate_came_from(c.came_from):
                 log.error('Invalid came_from (not server-relative): %r', c.came_from)
                 raise HTTPBadRequest()
         else:
             c.came_from = url('home')
         if request.POST:
             # import Login Form validator class
             login_form = LoginForm()()
             try:
                 c.form_result = login_form.to_python(dict(request.POST))
                 # form checks for username/password, now we're authenticated
                 username = c.form_result['username']
                 user = User.get_by_username_or_email(username, case_insensitive=True)
             except formencode.Invalid as errors:
                 defaults = errors.value
                 # remove password from filling in form again
                 defaults.pop('password', None)
                 return htmlfill.render(
                     render('/login.html'),
                     defaults=errors.value,
                     errors=errors.error_dict or {},
                     prefix_error=False,
                     encoding="UTF-8",
                     force_defaults=False)
             except UserCreationError as e:
                 # container auth or other auth functions that create users on
                 # the fly can throw this exception signaling that there's issue
                 # with user creation, explanation should be provided in
                 # Exception itself
                 h.flash(e, 'error')
             else:
                 auth_user = log_in_user(user, c.form_result['remember'], is_external_auth=False, ip_addr=request.ip_addr)
                 # TODO: handle auth_user is None as failed authentication?
                 raise HTTPFound(location=c.came_from)
         else:
             # redirect if already logged in
             if not request.authuser.is_anonymous:
                 raise HTTPFound(location=c.came_from)
             # continue to show login to default user
         return render('/login.html')
     @HasPermissionAnyDecorator('hg.admin', 'hg.register.auto_activate',
                                'hg.register.manual_activate')
     def register(self):
         def_user_perms = AuthUser(dbuser=User.get_default_user()).permissions['global']
         c.auto_active = 'hg.register.auto_activate' in def_user_perms
         settings = Setting.get_app_settings()
         captcha_private_key = settings.get('captcha_private_key')
         c.captcha_active = bool(captcha_private_key)
         c.captcha_public_key = settings.get('captcha_public_key')
         if request.POST:
             register_form = RegisterForm()()
             try:
                 form_result = register_form.to_python(dict(request.POST))
                 form_result['active'] = c.auto_active
                 if c.captcha_active:
                     from kallithea.lib.recaptcha import submit
                     response = submit(request.POST.get('g-recaptcha-response'),
                                       private_key=captcha_private_key,
                                       remoteip=request.ip_addr)
                     if not response.is_valid:
                         _value = form_result
                         _msg = _('Bad captcha')
                         error_dict = {'recaptcha_field': _msg}
                         raise formencode.Invalid(_msg, _value, None,
                                                  error_dict=error_dict)
                 UserModel().create_registration(form_result)
                 h.flash(_('You have successfully registered with %s') % (c.site_name or 'Kallithea'),
                         category='success')
                 Session().commit()
                 raise HTTPFound(location=url('login_home'))
             except formencode.Invalid as errors:
                 return htmlfill.render(
                     render('/register.html'),
                     defaults=errors.value,
                     errors=errors.error_dict or {},
                     prefix_error=False,
                     encoding="UTF-8",
                     force_defaults=False)
             except UserCreationError as e:
                 # container auth or other auth functions that create users on
                 # the fly can throw this exception signaling that there's issue
                 # with user creation, explanation should be provided in
                 # Exception itself
                 h.flash(e, 'error')
         return render('/register.html')
     def password_reset(self):
         settings = Setting.get_app_settings()
         captcha_private_key = settings.get('captcha_private_key')
         c.captcha_active = bool(captcha_private_key)
         c.captcha_public_key = settings.get('captcha_public_key')
         if request.POST:
             password_reset_form = PasswordResetRequestForm()()
             try:
                 form_result = password_reset_form.to_python(dict(request.POST))
                 if c.captcha_active:
                     from kallithea.lib.recaptcha import submit
                     response = submit(request.POST.get('g-recaptcha-response'),
                                       private_key=captcha_private_key,
                                       remoteip=request.ip_addr)
                     if not response.is_valid:
                         _value = form_result
                         _msg = _('Bad captcha')
                         error_dict = {'recaptcha_field': _msg}
                         raise formencode.Invalid(_msg, _value, None,
                                                  error_dict=error_dict)
                 redirect_link = UserModel().send_reset_password_email(form_result)
                 h.flash(_('A password reset confirmation code has been sent'),
                             category='success')
                 raise HTTPFound(location=redirect_link)
             except formencode.Invalid as errors:
                 return htmlfill.render(
                     render('/password_reset.html'),
                     defaults=errors.value,
                     errors=errors.error_dict or {},
                     prefix_error=False,
                     encoding="UTF-8",
                     force_defaults=False)
         return render('/password_reset.html')
     def password_reset_confirmation(self):
         # This controller handles both GET and POST requests, though we
         # only ever perform the actual password change on POST (since
         # GET requests are not allowed to have side effects, and do not
         # receive automatic CSRF protection).
         # The template needs the email address outside of the form.
         c.email = request.params.get('email')
         c.timestamp = request.params.get('timestamp') or ''
         c.token = request.params.get('token') or ''
         if not request.POST:
             return render('/password_reset_confirmation.html')
         form = PasswordResetConfirmationForm()()
         try:
             form_result = form.to_python(dict(request.POST))
         except formencode.Invalid as errors:
             return htmlfill.render(
                 render('/password_reset_confirmation.html'),
                 defaults=errors.value,
                 errors=errors.error_dict or {},
                 prefix_error=False,
                 encoding='UTF-8')
         if not UserModel().verify_reset_password_token(
             form_result['email'],
             form_result['timestamp'],
             form_result['token'],
         ):
             return htmlfill.render(
                 render('/password_reset_confirmation.html'),
                 defaults=form_result,
                 errors={'token': _('Invalid password reset token')},
                 prefix_error=False,
                 encoding='UTF-8')
         UserModel().reset_password(form_result['email'], form_result['password'])
         h.flash(_('Successfully updated password'), category='success')
         raise HTTPFound(location=url('login_home'))
     def logout(self):
         session.delete()
         log.info('Logging out and deleting session for user')
         raise HTTPFound(location=url('home'))
     def session_csrf_secret_token(self):
         """Return the CSRF protection token for the session - just like it
         could have been screen scraped from a page with a form.
         Only intended for testing but might also be useful for other kinds
         of automation.
         """
         return h.session_csrf_secret_token()

kallithea/controllers/pullrequests.py

➞

Show inline comments

 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
+#
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.controllers.pullrequests
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 pull requests controller for Kallithea for initializing pull requests
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: May 7, 2012
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 import logging
 import traceback
 import formencode
 import mercurial.unionrepo
 from tg import request
 from tg import tmpl_context as c
 from tg.i18n import ugettext as _
 from webob.exc import HTTPBadRequest, HTTPForbidden, HTTPFound, HTTPNotFound
 from kallithea.config.routing import url
 from kallithea.controllers.changeset import _context_url, _ignorews_url, create_cs_pr_comment, delete_cs_pr_comment
 from kallithea.lib import diffs
 from kallithea.lib import helpers as h
 from kallithea.lib.auth import HasRepoPermissionLevelDecorator, LoginRequired
 from kallithea.lib.base import BaseRepoController, jsonify, render
 from kallithea.lib.graphmod import graph_data
 from kallithea.lib.page import Page
-from kallithea.lib.utils2 import ascii_bytes, safe_bytes, safe_int, safe_str
 from kallithea.lib.utils2 import ascii_bytes, safe_bytes, safe_int
 from kallithea.lib.vcs.exceptions import ChangesetDoesNotExistError, EmptyRepositoryError
 from kallithea.model.changeset_status import ChangesetStatusModel
 from kallithea.model.comment import ChangesetCommentsModel
 from kallithea.model.db import ChangesetStatus, PullRequest, PullRequestReviewer, Repository, User
 from kallithea.model.forms import PullRequestForm, PullRequestPostForm
 from kallithea.model.meta import Session
 from kallithea.model.pull_request import CreatePullRequestAction, CreatePullRequestIterationAction, PullRequestModel
 log = logging.getLogger(__name__)
 def _get_reviewer(user_id):
     """Look up user by ID and validate it as a potential reviewer."""
     try:
         user = User.get(int(user_id))
     except ValueError:
         user = None
     if user is None or user.is_default_user:
         h.flash(_('Invalid reviewer "%s" specified') % user_id, category='error')
         raise HTTPBadRequest()
     return user
 class PullrequestsController(BaseRepoController):
     def _get_repo_refs(self, repo, rev=None, branch=None, branch_rev=None):
         """return a structure with repo's interesting changesets, suitable for
         the selectors in pullrequest.html
         rev: a revision that must be in the list somehow and selected by default
         branch: a branch that must be in the list and selected by default - even if closed
         branch_rev: a revision of which peers should be preferred and available."""
         # list named branches that has been merged to this named branch - it should probably merge back
         peers = []
         if rev:
             rev = safe_str(rev)
         if branch:
             branch = safe_str(branch)
         if branch_rev:
             # a revset not restricting to merge() would be better
             # (especially because it would get the branch point)
             # ... but is currently too expensive
             # including branches of children could be nice too
             peerbranches = set()
             for i in repo._repo.revs(
                 b"sort(parents(branch(id(%s)) and merge()) - branch(id(%s)), -rev)",
                 ascii_bytes(branch_rev), ascii_bytes(branch_rev),
             ):
                 for abranch in repo.get_changeset(i).branches:
                     if abranch not in peerbranches:
                         n = 'branch:%s:%s' % (abranch, repo.get_changeset(abranch).raw_id)
                         peers.append((n, abranch))
                         peerbranches.add(abranch)
         selected = None
         tiprev = repo.tags.get('tip')
         tipbranch = None
         branches = []
         for abranch, branchrev in repo.branches.items():
             n = 'branch:%s:%s' % (abranch, branchrev)
             desc = abranch
             if branchrev == tiprev:
                 tipbranch = abranch
                 desc = '%s (current tip)' % desc
             branches.append((n, desc))
             if rev == branchrev:
                 selected = n
             if branch == abranch:
                 if not rev:
                     selected = n
                 branch = None
         if branch:  # branch not in list - it is probably closed
             branchrev = repo.closed_branches.get(branch)
             if branchrev:
                 n = 'branch:%s:%s' % (branch, branchrev)
                 branches.append((n, _('%s (closed)') % branch))
                 selected = n
                 branch = None
             if branch:
                 log.debug('branch %r not found in %s', branch, repo)
         bookmarks = []
         for bookmark, bookmarkrev in repo.bookmarks.items():
             n = 'book:%s:%s' % (bookmark, bookmarkrev)
             bookmarks.append((n, bookmark))
             if rev == bookmarkrev:
                 selected = n
         tags = []
         for tag, tagrev in repo.tags.items():
             if tag == 'tip':
                 continue
             n = 'tag:%s:%s' % (tag, tagrev)
             tags.append((n, tag))
             # note: even if rev == tagrev, don't select the static tag - it must be chosen explicitly
         # prio 1: rev was selected as existing entry above
         # prio 2: create special entry for rev; rev _must_ be used
         specials = []
         if rev and selected is None:
             selected = 'rev:%s:%s' % (rev, rev)
             specials = [(selected, '%s: %s' % (_("Changeset"), rev[:12]))]
         # prio 3: most recent peer branch
         if peers and not selected:
             selected = peers[0][0]
         # prio 4: tip revision
         if not selected:
             if h.is_hg(repo):
                 if tipbranch:
                     selected = 'branch:%s:%s' % (tipbranch, tiprev)
                 else:
                     selected = 'tag:null:' + repo.EMPTY_CHANGESET
                     tags.append((selected, 'null'))
             else:
                 if 'master' in repo.branches:
                     selected = 'branch:master:%s' % repo.branches['master']
                 else:
                     k, v = list(repo.branches.items())[0]
                     selected = 'branch:%s:%s' % (k, v)
         groups = [(specials, _("Special")),
                   (peers, _("Peer branches")),
                   (bookmarks, _("Bookmarks")),
                   (branches, _("Branches")),
                   (tags, _("Tags")),
+                  ]
         return [g for g in groups if g[0]], selected
     def _is_allowed_to_change_status(self, pull_request):
         if pull_request.is_closed():
             return False
         owner = request.authuser.user_id == pull_request.owner_id
         reviewer = PullRequestReviewer.query() \
             .filter(PullRequestReviewer.pull_request == pull_request) \
             .filter(PullRequestReviewer.user_id == request.authuser.user_id) \
             .count() != 0
         return request.authuser.admin or owner or reviewer
     @LoginRequired(allow_default_user=True)
     @HasRepoPermissionLevelDecorator('read')
     def show_all(self, repo_name):
         c.from_ = request.GET.get('from_') or ''
         c.closed = request.GET.get('closed') or ''
         url_params = {}
         if c.from_:
             url_params['from_'] = 1
         if c.closed:
             url_params['closed'] = 1
         p = safe_int(request.GET.get('page'), 1)
         q = PullRequest.query(include_closed=c.closed, sorted=True)
         if c.from_:
             q = q.filter_by(org_repo=c.db_repo)
         else:
             q = q.filter_by(other_repo=c.db_repo)
         c.pull_requests = q.all()
         c.pullrequests_pager = Page(c.pull_requests, page=p, items_per_page=100, **url_params)
         return render('/pullrequests/pullrequest_show_all.html')
     @LoginRequired()
     def show_my(self):
         c.closed = request.GET.get('closed') or ''
         c.my_pull_requests = PullRequest.query(
             include_closed=c.closed,
             sorted=True,
         ).filter_by(owner_id=request.authuser.user_id).all()
         c.participate_in_pull_requests = []
         c.participate_in_pull_requests_todo = []
         done_status = set([ChangesetStatus.STATUS_APPROVED, ChangesetStatus.STATUS_REJECTED])
         for pr in PullRequest.query(
             include_closed=c.closed,
             reviewer_id=request.authuser.user_id,
             sorted=True,
         ):
             status = pr.user_review_status(request.authuser.user_id) # very inefficient!!!
             if status in done_status:
                 c.participate_in_pull_requests.append(pr)
             else:
                 c.participate_in_pull_requests_todo.append(pr)
         return render('/pullrequests/pullrequest_show_my.html')
     @LoginRequired()
     @HasRepoPermissionLevelDecorator('read')
     def index(self):
         org_repo = c.db_repo
         org_scm_instance = org_repo.scm_instance
         try:
             org_scm_instance.get_changeset()
         except EmptyRepositoryError as e:
             h.flash(_('There are no changesets yet'),
                     category='warning')
             raise HTTPFound(location=url('summary_home', repo_name=org_repo.repo_name))
         org_rev = request.GET.get('rev_end')
         # rev_start is not directly useful - its parent could however be used
         # as default for other and thus give a simple compare view
         rev_start = request.GET.get('rev_start')
         other_rev = None
         if rev_start:
             starters = org_repo.get_changeset(rev_start).parents
             if starters:
                 other_rev = starters[0].raw_id
             else:
                 other_rev = org_repo.scm_instance.EMPTY_CHANGESET
         branch = request.GET.get('branch')
         c.cs_repos = [(org_repo.repo_name, org_repo.repo_name)]
         c.default_cs_repo = org_repo.repo_name
         c.cs_refs, c.default_cs_ref = self._get_repo_refs(org_scm_instance, rev=org_rev, branch=branch)
         default_cs_ref_type, default_cs_branch, default_cs_rev = c.default_cs_ref.split(':')
         if default_cs_ref_type != 'branch':
             default_cs_branch = org_repo.get_changeset(default_cs_rev).branch
         # add org repo to other so we can open pull request against peer branches on itself
         c.a_repos = [(org_repo.repo_name, '%s (self)' % org_repo.repo_name)]
         if org_repo.parent:
             # add parent of this fork also and select it.
@@ @@ -397,248 +391,248 @@ class PullrequestsController(BaseRepoCon @@
         new_reviewers = set(_get_reviewer(s) for s in _form['review_members'])
         old_reviewers = set(_get_reviewer(s) for s in _form['org_review_members'])
         other_added = cur_reviewers - old_reviewers
         other_removed = old_reviewers - cur_reviewers
         if other_added:
             h.flash(_('Meanwhile, the following reviewers have been added: %s') %
                     (', '.join(u.username for u in other_added)),
                     category='warning')
         if other_removed:
             h.flash(_('Meanwhile, the following reviewers have been removed: %s') %
                     (', '.join(u.username for u in other_removed)),
                     category='warning')
         if _form['updaterev']:
             return self.create_new_iteration(pull_request,
                                       _form['updaterev'],
                                       _form['pullrequest_title'],
                                       _form['pullrequest_desc'],
                                       new_reviewers)
         added_reviewers = new_reviewers - old_reviewers - cur_reviewers
         removed_reviewers = (old_reviewers - new_reviewers) & cur_reviewers
         old_description = pull_request.description
         pull_request.title = _form['pullrequest_title']
         pull_request.description = _form['pullrequest_desc'].strip() or _('No description')
         pull_request.owner = User.get_by_username(_form['owner'])
         user = User.get(request.authuser.user_id)
         PullRequestModel().mention_from_description(user, pull_request, old_description)
         PullRequestModel().add_reviewers(user, pull_request, added_reviewers)
         PullRequestModel().remove_reviewers(user, pull_request, removed_reviewers)
         Session().commit()
         h.flash(_('Pull request updated'), category='success')
         raise HTTPFound(location=pull_request.url())
     @LoginRequired()
     @HasRepoPermissionLevelDecorator('read')
     @jsonify
     def delete(self, repo_name, pull_request_id):
         pull_request = PullRequest.get_or_404(pull_request_id)
         # only owner can delete it !
         if pull_request.owner_id == request.authuser.user_id:
             PullRequestModel().delete(pull_request)
             Session().commit()
             h.flash(_('Successfully deleted pull request'),
                     category='success')
             raise HTTPFound(location=url('my_pullrequests'))
         raise HTTPForbidden()
     @LoginRequired(allow_default_user=True)
     @HasRepoPermissionLevelDecorator('read')
     def show(self, repo_name, pull_request_id, extra=None):
         c.pull_request = PullRequest.get_or_404(pull_request_id)
         c.allowed_to_change_status = self._is_allowed_to_change_status(c.pull_request)
         cc_model = ChangesetCommentsModel()
         cs_model = ChangesetStatusModel()
         # pull_requests repo_name we opened it against
         # ie. other_repo must match
         if repo_name != c.pull_request.other_repo.repo_name:
             raise HTTPNotFound
         # load compare data into template context
         c.cs_repo = c.pull_request.org_repo
         (c.cs_ref_type,
          c.cs_ref_name,
          c.cs_rev) = c.pull_request.org_ref.split(':')
         c.a_repo = c.pull_request.other_repo
         (c.a_ref_type,
          c.a_ref_name,
          c.a_rev) = c.pull_request.other_ref.split(':') # a_rev is ancestor
         org_scm_instance = c.cs_repo.scm_instance # property with expensive cache invalidation check!!!
         c.cs_ranges = []
         for x in c.pull_request.revisions:
             try:
                 c.cs_ranges.append(org_scm_instance.get_changeset(x))
             except ChangesetDoesNotExistError:
                 c.cs_ranges = []
                 h.flash(_('Revision %s not found in %s') % (x, c.cs_repo.repo_name),
                     'error')
                 break
         c.cs_ranges_org = None # not stored and not important and moving target - could be calculated ...
         revs = [ctx.revision for ctx in reversed(c.cs_ranges)]
         c.jsdata = graph_data(org_scm_instance, revs)
         c.is_range = False
         try:
             if c.a_ref_type == 'rev': # this looks like a free range where target is ancestor
                 cs_a = org_scm_instance.get_changeset(c.a_rev)
                 root_parents = c.cs_ranges[0].parents
                 c.is_range = cs_a in root_parents
                 #c.merge_root = len(root_parents) > 1 # a range starting with a merge might deserve a warning
         except ChangesetDoesNotExistError: # probably because c.a_rev not found
             pass
         except IndexError: # probably because c.cs_ranges is empty, probably because revisions are missing
             pass
         avail_revs = set()
         avail_show = []
         c.cs_branch_name = c.cs_ref_name
         c.a_branch_name = None
         other_scm_instance = c.a_repo.scm_instance
         c.update_msg = ""
         c.update_msg_other = ""
         try:
             if not c.cs_ranges:
                 c.update_msg = _('Error: changesets not found when displaying pull request from %s.') % c.cs_rev
             elif org_scm_instance.alias == 'hg' and c.a_ref_name != 'ancestor':
                 if c.cs_ref_type != 'branch':
                     c.cs_branch_name = org_scm_instance.get_changeset(c.cs_ref_name).branch # use ref_type ?
                 c.a_branch_name = c.a_ref_name
                 if c.a_ref_type != 'branch':
                     try:
                         c.a_branch_name = other_scm_instance.get_changeset(c.a_ref_name).branch # use ref_type ?
                     except EmptyRepositoryError:
                         c.a_branch_name = 'null' # not a branch name ... but close enough
                 # candidates: descendants of old head that are on the right branch
                 #             and not are the old head itself ...
                 #             and nothing at all if old head is a descendant of target ref name
                 if not c.is_range and other_scm_instance._repo.revs('present(%s)::&%s', c.cs_ranges[-1].raw_id, c.a_branch_name):
                     c.update_msg = _('This pull request has already been merged to %s.') % c.a_branch_name
                 elif c.pull_request.is_closed():
                     c.update_msg = _('This pull request has been closed and can not be updated.')
                 else: # look for descendants of PR head on source branch in org repo
                     avail_revs = org_scm_instance._repo.revs('%s:: & branch(%s)',
                                                              revs[0], c.cs_branch_name)
                     if len(avail_revs) > 1: # more than just revs[0]
                         # also show changesets that not are descendants but would be merged in
                         targethead = other_scm_instance.get_changeset(c.a_branch_name).raw_id
                         if org_scm_instance.path != other_scm_instance.path:
                             # Note: org_scm_instance.path must come first so all
                             # valid revision numbers are 100% org_scm compatible
                             # - both for avail_revs and for revset results
                             hgrepo = mercurial.unionrepo.makeunionrepository(org_scm_instance.baseui,
                                                                    safe_bytes(org_scm_instance.path),
                                                                    safe_bytes(other_scm_instance.path))
                         else:
                             hgrepo = org_scm_instance._repo
                         show = set(hgrepo.revs('::%ld & !::parents(%s) & !::%s',
                                                avail_revs, revs[0], targethead))
                         if show:
                             c.update_msg = _('The following additional changes are available on %s:') % c.cs_branch_name
                         else:
                             c.update_msg = _('No additional changesets found for iterating on this pull request.')
                     else:
                         show = set()
                         avail_revs = set() # drop revs[0]
                         c.update_msg = _('No additional changesets found for iterating on this pull request.')
                     # TODO: handle branch heads that not are tip-most
                     brevs = org_scm_instance._repo.revs('%s - %ld - %s', c.cs_branch_name, avail_revs, revs[0])
                     if brevs:
                         # also show changesets that are on branch but neither ancestors nor descendants
                         show.update(org_scm_instance._repo.revs('::%ld - ::%ld - ::%s', brevs, avail_revs, c.a_branch_name))
                         show.add(revs[0]) # make sure graph shows this so we can see how they relate
                         c.update_msg_other = _('Note: Branch %s has another head: %s.') % (c.cs_branch_name,
                             h.short_id(org_scm_instance.get_changeset((max(brevs))).raw_id))
                     avail_show = sorted(show, reverse=True)
             elif org_scm_instance.alias == 'git':
                 c.cs_repo.scm_instance.get_changeset(c.cs_rev) # check it exists - raise ChangesetDoesNotExistError if not
                 c.update_msg = _("Git pull requests don't support iterating yet.")
         except ChangesetDoesNotExistError:
             c.update_msg = _('Error: some changesets not found when displaying pull request from %s.') % c.cs_rev
         c.avail_revs = avail_revs
         c.avail_cs = [org_scm_instance.get_changeset(r) for r in avail_show]
         c.avail_jsdata = graph_data(org_scm_instance, avail_show)
         raw_ids = [x.raw_id for x in c.cs_ranges]
         c.cs_comments = c.cs_repo.get_comments(raw_ids)
         c.cs_statuses = c.cs_repo.statuses(raw_ids)
         ignore_whitespace = request.GET.get('ignorews') == '1'
         line_context = safe_int(request.GET.get('context'), 3)
         c.ignorews_url = _ignorews_url
         c.context_url = _context_url
         fulldiff = request.GET.get('fulldiff')
         diff_limit = None if fulldiff else self.cut_off_limit
         # we swap org/other ref since we run a simple diff on one repo
         log.debug('running diff between %s and %s in %s',
                   c.a_rev, c.cs_rev, org_scm_instance.path)
         try:
-            raw_diff = diffs.get_diff(org_scm_instance, rev1=safe_str(c.a_rev), rev2=safe_str(c.cs_rev),
             raw_diff = diffs.get_diff(org_scm_instance, rev1=c.a_rev, rev2=c.cs_rev,
                                       ignore_whitespace=ignore_whitespace, context=line_context)
         except ChangesetDoesNotExistError:
             raw_diff = safe_bytes(_("The diff can't be shown - the PR revisions could not be found."))
         diff_processor = diffs.DiffProcessor(raw_diff, diff_limit=diff_limit)
         c.limited_diff = diff_processor.limited_diff
         c.file_diff_data = []
         c.lines_added = 0
         c.lines_deleted = 0
         for f in diff_processor.parsed:
             st = f['stats']
             c.lines_added += st['added']
             c.lines_deleted += st['deleted']
             filename = f['filename']
             fid = h.FID('', filename)
             html_diff = diffs.as_html(enable_comments=True, parsed_lines=[f])
             c.file_diff_data.append((fid, None, f['operation'], f['old_filename'], filename, html_diff, st))
         # inline comments
         c.inline_cnt = 0
         c.inline_comments = cc_model.get_inline_comments(
                                 c.db_repo.repo_id,
                                 pull_request=pull_request_id)
         # count inline comments
         for __, lines in c.inline_comments:
             for comments in lines.values():
                 c.inline_cnt += len(comments)
         # comments
         c.comments = cc_model.get_comments(c.db_repo.repo_id, pull_request=pull_request_id)
         # (badly named) pull-request status calculation based on reviewer votes
         (c.pull_request_reviewers,
          c.pull_request_pending_reviewers,
          c.current_voting_result,
          ) = cs_model.calculate_pull_request_result(c.pull_request)
         c.changeset_statuses = ChangesetStatus.STATUSES
         c.is_ajax_preview = False
         c.ancestors = None # [c.a_rev] ... but that is shown in an other way
         return render('/pullrequests/pullrequest_show.html')
     @LoginRequired()
     @HasRepoPermissionLevelDecorator('read')
     @jsonify
     def comment(self, repo_name, pull_request_id):
         pull_request = PullRequest.get_or_404(pull_request_id)
         allowed_to_change_status = self._is_allowed_to_change_status(pull_request)
         return create_cs_pr_comment(repo_name, pull_request=pull_request,
                 allowed_to_change_status=allowed_to_change_status)
     @LoginRequired()
     @HasRepoPermissionLevelDecorator('read')
     @jsonify
     def delete_comment(self, repo_name, comment_id):
         return delete_cs_pr_comment(repo_name, comment_id)

kallithea/controllers/search.py

➞

Show inline comments

 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
+#
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.controllers.search
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 Search controller for Kallithea
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Aug 7, 2010
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 import logging
 import traceback
 from tg import config, request
 from tg import tmpl_context as c
 from tg.i18n import ugettext as _
 from whoosh.index import EmptyIndexError, exists_in, open_dir
 from whoosh.qparser import QueryParser, QueryParserError
 from whoosh.query import Phrase, Prefix
 from kallithea.lib.auth import LoginRequired
 from kallithea.lib.base import BaseRepoController, render
 from kallithea.lib.indexers import CHGSET_IDX_NAME, CHGSETS_SCHEMA, IDX_NAME, SCHEMA, WhooshResultWrapper
 from kallithea.lib.page import Page
-from kallithea.lib.utils2 import safe_int, safe_str
 from kallithea.lib.utils2 import safe_int
 from kallithea.model.repo import RepoModel
 log = logging.getLogger(__name__)
 class SearchController(BaseRepoController):
     @LoginRequired(allow_default_user=True)
     def index(self, repo_name=None):
         c.repo_name = repo_name
         c.formated_results = []
         c.runtime = ''
         c.cur_query = request.GET.get('q', None)
         c.cur_type = request.GET.get('type', 'content')
         c.cur_search = search_type = {'content': 'content',
                                       'commit': 'message',
                                       'path': 'path',
                                       'repository': 'repository'
                                       }.get(c.cur_type, 'content')
         index_name = {
             'content': IDX_NAME,
             'commit': CHGSET_IDX_NAME,
             'path': IDX_NAME
         }.get(c.cur_type, IDX_NAME)
         schema_defn = {
             'content': SCHEMA,
             'commit': CHGSETS_SCHEMA,
             'path': SCHEMA
         }.get(c.cur_type, SCHEMA)
         log.debug('IDX: %s', index_name)
         log.debug('SCHEMA: %s', schema_defn)
         if c.cur_query:
             cur_query = c.cur_query.lower()
             log.debug(cur_query)
         if c.cur_query:
             p = safe_int(request.GET.get('page'), 1)
             highlight_items = set()
             index_dir = config['index_dir']
             try:
                 if not exists_in(index_dir, index_name):
                     raise EmptyIndexError
                 idx = open_dir(index_dir, indexname=index_name)
                 searcher = idx.searcher()
                 qp = QueryParser(search_type, schema=schema_defn)
                 if c.repo_name:
                     # use "repository_rawname:" instead of "repository:"
                     # for case-sensitive matching
                     cur_query = u'repository_rawname:%s %s' % (c.repo_name, cur_query)
                 try:
                     query = qp.parse(unicode(cur_query))
                     # extract words for highlight
                     if isinstance(query, Phrase):
                         highlight_items.update(query.words)
                     elif isinstance(query, Prefix):
                         highlight_items.add(query.text)
                     else:
                         for i in query.all_terms():
                             if i[0] in ['content', 'message']:
                                 highlight_items.add(i[1])
                     matcher = query.matcher(searcher)
                     log.debug('query: %s', query)
                     log.debug('hl terms: %s', highlight_items)
                     results = searcher.search(query)
                     res_ln = len(results)
                     c.runtime = '%s results (%.3f seconds)' % (
                         res_ln, results.runtime
+                    )
                     repo_location = RepoModel().repos_path
                     c.formated_results = Page(
                         WhooshResultWrapper(search_type, searcher, matcher,
                                             highlight_items, repo_location),
                         page=p,
                         item_count=res_ln,
                         items_per_page=10,
                         type=safe_str(c.cur_type),
                         q=safe_str(c.cur_query),
                         type=c.cur_type,
                         q=c.cur_query,
+                    )
                 except QueryParserError:
                     c.runtime = _('Invalid search query. Try quoting it.')
                 searcher.close()
             except EmptyIndexError:
                 log.error("Empty search index - run 'kallithea-cli index-create' regularly")
                 c.runtime = _('The server has no search index.')
             except Exception:
                 log.error(traceback.format_exc())
                 c.runtime = _('An error occurred during search operation.')
         # Return a rendered template
         return render('/search/search.html')

kallithea/lib/auth_modules/auth_container.py

➞

Show inline comments

 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
+#
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.lib.auth_modules.auth_container
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 Kallithea container based authentication plugin
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Created on Nov 17, 2012
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 import logging
 from kallithea.lib import auth_modules
 from kallithea.lib.compat import hybrid_property
-from kallithea.lib.utils2 import safe_str, str2bool
 from kallithea.lib.utils2 import str2bool
 from kallithea.model.db import Setting
 log = logging.getLogger(__name__)
 class KallitheaAuthPlugin(auth_modules.KallitheaExternalAuthPlugin):
     def __init__(self):
         pass
     @hybrid_property
     def name(self):
         return "container"
     @hybrid_property
     def is_container_auth(self):
         return True
     def settings(self):
         settings = [
+            {
                 "name": "header",
                 "validator": self.validators.UnicodeString(strip=True, not_empty=True),
                 "type": "string",
                 "description": "Request header to extract the username from",
                 "default": "REMOTE_USER",
                 "formname": "Username header"
             },
+            {
                 "name": "email_header",
                 "validator": self.validators.UnicodeString(strip=True),
                 "type": "string",
                 "description": "Optional request header to extract the email from",
                 "default": "",
                 "formname": "Email header"
             },
+            {
                 "name": "firstname_header",
                 "validator": self.validators.UnicodeString(strip=True),
                 "type": "string",
                 "description": "Optional request header to extract the first name from",
                 "default": "",
                 "formname": "Firstname header"
             },
+            {
                 "name": "lastname_header",
                 "validator": self.validators.UnicodeString(strip=True),
                 "type": "string",
                 "description": "Optional request header to extract the last name from",
                 "default": "",
                 "formname": "Lastname header"
             },
+            {
                 "name": "fallback_header",
                 "validator": self.validators.UnicodeString(strip=True),
                 "type": "string",
                 "description": "Request header to extract the user from when main one fails",
                 "default": "HTTP_X_FORWARDED_USER",
                 "formname": "Fallback header"
             },
+            {
                 "name": "clean_username",
                 "validator": self.validators.StringBoolean(if_missing=False),
                 "type": "bool",
                 "description": "Perform cleaning of user, if passed user has @ in username "
                                "then first part before @ is taken. "
                                "If there's \\ in the username only the part after \\ is taken",
                 "default": "True",
                 "formname": "Clean username"
             },
+        ]
         return settings
     def use_fake_password(self):
         return True
     def _clean_username(self, username):
         # Removing realm and domain from username
         username = username.partition('@')[0]
         username = username.rpartition('\\')[2]
         return username
     def _get_username(self, environ, settings):
         username = None
         environ = environ or {}
         if not environ:
             log.debug('got empty environ: %s', environ)
         settings = settings or {}
         if settings.get('header'):
             header = settings.get('header')
             username = environ.get(header)
             log.debug('extracted %s:%s', header, username)
         # fallback mode
         if not username and settings.get('fallback_header'):
             header = settings.get('fallback_header')
             username = environ.get(header)
             log.debug('extracted %s:%s', header, username)
         if username and str2bool(settings.get('clean_username')):
             log.debug('Received username %s from container', username)
             username = self._clean_username(username)
             log.debug('New cleanup user is: %s', username)
         return username
     def get_user(self, username=None, **kwargs):
         """
         Helper method for user fetching in plugins, by default it's using
         simple fetch by username, but this method can be customized in plugins
         eg. container auth plugin to fetch user by environ params
         :param username: username if given to fetch
         :param kwargs: extra arguments needed for user fetching.
         """
         environ = kwargs.get('environ') or {}
         settings = kwargs.get('settings') or {}
         username = self._get_username(environ, settings)
         # we got the username, so use default method now
         return super(KallitheaAuthPlugin, self).get_user(username)
     def auth(self, userobj, username, password, settings, **kwargs):
         """
         Gets the container_auth username (or email). It tries to get username
         from REMOTE_USER if this plugin is enabled, if that fails
         it tries to get username from HTTP_X_FORWARDED_USER if fallback header
         is set. clean_username extracts the username from this data if it's
         having @ in it.
         Return None on failure. On success, return a dictionary of the form:
             see: KallitheaAuthPluginBase.auth_func_attrs
         :param userobj:
         :param username:
         :param password:
         :param settings:
         :param kwargs:
         """
         environ = kwargs.get('environ')
         if not environ:
             log.debug('Empty environ data skipping...')
             return None
         if not userobj:
             userobj = self.get_user('', environ=environ, settings=settings)
         # we don't care passed username/password for container auth plugins.
         # only way to log in is using environ
         username = None
         if userobj:
-            username = safe_str(getattr(userobj, 'username'))
             username = getattr(userobj, 'username')
         if not username:
             # we don't have any objects in DB, user doesn't exist, extract
             # username from environ based on the settings
             username = self._get_username(environ, settings)
         # if cannot fetch username, it's a no-go for this plugin to proceed
         if not username:
             return None
         # old attrs fetched from Kallithea database
         admin = getattr(userobj, 'admin', False)
         email = environ.get(settings.get('email_header'), getattr(userobj, 'email', ''))
         firstname = environ.get(settings.get('firstname_header'), getattr(userobj, 'firstname', ''))
         lastname = environ.get(settings.get('lastname_header'), getattr(userobj, 'lastname', ''))
         user_data = {
             'username': username,
             'firstname': firstname or username,
             'lastname': lastname or '',
             'groups': [],
             'email': email or '',
             'admin': admin or False,
             'extern_name': username,
+        }
         log.info('user `%s` authenticated correctly', user_data['username'])
         return user_data
     def get_managed_fields(self):
         fields = ['username', 'password']
         if(Setting.get_by_name('auth_container_email_header').app_settings_value):
             fields.append('email')
         if(Setting.get_by_name('auth_container_firstname_header').app_settings_value):
             fields.append('firstname')
         if(Setting.get_by_name('auth_container_lastname_header').app_settings_value):
             fields.append('lastname')
         return fields

kallithea/lib/auth_modules/auth_ldap.py

➞

Show inline comments

 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
+#
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.lib.auth_modules.auth_ldap
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 Kallithea authentication plugin for LDAP
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Created on Nov 17, 2010
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 import logging
 from kallithea.lib import auth_modules
 from kallithea.lib.compat import hybrid_property
 from kallithea.lib.exceptions import LdapConnectionError, LdapImportError, LdapPasswordError, LdapUsernameError
 from kallithea.lib.utils2 import safe_str
 log = logging.getLogger(__name__)
 try:
     import ldap
     import ldap.filter
 except ImportError:
     # means that python-ldap is not installed
     ldap = None
 class AuthLdap(object):
     def __init__(self, server, base_dn, port=None, bind_dn='', bind_pass='',
                  tls_kind='LDAPS', tls_reqcert='DEMAND', cacertdir=None, ldap_version=3,
                  ldap_filter='(&(objectClass=user)(!(objectClass=computer)))',
                  search_scope='SUBTREE', attr_login='uid'):
         if ldap is None:
             raise LdapImportError
         self.ldap_version = ldap_version
         self.TLS_KIND = tls_kind
         OPT_X_TLS_DEMAND = 2
         self.TLS_REQCERT = getattr(ldap, 'OPT_X_TLS_%s' % tls_reqcert,
                                    OPT_X_TLS_DEMAND)
         self.cacertdir = cacertdir
         protocol = 'ldaps' if self.TLS_KIND == 'LDAPS' else 'ldap'
         if not port:
             port = 636 if self.TLS_KIND == 'LDAPS' else 389
         self.LDAP_SERVER = str(', '.join(
             "%s://%s:%s" % (protocol,
                             host.strip(),
                             port)
             for host in server.split(',')))
         self.LDAP_BIND_DN = safe_str(bind_dn)
         self.LDAP_BIND_PASS = safe_str(bind_pass)
         self.LDAP_BIND_DN = bind_dn
         self.LDAP_BIND_PASS = bind_pass
         self.BASE_DN = safe_str(base_dn)
         self.LDAP_FILTER = safe_str(ldap_filter)
         self.BASE_DN = base_dn
         self.LDAP_FILTER = ldap_filter
         self.SEARCH_SCOPE = getattr(ldap, 'SCOPE_%s' % search_scope)
         self.attr_login = attr_login
     def authenticate_ldap(self, username, password):
         """
         Authenticate a user via LDAP and return his/her LDAP properties.
         Raises AuthenticationError if the credentials are rejected, or
         EnvironmentError if the LDAP server can't be reached.
         :param username: username
         :param password: password
         """
         if not password:
             log.debug("Attempt to authenticate LDAP user "
                       "with blank password rejected.")
             raise LdapPasswordError()
         if "," in username:
             raise LdapUsernameError("invalid character in username: ,")
         try:
             if self.cacertdir:
                 if hasattr(ldap, 'OPT_X_TLS_CACERTDIR'):
                     ldap.set_option(ldap.OPT_X_TLS_CACERTDIR, self.cacertdir)
                 else:
                     log.debug("OPT_X_TLS_CACERTDIR is not available - can't set %s", self.cacertdir)
             ldap.set_option(ldap.OPT_REFERRALS, ldap.OPT_OFF)
             ldap.set_option(ldap.OPT_RESTART, ldap.OPT_ON)
             ldap.set_option(ldap.OPT_TIMEOUT, 20)
             ldap.set_option(ldap.OPT_NETWORK_TIMEOUT, 10)
             ldap.set_option(ldap.OPT_TIMELIMIT, 15)
             if self.TLS_KIND != 'PLAIN':
                 ldap.set_option(ldap.OPT_X_TLS_REQUIRE_CERT, self.TLS_REQCERT)
             server = ldap.initialize(self.LDAP_SERVER)
             if self.ldap_version == 2:
                 server.protocol = ldap.VERSION2
             else:
                 server.protocol = ldap.VERSION3
             if self.TLS_KIND == 'START_TLS':
                 server.start_tls_s()
             if self.LDAP_BIND_DN and self.LDAP_BIND_PASS:
                 log.debug('Trying simple_bind with password and given DN: %s',
                           self.LDAP_BIND_DN)
                 server.simple_bind_s(self.LDAP_BIND_DN, self.LDAP_BIND_PASS)
             filter_ = '(&%s(%s=%s))' % (self.LDAP_FILTER,
                                         ldap.filter.escape_filter_chars(self.attr_login),
                                         ldap.filter.escape_filter_chars(username))
             log.debug("Authenticating %r filter %s at %s", self.BASE_DN,
                       filter_, self.LDAP_SERVER)
             lobjects = server.search_ext_s(self.BASE_DN, self.SEARCH_SCOPE,
                                            filter_)
             if not lobjects:
                 raise ldap.NO_SUCH_OBJECT()
             for (dn, _attrs) in lobjects:
                 if dn is None:
                     continue
                 try:
                     log.debug('Trying simple bind with %s', dn)
-                    server.simple_bind_s(dn, safe_str(password))
                     server.simple_bind_s(dn, password)
                     results = server.search_ext_s(dn, ldap.SCOPE_BASE,
                                                   '(objectClass=*)')
                     if len(results) == 1:
                         dn_, attrs = results[0]
                         assert dn_ == dn
                         return dn, attrs
                 except ldap.INVALID_CREDENTIALS:
                     log.debug("LDAP rejected password for user '%s': %s",
                               username, dn)
                     continue # accept authentication as another ldap user with same username
             log.debug("No matching LDAP objects for authentication "
                       "of '%s'", username)
             raise LdapPasswordError()
         except ldap.NO_SUCH_OBJECT:
             log.debug("LDAP says no such user '%s'", username)
             raise LdapUsernameError()
         except ldap.SERVER_DOWN:
             # [0] might be {'info': "TLS error -8179:Peer's Certificate issuer is not recognized.", 'desc': "Can't contact LDAP server"}
             raise LdapConnectionError("LDAP can't connect to authentication server")
 class KallitheaAuthPlugin(auth_modules.KallitheaExternalAuthPlugin):
     def __init__(self):
         self._logger = logging.getLogger(__name__)
         self._tls_kind_values = ["PLAIN", "LDAPS", "START_TLS"]
         self._tls_reqcert_values = ["NEVER", "ALLOW", "TRY", "DEMAND", "HARD"]
         self._search_scopes = ["BASE", "ONELEVEL", "SUBTREE"]
     @hybrid_property
     def name(self):
         return "ldap"
     def settings(self):
         settings = [
+            {
                 "name": "host",
                 "validator": self.validators.UnicodeString(strip=True),
                 "type": "string",
                 "description": "Host of the LDAP Server",
                 "formname": "LDAP Host"
             },
+            {
                 "name": "port",
                 "validator": self.validators.Number(strip=True),
                 "type": "string",
                 "description": "Port that the LDAP server is listening on. Defaults to 389 for PLAIN/START_TLS and 636 for LDAPS.",
                 "default": "",
                 "formname": "Custom LDAP Port"
             },
+            {
                 "name": "dn_user",
                 "validator": self.validators.UnicodeString(strip=True),
                 "type": "string",
                 "description": "User to connect to LDAP",
                 "formname": "Account"
             },
+            {
                 "name": "dn_pass",
                 "validator": self.validators.UnicodeString(strip=True),
                 "type": "password",
                 "description": "Password to connect to LDAP",
                 "formname": "Password"
             },
+            {
                 "name": "tls_kind",
                 "validator": self.validators.OneOf(self._tls_kind_values),
                 "type": "select",
                 "values": self._tls_kind_values,
                 "description": "TLS Type",
                 "default": 'LDAPS',
                 "formname": "Connection Security"
             },
+            {
                 "name": "tls_reqcert",
                 "validator": self.validators.OneOf(self._tls_reqcert_values),
                 "type": "select",
                 "values": self._tls_reqcert_values,
                 "description": "Require Cert over TLS?",
                 "formname": "Certificate Checks"
             },
+            {
                 "name": "cacertdir",
                 "validator": self.validators.UnicodeString(strip=True),
                 "type": "string",
                 "description": "Optional: Custom CA certificate directory for validating LDAPS",
                 "formname": "Custom CA Certificates"
             },
+            {
                 "name": "base_dn",
                 "validator": self.validators.UnicodeString(strip=True),
                 "type": "string",
                 "description": "Base DN to search (e.g., dc=mydomain,dc=com)",
                 "formname": "Base DN"
             },
+            {
                 "name": "filter",
                 "validator": self.validators.UnicodeString(strip=True),
                 "type": "string",
                 "description": "Filter to narrow results (e.g., ou=Users, etc)",
                 "formname": "LDAP Search Filter"
             },
+            {
                 "name": "search_scope",
                 "validator": self.validators.OneOf(self._search_scopes),
                 "type": "select",
                 "values": self._search_scopes,
                 "description": "How deep to search LDAP",
                 "formname": "LDAP Search Scope"
             },
+            {
                 "name": "attr_login",
                 "validator": self.validators.AttrLoginValidator(not_empty=True, strip=True),
                 "type": "string",
                 "description": "LDAP Attribute to map to user name",
                 "formname": "Login Attribute"
             },
+            {
                 "name": "attr_firstname",
                 "validator": self.validators.UnicodeString(strip=True),
                 "type": "string",
                 "description": "LDAP Attribute to map to first name",
                 "formname": "First Name Attribute"
             },
+            {
                 "name": "attr_lastname",
                 "validator": self.validators.UnicodeString(strip=True),
                 "type": "string",
                 "description": "LDAP Attribute to map to last name",
                 "formname": "Last Name Attribute"
             },
+            {
                 "name": "attr_email",
                 "validator": self.validators.UnicodeString(strip=True),
                 "type": "string",
                 "description": "LDAP Attribute to map to email address",
                 "formname": "Email Attribute"
+            }
+        ]
         return settings
     def use_fake_password(self):
         return True
     def auth(self, userobj, username, password, settings, **kwargs):
         """
         Given a user object (which may be null), username, a plaintext password,
         and a settings object (containing all the keys needed as listed in settings()),
         authenticate this user's login attempt.
         Return None on failure. On success, return a dictionary of the form:
             see: KallitheaAuthPluginBase.auth_func_attrs
         This is later validated for correctness
         """
         if not username or not password:
             log.debug('Empty username or password skipping...')
             return None
         kwargs = {
             'server': settings.get('host', ''),
             'base_dn': settings.get('base_dn', ''),
             'port': settings.get('port'),
             'bind_dn': settings.get('dn_user'),
             'bind_pass': settings.get('dn_pass'),
             'tls_kind': settings.get('tls_kind'),
             'tls_reqcert': settings.get('tls_reqcert'),
             'cacertdir': settings.get('cacertdir'),
             'ldap_filter': settings.get('filter'),
             'search_scope': settings.get('search_scope'),
             'attr_login': settings.get('attr_login'),
             'ldap_version': 3,
+        }
         if kwargs['bind_dn'] and not kwargs['bind_pass']:
             log.debug('Using dynamic binding.')
             kwargs['bind_dn'] = kwargs['bind_dn'].replace('$login', username)
             kwargs['bind_pass'] = password
         log.debug('Checking for ldap authentication')
         try:
             aldap = AuthLdap(**kwargs)
             (user_dn, ldap_attrs) = aldap.authenticate_ldap(username, password)
             log.debug('Got ldap DN response %s', user_dn)
             get_ldap_attr = lambda k: ldap_attrs.get(settings.get(k), [''])[0]
             # old attrs fetched from Kallithea database
             admin = getattr(userobj, 'admin', False)

kallithea/lib/base.py

➞

Show inline comments

 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
+#
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.lib.base
 ~~~~~~~~~~~~~~~~~~
 The base Controller API
 Provides the BaseController class for subclassing. And usage in different
 controllers
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Oct 06, 2010
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 import base64
 import datetime
 import logging
 import traceback
 import warnings
 import decorator
 import paste.auth.basic
 import paste.httpexceptions
 import paste.httpheaders
 import webob.exc
 from tg import TGController, config, render_template, request, response, session
 from tg import tmpl_context as c
 from tg.i18n import ugettext as _
 from kallithea import BACKENDS, __version__
 from kallithea.config.routing import url
 from kallithea.lib import auth_modules, ext_json
 from kallithea.lib.auth import AuthUser, HasPermissionAnyMiddleware
 from kallithea.lib.exceptions import UserCreationError
 from kallithea.lib.utils import get_repo_slug, is_valid_repo
-from kallithea.lib.utils2 import AttributeDict, ascii_bytes, safe_int, safe_str, safe_unicode, set_hook_environment, str2bool
 from kallithea.lib.utils2 import AttributeDict, ascii_bytes, safe_int, safe_unicode, set_hook_environment, str2bool
 from kallithea.lib.vcs.exceptions import ChangesetDoesNotExistError, EmptyRepositoryError, RepositoryError
 from kallithea.model import meta
 from kallithea.model.db import PullRequest, Repository, Setting, User
 from kallithea.model.scm import ScmModel
 log = logging.getLogger(__name__)
 def render(template_path):
     return render_template({'url': url}, 'mako', template_path)
 def _filter_proxy(ip):
     """
     HEADERS can have multiple ips inside the left-most being the original
     client, and each successive proxy that passed the request adding the IP
     address where it received the request from.
     :param ip:
     """
     if ',' in ip:
         _ips = ip.split(',')
         _first_ip = _ips[0].strip()
         log.debug('Got multiple IPs %s, using %s', ','.join(_ips), _first_ip)
         return _first_ip
     return ip
 def _get_ip_addr(environ):
     proxy_key = 'HTTP_X_REAL_IP'
     proxy_key2 = 'HTTP_X_FORWARDED_FOR'
     def_key = 'REMOTE_ADDR'
     ip = environ.get(proxy_key)
     if ip:
         return _filter_proxy(ip)
     ip = environ.get(proxy_key2)
     if ip:
         return _filter_proxy(ip)
     ip = environ.get(def_key, '0.0.0.0')
     return _filter_proxy(ip)
 def get_path_info(environ):
     """Return unicode PATH_INFO from environ ... using tg.original_request if available.
     """
     org_req = environ.get('tg.original_request')
     if org_req is not None:
         environ = org_req.environ
     return safe_unicode(environ['PATH_INFO'])
 def log_in_user(user, remember, is_external_auth, ip_addr):
     """
     Log a `User` in and update session and cookies. If `remember` is True,
     the session cookie is set to expire in a year; otherwise, it expires at
     the end of the browser session.
     Returns populated `AuthUser` object.
     """
     # It should not be possible to explicitly log in as the default user.
     assert not user.is_default_user, user
     auth_user = AuthUser.make(dbuser=user, is_external_auth=is_external_auth, ip_addr=ip_addr)
     if auth_user is None:
         return None
     user.update_lastlogin()
     meta.Session().commit()
     # Start new session to prevent session fixation attacks.
     session.invalidate()
     session['authuser'] = cookie = auth_user.to_cookie()
     # If they want to be remembered, update the cookie.
     # NOTE: Assumes that beaker defaults to browser session cookie.
     if remember:
         t = datetime.datetime.now() + datetime.timedelta(days=365)
         session._set_cookie_expires(t)
     session.save()
     log.info('user %s is now authenticated and stored in '
              'session, session attrs %s', user.username, cookie)
     # dumps session attrs back to cookie
     session._update_cookie_out()
     return auth_user
 class BasicAuth(paste.auth.basic.AuthBasicAuthenticator):
     def __init__(self, realm, authfunc, auth_http_code=None):
         self.realm = realm
         self.authfunc = authfunc
         self._rc_auth_http_code = auth_http_code
     def build_authentication(self, environ):
         head = paste.httpheaders.WWW_AUTHENTICATE.tuples('Basic realm="%s"' % self.realm)
         # Consume the whole body before sending a response
         try:
             request_body_size = int(environ.get('CONTENT_LENGTH', 0))
         except (ValueError):
             request_body_size = 0
         environ['wsgi.input'].read(request_body_size)
         if self._rc_auth_http_code and self._rc_auth_http_code == '403':
             # return 403 if alternative http return code is specified in
             # Kallithea config
             return paste.httpexceptions.HTTPForbidden(headers=head)
         return paste.httpexceptions.HTTPUnauthorized(headers=head)
     def authenticate(self, environ):
         authorization = paste.httpheaders.AUTHORIZATION(environ)
         if not authorization:
             return self.build_authentication(environ)
         (authmeth, auth) = authorization.split(' ', 1)
         if 'basic' != authmeth.lower():
             return self.build_authentication(environ)
         auth = base64.b64decode(auth.strip())
         _parts = auth.split(':', 1)
         if len(_parts) == 2:
             username, password = _parts
             if self.authfunc(username, password, environ) is not None:
                 return username
         return self.build_authentication(environ)
     __call__ = authenticate
 class BaseVCSController(object):
     """Base controller for handling Mercurial/Git protocol requests
     (coming from a VCS client, and not a browser).
     """
     scm_alias = None # 'hg' / 'git'
     def __init__(self, application, config):
         self.application = application
         self.config = config
         # base path of repo locations
         self.basepath = self.config['base_path']
         # authenticate this VCS request using the authentication modules
         self.authenticate = BasicAuth('', auth_modules.authenticate,
                                       config.get('auth_ret_code'))
     @classmethod
     def parse_request(cls, environ):
         """If request is parsed as a request for this VCS, return a namespace with the parsed request.
         If the request is unknown, return None.
         """
         raise NotImplementedError()
     def _authorize(self, environ, action, repo_name, ip_addr):
         """Authenticate and authorize user.
         Since we're dealing with a VCS client and not a browser, we only
         support HTTP basic authentication, either directly via raw header
         inspection, or by using container authentication to delegate the
         authentication to the web server.
         Returns (user, None) on successful authentication and authorization.
         Returns (None, wsgi_app) to send the wsgi_app response to the client.
         """
         # Use anonymous access if allowed for action on repo.
         default_user = User.get_default_user(cache=True)
         default_authuser = AuthUser.make(dbuser=default_user, ip_addr=ip_addr)
         if default_authuser is None:
             log.debug('No anonymous access at all') # move on to proper user auth
         else:
             if self._check_permission(action, default_authuser, repo_name):
                 return default_authuser, None
             log.debug('Not authorized to access this repository as anonymous user')
         username = None
         #==============================================================
         # DEFAULT PERM FAILED OR ANONYMOUS ACCESS IS DISABLED SO WE
         # NEED TO AUTHENTICATE AND ASK FOR AUTH USER PERMISSIONS
         #==============================================================
         # try to auth based on environ, container auth methods
         log.debug('Running PRE-AUTH for container based authentication')
         pre_auth = auth_modules.authenticate('', '', environ)
         if pre_auth is not None and pre_auth.get('username'):
             username = pre_auth['username']
         log.debug('PRE-AUTH got %s as username', username)
         # If not authenticated by the container, running basic auth
         if not username:
-            self.authenticate.realm = safe_str(self.config['realm'])
             self.authenticate.realm = self.config['realm']
             result = self.authenticate(environ)
             if isinstance(result, str):
                 paste.httpheaders.AUTH_TYPE.update(environ, 'basic')
                 paste.httpheaders.REMOTE_USER.update(environ, result)
                 username = result
             else:
                 return None, result.wsgi_application
         #==============================================================
         # CHECK PERMISSIONS FOR THIS REQUEST USING GIVEN USERNAME
         #==============================================================
         try:
             user = User.get_by_username_or_email(username)
         except Exception:
             log.error(traceback.format_exc())
             return None, webob.exc.HTTPInternalServerError()
         authuser = AuthUser.make(dbuser=user, ip_addr=ip_addr)
         if authuser is None:
             return None, webob.exc.HTTPForbidden()
         if not self._check_permission(action, authuser, repo_name):
             return None, webob.exc.HTTPForbidden()
         return user, None
     def _handle_request(self, environ, start_response):
         raise NotImplementedError()
     def _check_permission(self, action, authuser, repo_name):
         """
         Checks permissions using action (push/pull) user and repository
         name
         :param action: 'push' or 'pull' action
         :param user: `User` instance
         :param repo_name: repository name
         """
         if action == 'push':
             if not HasPermissionAnyMiddleware('repository.write',
                                               'repository.admin')(authuser,
                                                                   repo_name):
                 return False
         else:
             #any other action need at least read permission
             if not HasPermissionAnyMiddleware('repository.read',
                                               'repository.write',
                                               'repository.admin')(authuser,
                                                                   repo_name):
                 return False
         return True
     def _get_ip_addr(self, environ):
         return _get_ip_addr(environ)
     def __call__(self, environ, start_response):
         try:
             # try parsing a request for this VCS - if it fails, call the wrapped app
             parsed_request = self.parse_request(environ)
             if parsed_request is None:
                 return self.application(environ, start_response)
             # skip passing error to error controller
             environ['pylons.status_code_redirect'] = True
             # quick check if repo exists...
             if not is_valid_repo(parsed_request.repo_name, self.basepath, self.scm_alias):
                 raise webob.exc.HTTPNotFound()
             if parsed_request.action is None:
                 # Note: the client doesn't get the helpful error message
                 raise webob.exc.HTTPBadRequest('Unable to detect pull/push action for %r! Are you using a nonstandard command or client?' % parsed_request.repo_name)
             #======================================================================
             # CHECK PERMISSIONS
             #======================================================================
             ip_addr = self._get_ip_addr(environ)
             user, response_app = self._authorize(environ, parsed_request.action, parsed_request.repo_name, ip_addr)
             if response_app is not None:
                 return response_app(environ, start_response)
             #======================================================================
             # REQUEST HANDLING
             #======================================================================
             set_hook_environment(user.username, ip_addr,
                 parsed_request.repo_name, self.scm_alias, parsed_request.action)
             try:
                 log.info('%s action on %s repo "%s" by "%s" from %s',
-                         parsed_request.action, self.scm_alias, parsed_request.repo_name, safe_str(user.username), ip_addr)
                          parsed_request.action, self.scm_alias, parsed_request.repo_name, user.username, ip_addr)
                 app = self._make_app(parsed_request)
                 return app(environ, start_response)
             except Exception:
                 log.error(traceback.format_exc())
                 raise webob.exc.HTTPInternalServerError()
         except webob.exc.HTTPException as e:
             return e(environ, start_response)
 class BaseController(TGController):
     def _before(self, *args, **kwargs):
         """
         _before is called before controller methods and after __call__
         """
         if request.needs_csrf_check:
             # CSRF protection: Whenever a request has ambient authority (whether
             # through a session cookie or its origin IP address), it must include
             # the correct token, unless the HTTP method is GET or HEAD (and thus
             # guaranteed to be side effect free. In practice, the only situation
             # where we allow side effects without ambient authority is when the
             # authority comes from an API key; and that is handled above.
             from kallithea.lib import helpers as h
             token = request.POST.get(h.session_csrf_secret_name)
             if not token or token != h.session_csrf_secret_token():
                 log.error('CSRF check failed')
                 raise webob.exc.HTTPForbidden()
         c.kallithea_version = __version__
         rc_config = Setting.get_app_settings()
         # Visual options
         c.visual = AttributeDict({})
         ## DB stored
         c.visual.show_public_icon = str2bool(rc_config.get('show_public_icon'))
         c.visual.show_private_icon = str2bool(rc_config.get('show_private_icon'))
         c.visual.stylify_metalabels = str2bool(rc_config.get('stylify_metalabels'))
         c.visual.page_size = safe_int(rc_config.get('dashboard_items', 100))
         c.visual.admin_grid_items = safe_int(rc_config.get('admin_grid_items', 100))
         c.visual.repository_fields = str2bool(rc_config.get('repository_fields'))
         c.visual.show_version = str2bool(rc_config.get('show_version'))
         c.visual.use_gravatar = str2bool(rc_config.get('use_gravatar'))
         c.visual.gravatar_url = rc_config.get('gravatar_url')
         c.ga_code = rc_config.get('ga_code')
         # TODO: replace undocumented backwards compatibility hack with db upgrade and rename ga_code
         if c.ga_code and '<' not in c.ga_code:
             c.ga_code = '''<script type="text/javascript">
                 var _gaq = _gaq || [];
                 _gaq.push(['_setAccount', '%s']);
                 _gaq.push(['_trackPageview']);
                 (function() {
                     var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true;
                     ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
                     var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s);
                     })();
             </script>''' % c.ga_code
         c.site_name = rc_config.get('title')
         c.clone_uri_tmpl = rc_config.get('clone_uri_tmpl') or Repository.DEFAULT_CLONE_URI
         c.clone_ssh_tmpl = rc_config.get('clone_ssh_tmpl') or Repository.DEFAULT_CLONE_SSH
         ## INI stored
         c.visual.allow_repo_location_change = str2bool(config.get('allow_repo_location_change', True))
         c.visual.allow_custom_hooks_settings = str2bool(config.get('allow_custom_hooks_settings', True))
         c.ssh_enabled = str2bool(config.get('ssh_enabled', False))
         c.instance_id = config.get('instance_id')
         c.issues_url = config.get('bugtracker', url('issues_url'))
         # END CONFIG VARS
         c.repo_name = get_repo_slug(request)  # can be empty
         c.backends = list(BACKENDS)
         self.cut_off_limit = safe_int(config.get('cut_off_limit'))
         c.my_pr_count = PullRequest.query(reviewer_id=request.authuser.user_id, include_closed=False).count()
         self.scm_model = ScmModel()
     @staticmethod
     def _determine_auth_user(session_authuser, ip_addr):
         """
         Create an `AuthUser` object given the API key/bearer token
         (if any) and the value of the authuser session cookie.
         Returns None if no valid user is found (like not active or no access for IP).
         """
         # Authenticate by session cookie
         # In ancient login sessions, 'authuser' may not be a dict.
         # In that case, the user will have to log in again.
         # v0.3 and earlier included an 'is_authenticated' key; if present,
         # this must be True.
         if isinstance(session_authuser, dict) and session_authuser.get('is_authenticated', True):
             return AuthUser.from_cookie(session_authuser, ip_addr=ip_addr)
         # Authenticate by auth_container plugin (if enabled)
         if any(
             plugin.is_container_auth
             for plugin in auth_modules.get_auth_plugins()
         ):
             try:
                 user_info = auth_modules.authenticate('', '', request.environ)
             except UserCreationError as e:
                 from kallithea.lib import helpers as h
                 h.flash(e, 'error', logf=log.error)
             else:
                 if user_info is not None:
                     username = user_info['username']
                     user = User.get_by_username(username, case_insensitive=True)
                     return log_in_user(user, remember=False, is_external_auth=True, ip_addr=ip_addr)
         # User is default user (if active) or anonymous
         default_user = User.get_default_user(cache=True)
         authuser = AuthUser.make(dbuser=default_user, ip_addr=ip_addr)
         if authuser is None: # fall back to anonymous
             authuser = AuthUser(dbuser=default_user) # TODO: somehow use .make?
         return authuser
     @staticmethod
     def _basic_security_checks():
         """Perform basic security/sanity checks before processing the request."""
         # Only allow the following HTTP request methods.
         if request.method not in ['GET', 'HEAD', 'POST']:
             raise webob.exc.HTTPMethodNotAllowed()
         # Also verify the _method override - no longer allowed.
         if request.params.get('_method') is None:
             pass # no override, no problem
         else:
             raise webob.exc.HTTPMethodNotAllowed()
         # Make sure CSRF token never appears in the URL. If so, invalidate it.
         from kallithea.lib import helpers as h
         if h.session_csrf_secret_name in request.GET:
             log.error('CSRF key leak detected')
             session.pop(h.session_csrf_secret_name, None)
             session.save()
             h.flash(_('CSRF token leak has been detected - all form tokens have been expired'),
                     category='error')
         # WebOb already ignores request payload parameters for anything other
         # than POST/PUT, but double-check since other Kallithea code relies on
         # this assumption.
         if request.method not in ['POST', 'PUT'] and request.POST:
             log.error('%r request with payload parameters; WebOb should have stopped this', request.method)
             raise webob.exc.HTTPBadRequest()
     def __call__(self, environ, context):
         try:
             ip_addr = _get_ip_addr(environ)
             self._basic_security_checks()
             api_key = request.GET.get('api_key')
             try:
                 # Request.authorization may raise ValueError on invalid input
                 type, params = request.authorization
             except (ValueError, TypeError):
                 pass
             else:
                 if type.lower() == 'bearer':
                     api_key = params # bearer token is an api key too
             if api_key is None:
                 authuser = self._determine_auth_user(
                     session.get('authuser'),
                     ip_addr=ip_addr,
+                )
                 needs_csrf_check = request.method not in ['GET', 'HEAD']
             else:
                 dbuser = User.get_by_api_key(api_key)
                 if dbuser is None:
                     log.info('No db user found for authentication with API key ****%s from %s',
                              api_key[-4:], ip_addr)
                 authuser = AuthUser.make(dbuser=dbuser, is_external_auth=True, ip_addr=ip_addr)
                 needs_csrf_check = False # API key provides CSRF protection
             if authuser is None:
                 log.info('No valid user found')
                 raise webob.exc.HTTPForbidden()
             # set globals for auth user
             request.authuser = authuser
             request.ip_addr = ip_addr
             request.needs_csrf_check = needs_csrf_check
             log.info('IP: %s User: %s accessed %s',
                 request.ip_addr, request.authuser,

kallithea/lib/caching_query.py

➞

Show inline comments

 # apparently based on https://github.com/sqlalchemy/sqlalchemy/blob/rel_0_7/examples/beaker_caching/caching_query.py
 """caching_query.py
 Represent persistence structures which allow the usage of
 Beaker caching with SQLAlchemy.
 The three new concepts introduced here are:
  * CachingQuery - a Query subclass that caches and
    retrieves results in/from Beaker.
  * FromCache - a query option that establishes caching
    parameters on a Query
  * _params_from_query - extracts value parameters from
    a Query.
 The rest of what's here are standard SQLAlchemy and
 Beaker constructs.
 """
 import beaker
 from beaker.exceptions import BeakerException
 from sqlalchemy.orm.interfaces import MapperOption
 from sqlalchemy.orm.query import Query
 from sqlalchemy.sql import visitors
 from kallithea.lib.utils2 import safe_str
 class CachingQuery(Query):
     """A Query subclass which optionally loads full results from a Beaker
     cache region.
     The CachingQuery stores additional state that allows it to consult
     a Beaker cache before accessing the database:
     * A "region", which is a cache region argument passed to a
       Beaker CacheManager, specifies a particular cache configuration
       (including backend implementation, expiration times, etc.)
     * A "namespace", which is a qualifying name that identifies a
       group of keys within the cache.  A query that filters on a name
       might use the name "by_name", a query that filters on a date range
       to a joined table might use the name "related_date_range".
     When the above state is present, a Beaker cache is retrieved.
     The "namespace" name is first concatenated with
     a string composed of the individual entities and columns the Query
     requests, i.e. such as ``Query(User.id, User.name)``.
     The Beaker cache is then loaded from the cache manager based
     on the region and composed namespace.  The key within the cache
     itself is then constructed against the bind parameters specified
     by this query, which are usually literals defined in the
     WHERE clause.
     The FromCache mapper option below represent
     the "public" method of configuring this state upon the CachingQuery.
     """
     def __init__(self, manager, *args, **kw):
         self.cache_manager = manager
         Query.__init__(self, *args, **kw)
     def __iter__(self):
         """override __iter__ to pull results from Beaker
            if particular attributes have been configured.
            Note that this approach does *not* detach the loaded objects from
            the current session. If the cache backend is an in-process cache
            (like "memory") and lives beyond the scope of the current session's
            transaction, those objects may be expired. The method here can be
            modified to first expunge() each loaded item from the current
            session before returning the list of items, so that the items
            in the cache are not the same ones in the current Session.
         """
         if hasattr(self, '_cache_parameters'):
             return self.get_value(createfunc=lambda:
                                   list(Query.__iter__(self)))
         else:
             return Query.__iter__(self)
     def invalidate(self):
         """Invalidate the value represented by this Query."""
         cache, cache_key = _get_cache_parameters(self)
         cache.remove(cache_key)
     def get_value(self, merge=True, createfunc=None):
         """Return the value from the cache for this query.
         Raise KeyError if no value present and no
         createfunc specified.
         """
         cache, cache_key = _get_cache_parameters(self)
         ret = cache.get_value(cache_key, createfunc=createfunc)
         if merge:
             ret = self.merge_result(ret, load=False)
         return ret
     def set_value(self, value):
         """Set the value in the cache for this query."""
         cache, cache_key = _get_cache_parameters(self)
         cache.put(cache_key, value)
 def query_callable(manager, query_cls=CachingQuery):
     def query(*arg, **kw):
         return query_cls(manager, *arg, **kw)
     return query
 def get_cache_region(name, region):
     if region not in beaker.cache.cache_regions:
         raise BeakerException('Cache region `%s` not configured '
             'Check if proper cache settings are in the .ini files' % region)
     kw = beaker.cache.cache_regions[region]
     return beaker.cache.Cache._get_cache(name, kw)
 def _get_cache_parameters(query):
     """For a query with cache_region and cache_namespace configured,
     return the corresponding Cache instance and cache key, based
     on this query's current criterion and parameter values.
     """
     if not hasattr(query, '_cache_parameters'):
         raise ValueError("This Query does not have caching "
                          "parameters configured.")
     region, namespace, cache_key = query._cache_parameters
     namespace = _namespace_from_query(namespace, query)
     if cache_key is None:
         # cache key - the value arguments from this query's parameters.
         args = _params_from_query(query)
         args.append(query._limit)
         args.append(query._offset)
         cache_key = " ".join(str(x) for x in args)
     if cache_key is None:
         raise Exception('Cache key cannot be None')
     # get cache
     #cache = query.cache_manager.get_cache_region(namespace, region)
     cache = get_cache_region(namespace, region)
     # optional - hash the cache_key too for consistent length
     # import uuid
     # cache_key= str(uuid.uuid5(uuid.NAMESPACE_DNS, cache_key))
     return cache, cache_key
 def _namespace_from_query(namespace, query):
     # cache namespace - the token handed in by the
     # option + class we're querying against
     namespace = " ".join([namespace] + [str(x) for x in query._entities])
     # memcached wants this
     namespace = namespace.replace(' ', '_')
     return namespace
 def _set_cache_parameters(query, region, namespace, cache_key):
     if hasattr(query, '_cache_parameters'):
         region, namespace, cache_key = query._cache_parameters
         raise ValueError("This query is already configured "
                         "for region %r namespace %r" %
                         (region, namespace)
+                    )
-    query._cache_parameters = region, safe_str(namespace), cache_key
     query._cache_parameters = region, namespace, cache_key
 class FromCache(MapperOption):
     """Specifies that a Query should load results from a cache."""
     propagate_to_loaders = False
     def __init__(self, region, namespace, cache_key=None):
         """Construct a new FromCache.
         :param region: the cache region.  Should be a
         region configured in the Beaker CacheManager.
         :param namespace: the cache namespace.  Should
         be a name uniquely describing the target Query's
         lexical structure.
         :param cache_key: optional.  A string cache key
         that will serve as the key to the query.   Use this
         if your query has a huge amount of parameters (such
         as when using in_()) which correspond more simply to
         some other identifier.
         """
         self.region = region
         self.namespace = namespace
         self.cache_key = cache_key
     def process_query(self, query):
         """Process a Query during normal loading operation."""
         _set_cache_parameters(query, self.region, self.namespace,
                               self.cache_key)
 def _params_from_query(query):
     """Pull the bind parameter values from a query.
     This takes into account any scalar attribute bindparam set up.
     E.g. params_from_query(query.filter(Cls.foo==5).filter(Cls.bar==7)))
     would return [5, 7].
     """
     v = []
     def visit_bindparam(bind):
         if bind.key in query._params:
             value = query._params[bind.key]
         elif bind.callable:
             # lazyloader may dig a callable in here, intended
             # to late-evaluate params after autoflush is called.
             # convert to a scalar value.
             value = bind.callable()
         else:
             value = bind.value
         v.append(value)
     if query._criterion is not None:
         visitors.traverse(query._criterion, {}, {'bindparam': visit_bindparam})
     for f in query._from_obj:
         visitors.traverse(f, {}, {'bindparam': visit_bindparam})
     return v

kallithea/lib/hooks.py

➞

Show inline comments

 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
+#
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.lib.hooks
 ~~~~~~~~~~~~~~~~~~~
 Hooks run by Kallithea
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Aug 6, 2010
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 import os
 import sys
 import time
 import mercurial.scmutil
 from kallithea.lib import helpers as h
 from kallithea.lib.exceptions import UserCreationError
 from kallithea.lib.utils import action_logger, make_ui
-from kallithea.lib.utils2 import HookEnvironmentError, ascii_str, get_hook_environment, safe_bytes, safe_str
 from kallithea.lib.utils2 import HookEnvironmentError, ascii_str, get_hook_environment, safe_bytes
 from kallithea.lib.vcs.backends.base import EmptyChangeset
 from kallithea.model.db import Repository, User
 def _get_scm_size(alias, root_path):
     if not alias.startswith('.'):
         alias += '.'
     size_scm, size_root = 0, 0
-    for path, dirs, files in os.walk(safe_str(root_path)):
     for path, dirs, files in os.walk(root_path):
         if path.find(alias) != -1:
             for f in files:
                 try:
                     size_scm += os.path.getsize(os.path.join(path, f))
                 except OSError:
                     pass
         else:
             for f in files:
                 try:
                     size_root += os.path.getsize(os.path.join(path, f))
                 except OSError:
                     pass
     size_scm_f = h.format_byte_size(size_scm)
     size_root_f = h.format_byte_size(size_root)
     size_total_f = h.format_byte_size(size_root + size_scm)
     return size_scm_f, size_root_f, size_total_f
 def repo_size(ui, repo, hooktype=None, **kwargs):
     """Show size of Mercurial repository, to be called after push."""
     size_hg_f, size_root_f, size_total_f = _get_scm_size('.hg', repo.root)
     last_cs = repo[len(repo) - 1]
     msg = ('Repository size .hg: %s Checkout: %s Total: %s\n'
            'Last revision is now r%s:%s\n') % (
         size_hg_f, size_root_f, size_total_f, last_cs.rev(), ascii_str(last_cs.hex())[:12]
+    )
     ui.status(safe_bytes(msg))
 def log_pull_action(ui, repo, **kwargs):
     """Logs user last pull action
     Called as Mercurial hook outgoing.pull_logger or from Kallithea before invoking Git.
     Does *not* use the action from the hook environment but is always 'pull'.
     """
     ex = get_hook_environment()
     user = User.get_by_username(ex.username)
     action = 'pull'
     action_logger(user, action, ex.repository, ex.ip, commit=True)
     # extension hook call
     from kallithea import EXTENSIONS
     callback = getattr(EXTENSIONS, 'PULL_HOOK', None)
     if callable(callback):
         kw = {}
         kw.update(ex)
         callback(**kw)
     return 0
 def log_push_action(ui, repo, node, node_last, **kwargs):
     """
     Entry point for Mercurial hook changegroup.push_logger.
     The pushed changesets is given by the revset 'node:node_last'.
     Note: This hook is not only logging, but also the side effect invalidating
     cahes! The function should perhaps be renamed.
     """
     revs = [ascii_str(repo[r].hex()) for r in mercurial.scmutil.revrange(repo, [b'%s:%s' % (node, node_last)])]
     process_pushed_raw_ids(revs)
     return 0
 def process_pushed_raw_ids(revs):
     """
     Register that changes have been added to the repo - log the action *and* invalidate caches.
     Called from  Mercurial changegroup.push_logger calling hook log_push_action,
     or from the Git post-receive hook calling handle_git_post_receive ...
     or from scm _handle_push.
     """
     ex = get_hook_environment()
     action = '%s:%s' % (ex.action, ','.join(revs))
     action_logger(ex.username, action, ex.repository, ex.ip, commit=True)
     from kallithea.model.scm import ScmModel
     ScmModel().mark_for_invalidation(ex.repository)
     # extension hook call
     from kallithea import EXTENSIONS
     callback = getattr(EXTENSIONS, 'PUSH_HOOK', None)
     if callable(callback):
         kw = {'pushed_revs': revs}
         kw.update(ex)
         callback(**kw)
 def log_create_repository(repository_dict, created_by, **kwargs):
     """
     Post create repository Hook.
     :param repository: dict dump of repository object
     :param created_by: username who created repository
     available keys of repository_dict:
      'repo_type',
      'description',
      'private',
      'created_on',
      'enable_downloads',
      'repo_id',
      'owner_id',
      'enable_statistics',
      'clone_uri',
      'fork_id',
      'group_id',
      'repo_name'
     """
     from kallithea import EXTENSIONS
     callback = getattr(EXTENSIONS, 'CREATE_REPO_HOOK', None)
     if callable(callback):
         kw = {}
         kw.update(repository_dict)
         kw.update({'created_by': created_by})
         kw.update(kwargs)
         return callback(**kw)
     return 0
 def check_allowed_create_user(user_dict, created_by, **kwargs):
     # pre create hooks
     from kallithea import EXTENSIONS
     callback = getattr(EXTENSIONS, 'PRE_CREATE_USER_HOOK', None)
     if callable(callback):
         allowed, reason = callback(created_by=created_by, **user_dict)
         if not allowed:
             raise UserCreationError(reason)
 def log_create_user(user_dict, created_by, **kwargs):
     """
     Post create user Hook.
     :param user_dict: dict dump of user object
     available keys for user_dict:
      'username',
      'full_name_or_username',
      'full_contact',
      'user_id',
      'name',
      'firstname',
      'short_contact',
      'admin',
      'lastname',
      'ip_addresses',
      'ldap_dn',
      'email',
      'api_key',
      'last_login',
      'full_name',
      'active',
      'password',
      'emails',
     """
     from kallithea import EXTENSIONS
     callback = getattr(EXTENSIONS, 'CREATE_USER_HOOK', None)
     if callable(callback):
         return callback(created_by=created_by, **user_dict)
     return 0
 def log_delete_repository(repository_dict, deleted_by, **kwargs):
     """
     Post delete repository Hook.
     :param repository: dict dump of repository object
     :param deleted_by: username who deleted the repository
     available keys of repository_dict:
      'repo_type',
      'description',
      'private',
      'created_on',
      'enable_downloads',
      'repo_id',
      'owner_id',
      'enable_statistics',
      'clone_uri',
      'fork_id',
      'group_id',
      'repo_name'
     """
     from kallithea import EXTENSIONS
     callback = getattr(EXTENSIONS, 'DELETE_REPO_HOOK', None)
     if callable(callback):
         kw = {}
         kw.update(repository_dict)
         kw.update({'deleted_by': deleted_by,
                    'deleted_on': time.time()})
         kw.update(kwargs)
         return callback(**kw)
     return 0
 def log_delete_user(user_dict, deleted_by, **kwargs):
     """
     Post delete user Hook.
     :param user_dict: dict dump of user object
     available keys for user_dict:
      'username',
      'full_name_or_username',
      'full_contact',
      'user_id',
      'name',
      'firstname',
      'short_contact',
      'admin',
      'lastname',
      'ip_addresses',
      'ldap_dn',
      'email',
      'api_key',
      'last_login',
      'full_name',
      'active',
      'password',
      'emails',
     """
     from kallithea import EXTENSIONS
     callback = getattr(EXTENSIONS, 'DELETE_USER_HOOK', None)
     if callable(callback):
         return callback(deleted_by=deleted_by, **user_dict)
     return 0
 def _hook_environment(repo_path):
     """
     Create a light-weight environment for stand-alone scripts and return an UI and the
     db repository.
     Git hooks are executed as subprocess of Git while Kallithea is waiting, and
     they thus need enough info to be able to create an app environment and
     connect to the database.
     """
     import paste.deploy
     import kallithea.config.middleware
     extras = get_hook_environment()
     path_to_ini_file = extras['config']
     kallithea.CONFIG = paste.deploy.appconfig('config:' + path_to_ini_file)
     #logging.config.fileConfig(ini_file_path) # Note: we are in a different process - don't use configured logging
     kallithea.config.middleware.make_app(kallithea.CONFIG.global_conf, **kallithea.CONFIG.local_conf)
     # fix if it's not a bare repo
     if repo_path.endswith(os.sep + '.git'):
         repo_path = repo_path[:-5]
     repo = Repository.get_by_full_path(repo_path)
     if not repo:
         raise OSError('Repository %s not found in database'
                       % (safe_str(repo_path)))
         raise OSError('Repository %s not found in database' % repo_path)
     baseui = make_ui()
     return baseui, repo
 def handle_git_pre_receive(repo_path, git_stdin_lines):
     """Called from Git pre-receive hook"""
     # Currently unused. TODO: remove?
     return 0
 def handle_git_post_receive(repo_path, git_stdin_lines):
     """Called from Git post-receive hook"""
     try:
         baseui, repo = _hook_environment(repo_path)
     except HookEnvironmentError as e:
         sys.stderr.write("Skipping Kallithea Git post-recieve hook %r.\nGit was apparently not invoked by Kallithea: %s\n" % (sys.argv[0], e))
         return 0
     # the post push hook should never use the cached instance
     scm_repo = repo.scm_instance_no_cache()
     rev_data = []
     for l in git_stdin_lines:
         old_rev, new_rev, ref = l.strip().split(' ')
         _ref_data = ref.split('/')
         if _ref_data[1] in ['tags', 'heads']:
             rev_data.append({'old_rev': old_rev,
                              'new_rev': new_rev,
                              'ref': ref,
                              'type': _ref_data[1],
                              'name': '/'.join(_ref_data[2:])})
     git_revs = []
     for push_ref in rev_data:
         _type = push_ref['type']
         if _type == 'heads':
             if push_ref['old_rev'] == EmptyChangeset().raw_id:
                 # update the symbolic ref if we push new repo
                 if scm_repo.is_empty():
                     scm_repo._repo.refs.set_symbolic_ref(
                         b'HEAD',
                         b'refs/heads/%s' % safe_bytes(push_ref['name']))
                 # build exclude list without the ref
                 cmd = ['for-each-ref', '--format=%(refname)', 'refs/heads/*']
                 stdout = scm_repo.run_git_command(cmd)
                 ref = push_ref['ref']
                 heads = [head for head in stdout.splitlines() if head != ref]
                 # now list the git revs while excluding from the list
                 cmd = ['log', push_ref['new_rev'], '--reverse', '--pretty=format:%H']
                 cmd.append('--not')
                 cmd.extend(heads) # empty list is ok
                 stdout = scm_repo.run_git_command(cmd)
                 git_revs += stdout.splitlines()
             elif push_ref['new_rev'] == EmptyChangeset().raw_id:
                 # delete branch case
                 git_revs += ['delete_branch=>%s' % push_ref['name']]
             else:
                 cmd = ['log', '%(old_rev)s..%(new_rev)s' % push_ref,
                        '--reverse', '--pretty=format:%H']
                 stdout = scm_repo.run_git_command(cmd)
                 git_revs += stdout.splitlines()
         elif _type == 'tags':
             git_revs += ['tag=>%s' % push_ref['name']]
     process_pushed_raw_ids(git_revs)
     return 0
 # Almost exactly like Mercurial contrib/hg-ssh:
 def rejectpush(ui, **kwargs):
     """Mercurial hook to be installed as pretxnopen and prepushkey for read-only repos"""
     ex = get_hook_environment()
-    ui.warn(safe_bytes("Push access to %r denied\n" % safe_str(ex.repository)))
     ui.warn(safe_bytes("Push access to %r denied\n" % ex.repository))
     return 1

kallithea/lib/indexers/daemon.py

➞

Show inline comments

 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
+#
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.lib.indexers.daemon
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 A daemon will read from task table and run tasks
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Jan 26, 2010
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 import logging
 import os
 import sys
 import traceback
 from os.path import dirname
 from shutil import rmtree
 from time import mktime
 from whoosh.index import create_in, exists_in, open_dir
 from whoosh.qparser import QueryParser
 from kallithea.config.conf import INDEX_EXTENSIONS, INDEX_FILENAMES
 from kallithea.lib.indexers import CHGSET_IDX_NAME, CHGSETS_SCHEMA, IDX_NAME, SCHEMA
-from kallithea.lib.utils2 import safe_str, safe_unicode
 from kallithea.lib.utils2 import safe_unicode
 from kallithea.lib.vcs.exceptions import ChangesetError, NodeDoesNotExistError, RepositoryError
 from kallithea.model.db import Repository
 from kallithea.model.scm import ScmModel
 # Add location of top level folder to sys.path
 project_path = dirname(dirname(dirname(dirname(os.path.realpath(__file__)))))
 sys.path.append(project_path)
 log = logging.getLogger('whoosh_indexer')
 class WhooshIndexingDaemon(object):
     """
     Daemon for atomic indexing jobs
     """
     def __init__(self, indexname=IDX_NAME, index_location=None,
                  repo_location=None, repo_list=None,
                  repo_update_list=None):
         self.indexname = indexname
         self.index_location = index_location
         if not index_location:
             raise Exception('You have to provide index location')
         self.repo_location = repo_location
         if not repo_location:
             raise Exception('You have to provide repositories location')
         self.repo_paths = ScmModel().repo_scan(self.repo_location)
         # filter repo list
         if repo_list:
             repo_list = set(repo_list)
             self.filtered_repo_paths = {}
             for repo_name, repo in self.repo_paths.items():
                 if repo_name in repo_list:
                     self.filtered_repo_paths[repo_name] = repo
             self.repo_paths = self.filtered_repo_paths
         # filter update repo list
         self.filtered_repo_update_paths = {}
         if repo_update_list:
             self.filtered_repo_update_paths = {}
             for repo_name, repo in self.repo_paths.items():
                 if repo_name in repo_update_list:
                     self.filtered_repo_update_paths[repo_name] = repo
             self.repo_paths = self.filtered_repo_update_paths
         self.initial = True
         if not os.path.isdir(self.index_location):
             os.makedirs(self.index_location)
             log.info('Cannot run incremental index since it does not '
                      'yet exist - running full build')
         elif not exists_in(self.index_location, IDX_NAME):
             log.info('Running full index build, as the file content '
                      'index does not exist')
         elif not exists_in(self.index_location, CHGSET_IDX_NAME):
             log.info('Running full index build, as the changeset '
                      'index does not exist')
         else:
             self.initial = False
     def _get_index_revision(self, repo):
         db_repo = Repository.get_by_repo_name(repo.name)
         landing_rev = 'tip'
         if db_repo:
             _rev_type, _rev = db_repo.landing_rev
             landing_rev = _rev
         return landing_rev
     def _get_index_changeset(self, repo, index_rev=None):
         if not index_rev:
             index_rev = self._get_index_revision(repo)
         cs = repo.get_changeset(index_rev)
         return cs
     def get_paths(self, repo):
         """
         recursive walk in root dir and return a set of all path in that dir
         based on repository walk function
         """
         index_paths_ = set()
         try:
             cs = self._get_index_changeset(repo)
             for _topnode, _dirs, files in cs.walk('/'):
                 for f in files:
-                    index_paths_.add(os.path.join(safe_str(repo.path), safe_str(f.path)))
                     index_paths_.add(os.path.join(repo.path, f.path))
         except RepositoryError:
             log.debug(traceback.format_exc())
             pass
         return index_paths_
     def get_node(self, repo, path, index_rev=None):
         """
         gets a filenode based on given full path. It operates on string for
         hg git compatibility.
         gets a filenode based on given full path.
         :param repo: scm repo instance
         :param path: full path including root location
         :return: FileNode
         """
         # FIXME: paths should be normalized ... or even better: don't include repo.path
         path = safe_str(path)
         repo_path = safe_str(repo.path)
         assert path.startswith(repo_path)
         assert path[len(repo_path)] in (os.path.sep, os.path.altsep)
         node_path = path[len(repo_path) + 1:]
         assert path.startswith(repo.path)
         assert path[len(repo.path)] in (os.path.sep, os.path.altsep)
         node_path = path[len(repo.path) + 1:]
         cs = self._get_index_changeset(repo, index_rev=index_rev)
         node = cs.get_node(node_path)
         return node
     def is_indexable_node(self, node):
         """
         Just index the content of chosen files, skipping binary files
         """
         return (node.extension in INDEX_EXTENSIONS or node.name in INDEX_FILENAMES) and \
                not node.is_binary
     def get_node_mtime(self, node):
         return mktime(node.last_changeset.date.timetuple())
     def add_doc(self, writer, path, repo, repo_name, index_rev=None):
         """
         Adding doc to writer this function itself fetches data from
         the instance of vcs backend
         """
         try:
             node = self.get_node(repo, path, index_rev)
         except (ChangesetError, NodeDoesNotExistError):
             log.debug("    >> %s - not found in %s %s", path, repo, index_rev)
             return 0, 0
         indexed = indexed_w_content = 0
         if self.is_indexable_node(node):
             bytes_content = node.content
             if b'\0' in bytes_content:
                 log.warning('    >> %s - no text content', path)
                 u_content = u''
             else:
                 log.debug('    >> %s', path)
                 u_content = safe_unicode(bytes_content)
                 indexed_w_content += 1
         else:
             log.debug('    >> %s - not indexable', path)
             # just index file name without it's content
             u_content = u''
             indexed += 1
         writer.add_document(
             fileid=path,
             owner=repo.contact,
             repository_rawname=repo_name,
             repository=repo_name,
             path=path,
             content=u_content,
             modtime=self.get_node_mtime(node),
             extension=node.extension
+        )
         return indexed, indexed_w_content
     def index_changesets(self, writer, repo_name, repo, start_rev=None):
         """
         Add all changeset in the vcs repo starting at start_rev
         to the index writer
         :param writer: the whoosh index writer to add to
         :param repo_name: name of the repository from whence the
           changeset originates including the repository group
         :param repo: the vcs repository instance to index changesets for,
           the presumption is the repo has changesets to index
         :param start_rev=None: the full sha id to start indexing from
           if start_rev is None then index from the first changeset in
           the repo
         """
         if start_rev is None:
             start_rev = repo[0].raw_id
         log.debug('Indexing changesets in %s, starting at rev %s',
                   repo_name, start_rev)
         indexed = 0
         cs_iter = repo.get_changesets(start=start_rev)
         total = len(cs_iter)
         for cs in cs_iter:
             indexed += 1
             log.debug('    >> %s %s/%s', cs, indexed, total)
             writer.add_document(
                 raw_id=cs.raw_id,
                 owner=repo.contact,
                 date=cs._timestamp,
                 repository_rawname=repo_name,
                 repository=repo_name,
                 author=cs.author,
                 message=cs.message,
                 last=cs.last,
                 added=u' '.join(node.path for node in cs.added).lower(),
                 removed=u' '.join(node.path for node in cs.removed).lower(),
                 changed=u' '.join(node.path for node in cs.changed).lower(),
                 parents=u' '.join(cs.raw_id for cs in cs.parents),
+            )
         return indexed
     def index_files(self, file_idx_writer, repo_name, repo):
         """
         Index files for given repo_name
         :param file_idx_writer: the whoosh index writer to add to
         :param repo_name: name of the repository we're indexing
         :param repo: instance of vcs repo
         """
         i_cnt = iwc_cnt = 0
         log.debug('Building file index for %s @revision:%s', repo_name,
                                                 self._get_index_revision(repo))
         index_rev = self._get_index_revision(repo)
         for idx_path in self.get_paths(repo):
             i, iwc = self.add_doc(file_idx_writer, idx_path, repo, repo_name, index_rev)
             i_cnt += i
             iwc_cnt += iwc
         log.debug('added %s files %s with content for repo %s',
                   i_cnt + iwc_cnt, iwc_cnt, repo.path)
         return i_cnt, iwc_cnt
     def update_changeset_index(self):
         idx = open_dir(self.index_location, indexname=CHGSET_IDX_NAME)
         with idx.searcher() as searcher:
             writer = idx.writer()
             writer_is_dirty = False
             try:
                 indexed_total = 0
                 repo_name = None
                 for repo_name, repo in sorted(self.repo_paths.items()):
                     log.debug('Updating changeset index for repo %s', repo_name)
                     # skip indexing if there aren't any revs in the repo
                     num_of_revs = len(repo)
                     if num_of_revs < 1:
                         continue
                     qp = QueryParser('repository', schema=CHGSETS_SCHEMA)
                     q = qp.parse(u"last:t AND %s" % repo_name)
                     results = searcher.search(q)
                     # default to scanning the entire repo
                     last_rev = 0
                     start_id = None
                     if len(results) > 0:
                         # assuming that there is only one result, if not this
                         # may require a full re-index.
                         start_id = results[0]['raw_id']
                         last_rev = repo.get_changeset(revision=start_id).revision
                     # there are new changesets to index or a new repo to index
                     if last_rev == 0 or num_of_revs > last_rev + 1:
                         # delete the docs in the index for the previous
                         # last changeset(s)
                         for hit in results:
                             q = qp.parse(u"last:t AND %s AND raw_id:%s" %
                                             (repo_name, hit['raw_id']))
                             writer.delete_by_query(q)
                         # index from the previous last changeset + all new ones
                         indexed_total += self.index_changesets(writer,
                                                 repo_name, repo, start_id)
                         writer_is_dirty = True
                 log.debug('indexed %s changesets for repo %s',
                           indexed_total, repo_name
+                )
             finally:
                 if writer_is_dirty:
                     log.debug('>> COMMITING CHANGES TO CHANGESET INDEX<<')
                     writer.commit(merge=True)
                     log.debug('>>> FINISHED REBUILDING CHANGESET INDEX <<<')
                 else:
                     log.debug('>> NOTHING TO COMMIT TO CHANGESET INDEX<<')
     def update_file_index(self):
         log.debug(u'STARTING INCREMENTAL INDEXING UPDATE FOR EXTENSIONS %s '
                   'AND REPOS %s', INDEX_EXTENSIONS, ' and '.join(self.repo_paths))
         idx = open_dir(self.index_location, indexname=self.indexname)
         # The set of all paths in the index
         indexed_paths = set()
         # The set of all paths we need to re-index
         to_index = set()
         writer = idx.writer()
         writer_is_dirty = False
         try:
             with idx.reader() as reader:
                 # Loop over the stored fields in the index
                 for fields in reader.all_stored_fields():
                     indexed_path = fields['path']

kallithea/lib/middleware/simplehg.py

➞

Show inline comments

 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
+#
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.lib.middleware.simplehg
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 SimpleHg middleware for handling Mercurial protocol requests (push/clone etc.).
 It's implemented with basic auth function
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Apr 28, 2010
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 import logging
 import os
 import urllib.parse
 import mercurial.hgweb
 from kallithea.lib.base import BaseVCSController, get_path_info
 from kallithea.lib.utils import make_ui
-from kallithea.lib.utils2 import safe_bytes, safe_str
 from kallithea.lib.utils2 import safe_bytes
 log = logging.getLogger(__name__)
 def get_header_hgarg(environ):
     """Decode the special Mercurial encoding of big requests over multiple headers.
     >>> get_header_hgarg({})
     ''
     >>> get_header_hgarg({'HTTP_X_HGARG_0': ' ', 'HTTP_X_HGARG_1': 'a','HTTP_X_HGARG_2': '','HTTP_X_HGARG_3': 'b+c %20'})
     'ab+c %20'
     """
     chunks = []
     i = 1
     while True:
         v = environ.get('HTTP_X_HGARG_%d' % i)
         if v is None:
             break
         chunks.append(v)
         i += 1
     return ''.join(chunks)
 cmd_mapping = {
     # 'batch' is not in this list - it is handled explicitly
     'between': 'pull',
     'branches': 'pull',
     'branchmap': 'pull',
     'capabilities': 'pull',
     'changegroup': 'pull',
     'changegroupsubset': 'pull',
     'changesetdata': 'pull',
     'clonebundles': 'pull',
     'debugwireargs': 'pull',
     'filedata': 'pull',
     'getbundle': 'pull',
     'getlfile': 'pull',
     'heads': 'pull',
     'hello': 'pull',
     'known': 'pull',
     'lheads': 'pull',
     'listkeys': 'pull',
     'lookup': 'pull',
     'manifestdata': 'pull',
     'narrow_widen': 'pull',
     'protocaps': 'pull',
     'statlfile': 'pull',
     'stream_out': 'pull',
     'pushkey': 'push',
     'putlfile': 'push',
     'unbundle': 'push',
+    }
 class SimpleHg(BaseVCSController):
     scm_alias = 'hg'
     @classmethod
     def parse_request(cls, environ):
         http_accept = environ.get('HTTP_ACCEPT', '')
         if not http_accept.startswith('application/mercurial'):
             return None
         path_info = get_path_info(environ)
         if not path_info.startswith('/'): # it must!
             return None
         class parsed_request(object):
             repo_name = path_info[1:].rstrip('/')
             query_string = environ['QUERY_STRING']
             action = None
             for qry in query_string.split('&'):
                 parts = qry.split('=', 1)
                 if len(parts) == 2 and parts[0] == 'cmd':
                     cmd = parts[1]
                     if cmd == 'batch':
                         hgarg = get_header_hgarg(environ)
                         if not hgarg.startswith('cmds='):
                             action = 'push' # paranoid and safe
                             break
                         action = 'pull'
                         for cmd_arg in hgarg[5:].split(';'):
                             cmd, _args = urllib.parse.unquote_plus(cmd_arg).split(' ', 1)
                             op = cmd_mapping.get(cmd, 'push')
                             if op != 'pull':
                                 assert op == 'push'
                                 action = 'push'
                                 break
                     else:
                         action = cmd_mapping.get(cmd, 'push')
                     break # only process one cmd
         return parsed_request
     def _make_app(self, parsed_request):
         """
         Make an hgweb wsgi application.
         """
         str_repo_name = safe_str(parsed_request.repo_name)
         repo_path = os.path.join(safe_str(self.basepath), str_repo_name)
         repo_name = parsed_request.repo_name
         repo_path = os.path.join(self.basepath, repo_name)
         baseui = make_ui(repo_path=repo_path)
-        hgweb_app = mercurial.hgweb.hgweb(safe_bytes(repo_path), name=str_repo_name, baseui=baseui)
+        hgweb_app = mercurial.hgweb.hgweb(safe_bytes(repo_path), name=safe_bytes(repo_name), baseui=baseui)
         def wrapper_app(environ, start_response):
-            environ['REPO_NAME'] = str_repo_name # used by mercurial.hgweb.hgweb
+            environ['REPO_NAME'] = repo_name # used by mercurial.hgweb.hgweb
             return hgweb_app(environ, start_response)
         return wrapper_app

kallithea/lib/utils.py

➞

Show inline comments

 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
+#
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.lib.utils
 ~~~~~~~~~~~~~~~~~~~
 Utilities library for Kallithea
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Apr 18, 2010
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 import datetime
 import logging
 import os
 import re
 import sys
 import traceback
 from distutils.version import StrictVersion
 import beaker.cache
 import mercurial.config
 import mercurial.ui
 from tg.i18n import ugettext as _
 import kallithea.config.conf
 from kallithea.lib.exceptions import HgsubversionImportError
-from kallithea.lib.utils2 import ascii_bytes, aslist, get_current_authuser, safe_bytes, safe_str
 from kallithea.lib.utils2 import ascii_bytes, aslist, get_current_authuser, safe_bytes
 from kallithea.lib.vcs.backends.git.repository import GitRepository
 from kallithea.lib.vcs.backends.hg.repository import MercurialRepository
 from kallithea.lib.vcs.conf import settings
 from kallithea.lib.vcs.exceptions import RepositoryError, VCSError
 from kallithea.lib.vcs.utils.fakemod import create_module
 from kallithea.lib.vcs.utils.helpers import get_scm
 from kallithea.model import meta
 from kallithea.model.db import RepoGroup, Repository, Setting, Ui, User, UserGroup, UserLog
 log = logging.getLogger(__name__)
 REMOVED_REPO_PAT = re.compile(r'rm__\d{8}_\d{6}_\d{6}_.*')
 #==============================================================================
 # PERM DECORATOR HELPERS FOR EXTRACTING NAMES FOR PERM CHECKS
 #==============================================================================
 def get_repo_slug(request):
     _repo = request.environ['pylons.routes_dict'].get('repo_name')
     if _repo:
         _repo = _repo.rstrip('/')
     return _repo
 def get_repo_group_slug(request):
     _group = request.environ['pylons.routes_dict'].get('group_name')
     if _group:
         _group = _group.rstrip('/')
     return _group
 def get_user_group_slug(request):
     _group = request.environ['pylons.routes_dict'].get('id')
     _group = UserGroup.get(_group)
     if _group:
         return _group.users_group_name
     return None
 def _get_permanent_id(s):
     """Helper for decoding stable URLs with repo ID. For a string like '_123'
     return 123.
     """
     by_id_match = re.match(r'^_(\d+)$', s)
     if by_id_match is None:
         return None
     return int(by_id_match.group(1))
 def fix_repo_id_name(path):
     """
     Rewrite repo_name for _<ID> permanent URLs.
     Given a path, if the first path element is like _<ID>, return the path with
     this part expanded to the corresponding full repo name, else return the
     provided path.
     """
     first, rest = path, ''
     if '/' in path:
         first, rest_ = path.split('/', 1)
         rest = '/' + rest_
     repo_id = _get_permanent_id(first)
     if repo_id is not None:
         repo = Repository.get(repo_id)
         if repo is not None:
             return repo.repo_name + rest
     return path
 def action_logger(user, action, repo, ipaddr='', commit=False):
     """
     Action logger for various actions made by users
     :param user: user that made this action, can be a unique username string or
         object containing user_id attribute
     :param action: action to log, should be on of predefined unique actions for
         easy translations
     :param repo: string name of repository or object containing repo_id,
         that action was made on
     :param ipaddr: optional IP address from what the action was made
     """
     # if we don't get explicit IP address try to get one from registered user
     # in tmpl context var
     if not ipaddr:
         ipaddr = getattr(get_current_authuser(), 'ip_addr', '')
     if getattr(user, 'user_id', None):
         user_obj = User.get(user.user_id)
     elif isinstance(user, str):
         user_obj = User.get_by_username(user)
     else:
         raise Exception('You have to provide a user object or a username')
     if getattr(repo, 'repo_id', None):
         repo_obj = Repository.get(repo.repo_id)
         repo_name = repo_obj.repo_name
     elif isinstance(repo, str):
         repo_name = repo.lstrip('/')
         repo_obj = Repository.get_by_repo_name(repo_name)
     else:
         repo_obj = None
         repo_name = u''
     user_log = UserLog()
     user_log.user_id = user_obj.user_id
     user_log.username = user_obj.username
     user_log.action = action
     user_log.repository = repo_obj
     user_log.repository_name = repo_name
     user_log.action_date = datetime.datetime.now()
     user_log.user_ip = ipaddr
     meta.Session().add(user_log)
     log.info('Logging action:%s on %s by user:%s ip:%s',
              action, repo, user_obj, ipaddr)
     if commit:
         meta.Session().commit()
 def get_filesystem_repos(path):
     """
     Scans given path for repos and return (name,(type,path)) tuple
     :param path: path to scan for repositories
     :param recursive: recursive search and return names with subdirs in front
     """
     # remove ending slash for better results
-    path = safe_str(path.rstrip(os.sep))
     path = path.rstrip(os.sep)
     log.debug('now scanning in %s', path)
     def isdir(*n):
         return os.path.isdir(os.path.join(*n))
     for root, dirs, _files in os.walk(path):
         recurse_dirs = []
         for subdir in dirs:
             # skip removed repos
             if REMOVED_REPO_PAT.match(subdir):
                 continue
             # skip .<something> dirs TODO: rly? then we should prevent creating them ...
             if subdir.startswith('.'):
                 continue
             cur_path = os.path.join(root, subdir)
             if isdir(cur_path, '.git'):
                 log.warning('ignoring non-bare Git repo: %s', cur_path)
                 continue
             if (isdir(cur_path, '.hg') or
                 isdir(cur_path, '.svn') or
                 isdir(cur_path, 'objects') and (isdir(cur_path, 'refs') or
                                                 os.path.isfile(os.path.join(cur_path, 'packed-refs')))):
                 if not os.access(cur_path, os.R_OK) or not os.access(cur_path, os.X_OK):
                     log.warning('ignoring repo path without access: %s', cur_path)
                     continue
                 if not os.access(cur_path, os.W_OK):
                     log.warning('repo path without write access: %s', cur_path)
                 try:
                     scm_info = get_scm(cur_path)
                     assert cur_path.startswith(path)
                     repo_path = cur_path[len(path) + 1:]
                     yield repo_path, scm_info
                     continue # no recursion
                 except VCSError:
                     # We should perhaps ignore such broken repos, but especially
                     # the bare git detection is unreliable so we dive into it
                     pass
             recurse_dirs.append(subdir)
         dirs[:] = recurse_dirs
 def is_valid_repo_uri(repo_type, url, ui):
     """Check if the url seems like a valid remote repo location - raise an Exception if any problems"""
     if repo_type == 'hg':
         if url.startswith('http') or url.startswith('ssh'):
             # initially check if it's at least the proper URL
             # or does it pass basic auth
             MercurialRepository._check_url(url, ui)
         elif url.startswith('svn+http'):
             try:
                 from hgsubversion.svnrepo import svnremoterepo
             except ImportError:
                 raise HgsubversionImportError(_('Unable to activate hgsubversion support. '
                                                 'The "hgsubversion" library is missing'))
             svnremoterepo(ui, url).svn.uuid
         elif url.startswith('git+http'):
             raise NotImplementedError()
         else:
             raise Exception('URI %s not allowed' % (url,))
     elif repo_type == 'git':
         if url.startswith('http') or url.startswith('git'):
             # initially check if it's at least the proper URL
             # or does it pass basic auth
             GitRepository._check_url(url)
         elif url.startswith('svn+http'):
             raise NotImplementedError()
         elif url.startswith('hg+http'):
             raise NotImplementedError()
         else:
             raise Exception('URI %s not allowed' % (url))
 def is_valid_repo(repo_name, base_path, scm=None):
     """
     Returns True if given path is a valid repository False otherwise.
     If scm param is given also compare if given scm is the same as expected
     from scm parameter
     :param repo_name:
     :param base_path:
     :param scm:
     :return True: if given path is a valid repository
     """
     # TODO: paranoid security checks?
-    full_path = os.path.join(safe_str(base_path), safe_str(repo_name))
     full_path = os.path.join(base_path, repo_name)
     try:
         scm_ = get_scm(full_path)
         if scm:
             return scm_[0] == scm
         return True
     except VCSError:
         return False
 def is_valid_repo_group(repo_group_name, base_path, skip_path_check=False):
     """
     Returns True if given path is a repository group False otherwise
     :param repo_name:
     :param base_path:
     """
-    full_path = os.path.join(safe_str(base_path), safe_str(repo_group_name))
     full_path = os.path.join(base_path, repo_group_name)
     # check if it's not a repo
     if is_valid_repo(repo_group_name, base_path):
         return False
     try:
         # we need to check bare git repos at higher level
         # since we might match branches/hooks/info/objects or possible
         # other things inside bare git repo
         get_scm(os.path.dirname(full_path))
         return False
     except VCSError:
         pass
     # check if it's a valid path
     if skip_path_check or os.path.isdir(full_path):
         return True
     return False
 # propagated from mercurial documentation
 ui_sections = ['alias', 'auth',
                 'decode/encode', 'defaults',
                 'diff', 'email',
                 'extensions', 'format',
                 'merge-patterns', 'merge-tools',
                 'hooks', 'http_proxy',
                 'smtp', 'patch',
                 'paths', 'profiling',
                 'server', 'trusted',
                 'ui', 'web', ]
 def make_ui(repo_path=None):
     """
     Create an Mercurial 'ui' object based on database Ui settings, possibly
     augmenting with content from a hgrc file.
     """
     baseui = mercurial.ui.ui()
     # clean the baseui object
     baseui._ocfg = mercurial.config.config()
     baseui._ucfg = mercurial.config.config()
     baseui._tcfg = mercurial.config.config()
     sa = meta.Session()
     for ui_ in sa.query(Ui).all():
         if ui_.ui_active:
             log.debug('config from db: [%s] %s=%r', ui_.ui_section,
                       ui_.ui_key, ui_.ui_value)
             baseui.setconfig(ascii_bytes(ui_.ui_section), ascii_bytes(ui_.ui_key),
                              b'' if ui_.ui_value is None else safe_bytes(ui_.ui_value))
     # force set push_ssl requirement to False, Kallithea handles that
     baseui.setconfig(b'web', b'push_ssl', False)
     baseui.setconfig(b'web', b'allow_push', b'*')
     # prevent interactive questions for ssh password / passphrase
     ssh = baseui.config(b'ui', b'ssh', default=b'ssh')
     baseui.setconfig(b'ui', b'ssh', b'%s -oBatchMode=yes -oIdentitiesOnly=yes' % ssh)
     # push / pull hooks
     baseui.setconfig(b'hooks', b'changegroup.kallithea_log_push_action', b'python:kallithea.lib.hooks.log_push_action')
     baseui.setconfig(b'hooks', b'outgoing.kallithea_log_pull_action', b'python:kallithea.lib.hooks.log_pull_action')
     if repo_path is not None:
         hgrc_path = os.path.join(repo_path, '.hg', 'hgrc')
         if os.path.isfile(hgrc_path):
             log.debug('reading hgrc from %s', hgrc_path)
             cfg = mercurial.config.config()
             cfg.read(safe_bytes(hgrc_path))
             for section in ui_sections:
                 for k, v in cfg.items(section):
                     log.debug('config from file: [%s] %s=%s', section, k, v)
                     baseui.setconfig(ascii_bytes(section), ascii_bytes(k), safe_bytes(v))
         else:
             log.debug('hgrc file is not present at %s, skipping...', hgrc_path)
     return baseui
 def set_app_settings(config):
     """
     Updates app config with new settings from database
     :param config:
     """
     hgsettings = Setting.get_app_settings()
     for k, v in hgsettings.items():
         config[k] = v
 def set_vcs_config(config):
     """
     Patch VCS config with some Kallithea specific stuff
     :param config: kallithea.CONFIG
     """
     settings.BACKENDS = {
         'hg': 'kallithea.lib.vcs.backends.hg.MercurialRepository',
         'git': 'kallithea.lib.vcs.backends.git.GitRepository',
+    }
     settings.GIT_EXECUTABLE_PATH = config.get('git_path', 'git')
     settings.GIT_REV_FILTER = config.get('git_rev_filter', '--all').strip()
     settings.DEFAULT_ENCODINGS = aslist(config.get('default_encoding',
                                                         'utf-8'), sep=',')
 def set_indexer_config(config):
     """
     Update Whoosh index mapping
     :param config: kallithea.CONFIG
     """
     log.debug('adding extra into INDEX_EXTENSIONS')
     kallithea.config.conf.INDEX_EXTENSIONS.extend(re.split(r'\s+', config.get('index.extensions', '')))
     log.debug('adding extra into INDEX_FILENAMES')
     kallithea.config.conf.INDEX_FILENAMES.extend(re.split(r'\s+', config.get('index.filenames', '')))
 def map_groups(path):
     """
     Given a full path to a repository, create all nested groups that this
     repo is inside. This function creates parent-child relationships between
     groups and creates default perms for all new groups.
     :param paths: full path to repository
     """
     from kallithea.model.repo_group import RepoGroupModel
     sa = meta.Session()
     groups = path.split(Repository.url_sep())
     parent = None
     group = None
     # last element is repo in nested groups structure
     groups = groups[:-1]
     rgm = RepoGroupModel()
     owner = User.get_first_admin()
     for lvl, group_name in enumerate(groups):
         group_name = u'/'.join(groups[:lvl] + [group_name])
         group = RepoGroup.get_by_group_name(group_name)
         desc = '%s group' % group_name
         # skip folders that are now removed repos
         if REMOVED_REPO_PAT.match(group_name):
             break
         if group is None:
             log.debug('creating group level: %s group_name: %s',
                       lvl, group_name)
             group = RepoGroup(group_name, parent)
             group.group_description = desc
             group.owner = owner
             sa.add(group)
             rgm._create_default_perms(group)
             sa.flush()
         parent = group
     return group
 def repo2db_mapper(initial_repo_dict, remove_obsolete=False,
                    install_git_hooks=False, user=None, overwrite_git_hooks=False):
     """
     maps all repos given in initial_repo_dict, non existing repositories
     are created, if remove_obsolete is True it also check for db entries
     that are not in initial_repo_dict and removes them.
     :param initial_repo_dict: mapping with repositories found by scanning methods
     :param remove_obsolete: check for obsolete entries in database
     :param install_git_hooks: if this is True, also check and install git hook
         for a repo if missing
     :param overwrite_git_hooks: if this is True, overwrite any existing git hooks
         that may be encountered (even if user-deployed)
     """
     from kallithea.model.repo import RepoModel
     from kallithea.model.scm import ScmModel
     sa = meta.Session()
     repo_model = RepoModel()
     if user is None:
         user = User.get_first_admin()
     added = []
     # creation defaults
     defs = Setting.get_default_repo_settings(strip_prefix=True)
     enable_statistics = defs.get('repo_enable_statistics')
     enable_downloads = defs.get('repo_enable_downloads')
     private = defs.get('repo_private')
     for name, repo in initial_repo_dict.items():
         group = map_groups(name)

kallithea/lib/utils2.py

➞

Show inline comments

@@ @@ -140,474 +140,474 @@ def generate_api_key(): @@
         unreserved = ALPHA / DIGIT / "-" / "." / "_" / "~"
     """
     # Hexadecimal certainly qualifies as URL-safe.
     return ascii_str(binascii.hexlify(os.urandom(20)))
 def safe_int(val, default=None):
     """
     Returns int() of val if val is not convertable to int use default
     instead
     :param val:
     :param default:
     """
     try:
         val = int(val)
     except (ValueError, TypeError):
         val = default
     return val
 def remove_suffix(s, suffix):
     if s.endswith(suffix):
         s = s[:-1 * len(suffix)]
     return s
 def remove_prefix(s, prefix):
     if s.startswith(prefix):
         s = s[len(prefix):]
     return s
 def age(prevdate, show_short_version=False, now=None):
     """
     turns a datetime into an age string.
     If show_short_version is True, then it will generate a not so accurate but shorter string,
     example: 2days ago, instead of 2 days and 23 hours ago.
     :param prevdate: datetime object
     :param show_short_version: if it should approximate the date and return a shorter string
     :rtype: unicode
     :returns: unicode words describing age
     """
     now = now or datetime.datetime.now()
     order = ['year', 'month', 'day', 'hour', 'minute', 'second']
     deltas = {}
     future = False
     if prevdate > now:
         now, prevdate = prevdate, now
         future = True
     if future:
         prevdate = prevdate.replace(microsecond=0)
     # Get date parts deltas
     from dateutil import relativedelta
     for part in order:
         d = relativedelta.relativedelta(now, prevdate)
         deltas[part] = getattr(d, part + 's')
     # Fix negative offsets (there is 1 second between 10:59:59 and 11:00:00,
     # not 1 hour, -59 minutes and -59 seconds)
     for num, length in [(5, 60), (4, 60), (3, 24)]:  # seconds, minutes, hours
         part = order[num]
         carry_part = order[num - 1]
         if deltas[part] < 0:
             deltas[part] += length
             deltas[carry_part] -= 1
     # Same thing for days except that the increment depends on the (variable)
     # number of days in the month
     month_lengths = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]
     if deltas['day'] < 0:
         if prevdate.month == 2 and (prevdate.year % 4 == 0 and
             (prevdate.year % 100 != 0 or prevdate.year % 400 == 0)
         ):
             deltas['day'] += 29
         else:
             deltas['day'] += month_lengths[prevdate.month - 1]
         deltas['month'] -= 1
     if deltas['month'] < 0:
         deltas['month'] += 12
         deltas['year'] -= 1
     # In short version, we want nicer handling of ages of more than a year
     if show_short_version:
         if deltas['year'] == 1:
             # ages between 1 and 2 years: show as months
             deltas['month'] += 12
             deltas['year'] = 0
         if deltas['year'] >= 2:
             # ages 2+ years: round
             if deltas['month'] > 6:
                 deltas['year'] += 1
                 deltas['month'] = 0
     # Format the result
     fmt_funcs = {
         'year': lambda d: ungettext(u'%d year', '%d years', d) % d,
         'month': lambda d: ungettext(u'%d month', '%d months', d) % d,
         'day': lambda d: ungettext(u'%d day', '%d days', d) % d,
         'hour': lambda d: ungettext(u'%d hour', '%d hours', d) % d,
         'minute': lambda d: ungettext(u'%d minute', '%d minutes', d) % d,
         'second': lambda d: ungettext(u'%d second', '%d seconds', d) % d,
+    }
     for i, part in enumerate(order):
         value = deltas[part]
         if value == 0:
             continue
         if i < 5:
             sub_part = order[i + 1]
             sub_value = deltas[sub_part]
         else:
             sub_value = 0
         if sub_value == 0 or show_short_version:
             if future:
                 return _('in %s') % fmt_funcs[part](value)
             else:
                 return _('%s ago') % fmt_funcs[part](value)
         if future:
             return _('in %s and %s') % (fmt_funcs[part](value),
                 fmt_funcs[sub_part](sub_value))
         else:
             return _('%s and %s ago') % (fmt_funcs[part](value),
                 fmt_funcs[sub_part](sub_value))
     return _('just now')
 def uri_filter(uri):
     """
     Removes user:password from given url string
     :param uri:
     :rtype: unicode
     :returns: filtered list of strings
     """
     if not uri:
         return []
     proto = ''
     for pat in ('https://', 'http://', 'git://'):
         if uri.startswith(pat):
             uri = uri[len(pat):]
             proto = pat
             break
     # remove passwords and username
     uri = uri[uri.find('@') + 1:]
     # get the port
     cred_pos = uri.find(':')
     if cred_pos == -1:
         host, port = uri, None
     else:
         host, port = uri[:cred_pos], uri[cred_pos + 1:]
     return [_f for _f in [proto, host, port] if _f]
 def credentials_filter(uri):
     """
     Returns a url with removed credentials
     :param uri:
     """
     uri = uri_filter(uri)
     # check if we have port
     if len(uri) > 2 and uri[2]:
         uri[2] = ':' + uri[2]
     return ''.join(uri)
 def get_clone_url(clone_uri_tmpl, prefix_url, repo_name, repo_id, username=None):
     parsed_url = urlobject.URLObject(prefix_url)
     prefix = urllib.parse.unquote(parsed_url.path.rstrip('/'))
     try:
         system_user = pwd.getpwuid(os.getuid()).pw_name
     except Exception: # TODO: support all systems - especially Windows
         system_user = 'kallithea' # hardcoded default value ...
     args = {
         'scheme': parsed_url.scheme,
-        'user': urllib.parse.quote(safe_str(username or '')),
         'user': urllib.parse.quote(username or ''),
         'netloc': parsed_url.netloc + prefix,  # like "hostname:port/prefix" (with optional ":port" and "/prefix")
         'prefix': prefix, # undocumented, empty or starting with /
         'repo': repo_name,
         'repoid': str(repo_id),
         'system_user': system_user,
         'hostname': parsed_url.hostname,
+    }
     url = re.sub('{([^{}]+)}', lambda m: args.get(m.group(1), m.group(0)), clone_uri_tmpl)
     # remove leading @ sign if it's present. Case of empty user
     url_obj = urlobject.URLObject(url)
     if not url_obj.username:
         url_obj = url_obj.with_username(None)
     return str(url_obj)
 def get_changeset_safe(repo, rev):
     """
     Safe version of get_changeset if this changeset doesn't exists for a
     repo it returns a Dummy one instead
     :param repo:
     :param rev:
     """
     from kallithea.lib.vcs.backends.base import BaseRepository
     from kallithea.lib.vcs.exceptions import RepositoryError
     from kallithea.lib.vcs.backends.base import EmptyChangeset
     if not isinstance(repo, BaseRepository):
         raise Exception('You must pass an Repository '
                         'object as first argument got %s' % type(repo))
     try:
         cs = repo.get_changeset(rev)
     except (RepositoryError, LookupError):
         cs = EmptyChangeset(requested_revision=rev)
     return cs
 def datetime_to_time(dt):
     if dt:
         return time.mktime(dt.timetuple())
 def time_to_datetime(tm):
     if tm:
         if isinstance(tm, str):
             try:
                 tm = float(tm)
             except ValueError:
                 return
         return datetime.datetime.fromtimestamp(tm)
 # Must match regexp in kallithea/public/js/base.js MentionsAutoComplete()
 # Check char before @ - it must not look like we are in an email addresses.
 # Matching is greedy so we don't have to look beyond the end.
 MENTIONS_REGEX = re.compile(r'(?:^|(?<=[^a-zA-Z0-9]))@([a-zA-Z0-9][-_.a-zA-Z0-9]*[a-zA-Z0-9])')
 def extract_mentioned_usernames(text):
     r"""
     Returns list of (possible) usernames @mentioned in given text.
     >>> extract_mentioned_usernames('@1-2.a_X,@1234 not@not @ddd@not @n @ee @ff @gg, @gg;@hh @n\n@zz,')
     ['1-2.a_X', '1234', 'ddd', 'ee', 'ff', 'gg', 'gg', 'hh', 'zz']
     """
     return MENTIONS_REGEX.findall(text)
 def extract_mentioned_users(text):
     """ Returns set of actual database Users @mentioned in given text. """
     from kallithea.model.db import User
     result = set()
     for name in extract_mentioned_usernames(text):
         user = User.get_by_username(name, case_insensitive=True)
         if user is not None and not user.is_default_user:
             result.add(user)
     return result
 class AttributeDict(dict):
     def __getattr__(self, attr):
         return self.get(attr, None)
     __setattr__ = dict.__setitem__
     __delattr__ = dict.__delitem__
 def obfuscate_url_pw(engine):
     from sqlalchemy.engine import url as sa_url
     from sqlalchemy.exc import ArgumentError
     try:
         _url = sa_url.make_url(engine or '')
     except ArgumentError:
         return engine
     if _url.password:
         _url.password = 'XXXXX'
     return str(_url)
 class HookEnvironmentError(Exception): pass
 def get_hook_environment():
     """
     Get hook context by deserializing the global KALLITHEA_EXTRAS environment
     variable.
     Called early in Git out-of-process hooks to get .ini config path so the
     basic environment can be configured properly. Also used in all hooks to get
     information about the action that triggered it.
     """
     try:
         kallithea_extras = os.environ['KALLITHEA_EXTRAS']
     except KeyError:
         raise HookEnvironmentError("Environment variable KALLITHEA_EXTRAS not found")
     extras = json.loads(kallithea_extras)
     for k in ['username', 'repository', 'scm', 'action', 'ip', 'config']:
         try:
             extras[k]
         except KeyError:
             raise HookEnvironmentError('Missing key %s in KALLITHEA_EXTRAS %s' % (k, extras))
     return AttributeDict(extras)
 def set_hook_environment(username, ip_addr, repo_name, repo_alias, action=None):
     """Prepare global context for running hooks by serializing data in the
     global KALLITHEA_EXTRAS environment variable.
     Most importantly, this allow Git hooks to do proper logging and updating of
     caches after pushes.
     Must always be called before anything with hooks are invoked.
     """
     from kallithea import CONFIG
     extras = {
         'ip': ip_addr, # used in log_push/pull_action action_logger
         'username': username,
         'action': action or 'push_local', # used in log_push_action_raw_ids action_logger
         'repository': repo_name,
         'scm': repo_alias, # used to pick hack in log_push_action_raw_ids
         'config': CONFIG['__file__'], # used by git hook to read config
+    }
     os.environ['KALLITHEA_EXTRAS'] = json.dumps(extras)
 def get_current_authuser():
     """
     Gets kallithea user from threadlocal tmpl_context variable if it's
     defined, else returns None.
     """
     from tg import tmpl_context
     try:
         return getattr(tmpl_context, 'authuser', None)
     except TypeError:  # No object (name: context) has been registered for this thread
         return None
 class OptionalAttr(object):
     """
     Special Optional Option that defines other attribute. Example::
         def test(apiuser, userid=Optional(OAttr('apiuser')):
             user = Optional.extract(userid)
             # calls
     """
     def __init__(self, attr_name):
         self.attr_name = attr_name
     def __repr__(self):
         return '<OptionalAttr:%s>' % self.attr_name
     def __call__(self):
         return self
 # alias
 OAttr = OptionalAttr
 class Optional(object):
     """
     Defines an optional parameter::
         param = param.getval() if isinstance(param, Optional) else param
         param = param() if isinstance(param, Optional) else param
     is equivalent of::
         param = Optional.extract(param)
     """
     def __init__(self, type_):
         self.type_ = type_
     def __repr__(self):
         return '<Optional:%s>' % self.type_.__repr__()
     def __call__(self):
         return self.getval()
     def getval(self):
         """
         returns value from this Optional instance
         """
         if isinstance(self.type_, OAttr):
             # use params name
             return self.type_.attr_name
         return self.type_
     @classmethod
     def extract(cls, val):
         """
         Extracts value from Optional() instance
         :param val:
         :return: original value if it's not Optional instance else
             value of instance
         """
         if isinstance(val, cls):
             return val.getval()
         return val
 def urlreadable(s, _cleanstringsub=re.compile('[^-a-zA-Z0-9./]+').sub):
-    return _cleanstringsub('_', safe_str(s)).rstrip('_')
     return _cleanstringsub('_', s).rstrip('_')
 def recursive_replace(str_, replace=' '):
     """
     Recursive replace of given sign to just one instance
     :param str_: given string
     :param replace: char to find and replace multiple instances
     Examples::
     >>> recursive_replace("Mighty---Mighty-Bo--sstones",'-')
     'Mighty-Mighty-Bo-sstones'
     """
     if str_.find(replace * 2) == -1:
         return str_
     else:
         str_ = str_.replace(replace * 2, replace)
         return recursive_replace(str_, replace)
 def repo_name_slug(value):
     """
     Return slug of name of repository
     This function is called on each creation/modification
     of repository to prevent bad names in repo
     """
     slug = remove_formatting(value)
     slug = strip_tags(slug)
     for c in r"""`?=[]\;'"<>,/~!@#$%^&*()+{}|: """:
         slug = slug.replace(c, '-')
     slug = recursive_replace(slug, '-')
     slug = collapse(slug, '-')
     return slug
 def ask_ok(prompt, retries=4, complaint='Yes or no please!'):
     while True:
         ok = input(prompt)
         if ok in ('y', 'ye', 'yes'):
             return True
         if ok in ('n', 'no', 'nop', 'nope'):
             return False
         retries = retries - 1
         if retries < 0:
             raise IOError
         print(complaint)

kallithea/lib/vcs/backends/git/changeset.py

➞

Show inline comments

 import re
 from io import BytesIO
 from itertools import chain
 from subprocess import PIPE, Popen
 from dulwich import objects
 from dulwich.config import ConfigFile
 from kallithea.lib.vcs.backends.base import BaseChangeset, EmptyChangeset
 from kallithea.lib.vcs.conf import settings
 from kallithea.lib.vcs.exceptions import ChangesetDoesNotExistError, ChangesetError, ImproperArchiveTypeError, NodeDoesNotExistError, RepositoryError, VCSError
 from kallithea.lib.vcs.nodes import (
     AddedFileNodesGenerator, ChangedFileNodesGenerator, DirNode, FileNode, NodeKind, RemovedFileNodesGenerator, RootNode, SubModuleNode)
-from kallithea.lib.vcs.utils import ascii_bytes, ascii_str, date_fromtimestamp, safe_int, safe_str, safe_unicode
 from kallithea.lib.vcs.utils import ascii_bytes, ascii_str, date_fromtimestamp, safe_int, safe_unicode
 from kallithea.lib.vcs.utils.lazy import LazyProperty
 class GitChangeset(BaseChangeset):
     """
     Represents state of the repository at a revision.
     """
     def __init__(self, repository, revision):
         self._stat_modes = {}
         self.repository = repository
         revision = safe_str(revision)
         try:
             commit = self.repository._repo[ascii_bytes(revision)]
             if isinstance(commit, objects.Tag):
                 revision = safe_str(commit.object[1])
                 commit = self.repository._repo.get_object(commit.object[1])
         except KeyError:
             raise RepositoryError("Cannot get object with id %s" % revision)
         self.raw_id = ascii_str(commit.id)
         self.short_id = self.raw_id[:12]
         self._commit = commit  # a Dulwich Commmit with .id
         self._tree_id = commit.tree
         self._committer_property = 'committer'
         self._author_property = 'author'
         self._date_property = 'commit_time'
         self._date_tz_property = 'commit_timezone'
         self.revision = repository.revisions.index(self.raw_id)
         self.nodes = {}
         self._paths = {}
     @LazyProperty
     def bookmarks(self):
         return ()
     @LazyProperty
     def message(self):
         return safe_unicode(self._commit.message)
     @LazyProperty
     def committer(self):
         return safe_unicode(getattr(self._commit, self._committer_property))
     @LazyProperty
     def author(self):
         return safe_unicode(getattr(self._commit, self._author_property))
     @LazyProperty
     def date(self):
         return date_fromtimestamp(getattr(self._commit, self._date_property),
                                   getattr(self._commit, self._date_tz_property))
     @LazyProperty
     def _timestamp(self):
         return getattr(self._commit, self._date_property)
     @LazyProperty
     def status(self):
         """
         Returns modified, added, removed, deleted files for current changeset
         """
         return self.changed, self.added, self.removed
     @LazyProperty
     def tags(self):
         _tags = []
         for tname, tsha in self.repository.tags.items():
             if tsha == self.raw_id:
                 _tags.append(tname)
         return _tags
     @LazyProperty
     def branch(self):
         # Note: This function will return one branch name for the changeset -
         # that might not make sense in Git where branches() is a better match
         # for the basic model
         heads = self.repository._heads(reverse=False)
         ref = heads.get(self._commit.id)
         if ref:
             return safe_unicode(ref)
     @LazyProperty
     def branches(self):
         heads = self.repository._heads(reverse=True)
         return [b for b in heads if heads[b] == self._commit.id] # FIXME: Inefficient ... and returning None!
     def _fix_path(self, path):
         """
         Paths are stored without trailing slash so we need to get rid off it if
         needed.
         """
         if path.endswith('/'):
             path = path.rstrip('/')
         return path
     def _get_id_for_path(self, path):
         path = safe_str(path)
         # FIXME: Please, spare a couple of minutes and make those codes cleaner;
         if path not in self._paths:
             path = path.strip('/')
             # set root tree
             tree = self.repository._repo[self._tree_id]
             if path == '':
                 self._paths[''] = tree.id
                 return tree.id
             splitted = path.split('/')
             dirs, name = splitted[:-1], splitted[-1]
             curdir = ''
             # initially extract things from root dir
             for item, stat, id in tree.items():
                 if curdir:
                     name = '/'.join((curdir, item))
                 else:
                     name = item
                 self._paths[name] = id
                 self._stat_modes[name] = stat
             for dir in dirs:
                 if curdir:
                     curdir = '/'.join((curdir, dir))
                 else:
                     curdir = dir
                 dir_id = None
                 for item, stat, id in tree.items():
                     if dir == item:
                         dir_id = id
                 if dir_id:
                     # Update tree
                     tree = self.repository._repo[dir_id]
                     if not isinstance(tree, objects.Tree):
                         raise ChangesetError('%s is not a directory' % curdir)
                 else:
                     raise ChangesetError('%s have not been found' % curdir)
                 # cache all items from the given traversed tree
                 for item, stat, id in tree.items():
                     if curdir:
                         name = '/'.join((curdir, item))
                     else:
                         name = item
                     self._paths[name] = id
                     self._stat_modes[name] = stat
             if path not in self._paths:
                 raise NodeDoesNotExistError("There is no file nor directory "
                     "at the given path '%s' at revision %s"
-                    % (path, safe_str(self.short_id)))
                     % (path, self.short_id))
         return self._paths[path]
     def _get_kind(self, path):
         obj = self.repository._repo[self._get_id_for_path(path)]
         if isinstance(obj, objects.Blob):
             return NodeKind.FILE
         elif isinstance(obj, objects.Tree):
             return NodeKind.DIR
     def _get_filectx(self, path):
         path = self._fix_path(path)
         if self._get_kind(path) != NodeKind.FILE:
             raise ChangesetError("File does not exist for revision %s at "
                 " '%s'" % (self.raw_id, path))
         return path
     def _get_file_nodes(self):
         return chain(*(t[2] for t in self.walk()))
     @LazyProperty
     def parents(self):
         """
         Returns list of parents changesets.
         """
         return [self.repository.get_changeset(ascii_str(parent_id))
                 for parent_id in self._commit.parents]
     @LazyProperty
     def children(self):
         """
         Returns list of children changesets.
         """
         rev_filter = settings.GIT_REV_FILTER
         so = self.repository.run_git_command(
             ['rev-list', rev_filter, '--children']
+        )
         return [
             self.repository.get_changeset(cs)
             for parts in (l.split(' ') for l in so.splitlines())
             if parts[0] == self.raw_id
             for cs in parts[1:]
+        ]
     def next(self, branch=None):
         if branch and self.branch != branch:
             raise VCSError('Branch option used on changeset not belonging '
                            'to that branch')
         cs = self
         while True:
             try:
                 next_ = cs.revision + 1
                 next_rev = cs.repository.revisions[next_]
             except IndexError:
                 raise ChangesetDoesNotExistError
             cs = cs.repository.get_changeset(next_rev)
             if not branch or branch == cs.branch:
                 return cs
     def prev(self, branch=None):
         if branch and self.branch != branch:
             raise VCSError('Branch option used on changeset not belonging '
                            'to that branch')
         cs = self
         while True:
             try:
                 prev_ = cs.revision - 1
                 if prev_ < 0:
                     raise IndexError
                 prev_rev = cs.repository.revisions[prev_]
             except IndexError:
                 raise ChangesetDoesNotExistError
             cs = cs.repository.get_changeset(prev_rev)
             if not branch or branch == cs.branch:
                 return cs
     def diff(self, ignore_whitespace=True, context=3):
         # Only used to feed diffstat
         rev1 = self.parents[0] if self.parents else self.repository.EMPTY_CHANGESET
         rev2 = self
         return b''.join(self.repository.get_diff(rev1, rev2,
                                     ignore_whitespace=ignore_whitespace,
                                     context=context))
     def get_file_mode(self, path):
         """
         Returns stat mode of the file at the given ``path``.
         """
         # ensure path is traversed
         path = safe_str(path)
         self._get_id_for_path(path)
         return self._stat_modes[path]
     def get_file_content(self, path):
         """
         Returns content of the file at given ``path``.
         """
         id = self._get_id_for_path(path)
         blob = self.repository._repo[id]
         return blob.as_pretty_string()
     def get_file_size(self, path):
         """
         Returns size of the file at given ``path``.
         """
         id = self._get_id_for_path(path)
         blob = self.repository._repo[id]
         return blob.raw_length()
     def get_file_changeset(self, path):
         """
         Returns last commit of the file at the given ``path``.
         """
         return self.get_file_history(path, limit=1)[0]
     def get_file_history(self, path, limit=None):
         """
         Returns history of file as reversed list of ``Changeset`` objects for
         which file at given ``path`` has been modified.
         TODO: This function now uses os underlying 'git' and 'grep' commands
         which is generally not good. Should be replaced with algorithm
         iterating commits.
         """
         self._get_filectx(path)
         f_path = safe_str(path)
         if limit is not None:
             cmd = ['log', '-n', str(safe_int(limit, 0)),
-                   '--pretty=format:%H', '-s', self.raw_id, '--', f_path]
+                   '--pretty=format:%H', '-s', self.raw_id, '--', path]
         else:
             cmd = ['log',
-                   '--pretty=format:%H', '-s', self.raw_id, '--', f_path]
+                   '--pretty=format:%H', '-s', self.raw_id, '--', path]
         so = self.repository.run_git_command(cmd)
         ids = re.findall(r'[0-9a-fA-F]{40}', so)
         return [self.repository.get_changeset(sha) for sha in ids]
     def get_file_history_2(self, path):
         """
         Returns history of file as reversed list of ``Changeset`` objects for
         which file at given ``path`` has been modified.
         """
         self._get_filectx(path)
         from dulwich.walk import Walker
         include = [self.raw_id]
         walker = Walker(self.repository._repo.object_store, include,
                         paths=[path], max_entries=1)
         return [self.repository.get_changeset(ascii_str(x.commit.id.decode))
                 for x in walker]
     def get_file_annotate(self, path):
         """
         Returns a generator of four element tuples with
             lineno, sha, changeset lazy loader and line
         """
         # TODO: This function now uses os underlying 'git' command which is
         # generally not good. Should be replaced with algorithm iterating
         # commits.
         cmd = ['blame', '-l', '--root', '-r', self.raw_id, '--', path]
         # -l     ==> outputs long shas (and we need all 40 characters)
         # --root ==> doesn't put '^' character for boundaries
         # -r sha ==> blames for the given revision
         so = self.repository.run_git_command(cmd)
         for i, blame_line in enumerate(so.split('\n')[:-1]):
             sha, line = re.split(r' ', blame_line, 1)
             yield (i + 1, sha, lambda sha=sha: self.repository.get_changeset(sha), line)
     def fill_archive(self, stream=None, kind='tgz', prefix=None,
                      subrepos=False):
         """
         Fills up given stream.
         :param stream: file like object.
         :param kind: one of following: ``zip``, ``tgz`` or ``tbz2``.
             Default: ``tgz``.
         :param prefix: name of root directory in archive.
             Default is repository name and changeset's raw_id joined with dash
             (``repo-tip.<KIND>``).
         :param subrepos: include subrepos in this archive.
         :raise ImproperArchiveTypeError: If given kind is wrong.
         :raise VcsError: If given stream is None
         """
         allowed_kinds = settings.ARCHIVE_SPECS
         if kind not in allowed_kinds:
             raise ImproperArchiveTypeError('Archive kind not supported use one'
                 'of %s' % ' '.join(allowed_kinds))
         if stream is None:
             raise VCSError('You need to pass in a valid stream for filling'
                            ' with archival data')
         if prefix is None:
             prefix = '%s-%s' % (self.repository.name, self.short_id)
         elif prefix.startswith('/'):
             raise VCSError("Prefix cannot start with leading slash")
         elif prefix.strip() == '':
             raise VCSError("Prefix cannot be empty")
         if kind == 'zip':
             frmt = 'zip'
         else:
             frmt = 'tar'
         _git_path = settings.GIT_EXECUTABLE_PATH
         cmd = '%s archive --format=%s --prefix=%s/ %s' % (_git_path,
                                                 frmt, prefix, self.raw_id)
         if kind == 'tgz':
             cmd += ' | gzip -9'
         elif kind == 'tbz2':
             cmd += ' | bzip2 -9'
         if stream is None:
             raise VCSError('You need to pass in a valid stream for filling'
                            ' with archival data')
         popen = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True,
                       cwd=self.repository.path)
         buffer_size = 1024 * 8
         chunk = popen.stdout.read(buffer_size)
         while chunk:
             stream.write(chunk)
             chunk = popen.stdout.read(buffer_size)
         # Make sure all descriptors would be read
         popen.communicate()
     def get_nodes(self, path):
         """
         Returns combined ``DirNode`` and ``FileNode`` objects list representing
         state of changeset at the given ``path``. If node at the given ``path``
         is not instance of ``DirNode``, ChangesetError would be raised.
         """
         if self._get_kind(path) != NodeKind.DIR:
             raise ChangesetError("Directory does not exist for revision %s at "
                 " '%s'" % (self.revision, path))
         path = self._fix_path(path)
         id = self._get_id_for_path(path)
         tree = self.repository._repo[id]
         dirnodes = []
         filenodes = []
         als = self.repository.alias
         for name, stat, id in tree.items():
             if path != '':
                 obj_path = '/'.join((path, name))
             else:
                 obj_path = name
             if objects.S_ISGITLINK(stat):
                 root_tree = self.repository._repo[self._tree_id]
                 cf = ConfigFile.from_file(BytesIO(self.repository._repo.get_object(root_tree[b'.gitmodules'][1]).data))
                 url = ascii_str(cf.get(('submodule', obj_path), 'url'))
                 dirnodes.append(SubModuleNode(obj_path, url=url, changeset=ascii_str(id),
                                               alias=als))
                 continue
             obj = self.repository._repo.get_object(id)
             if obj_path not in self._stat_modes:
                 self._stat_modes[obj_path] = stat
             if isinstance(obj, objects.Tree):
                 dirnodes.append(DirNode(obj_path, changeset=self))
             elif isinstance(obj, objects.Blob):
                 filenodes.append(FileNode(obj_path, changeset=self, mode=stat))
             else:
                 raise ChangesetError("Requested object should be Tree "
                                      "or Blob, is %r" % type(obj))
         nodes = dirnodes + filenodes
         for node in nodes:
             if node.path not in self.nodes:
                 self.nodes[node.path] = node
         nodes.sort()
         return nodes
     def get_node(self, path):
         """
         Returns ``Node`` object from the given ``path``. If there is no node at
         the given ``path``, ``ChangesetError`` would be raised.
         """
         path = self._fix_path(path)
         if path not in self.nodes:
             try:
                 id_ = self._get_id_for_path(path)
             except ChangesetError:
                 raise NodeDoesNotExistError("Cannot find one of parents' "
                     "directories for a given path: %s" % path)
             _GL = lambda m: m and objects.S_ISGITLINK(m)
             if _GL(self._stat_modes.get(path)):
                 tree = self.repository._repo[self._tree_id]
                 cf = ConfigFile.from_file(BytesIO(self.repository._repo.get_object(tree[b'.gitmodules'][1]).data))
                 url = ascii_str(cf.get(('submodule', path), 'url'))
                 node = SubModuleNode(path, url=url, changeset=ascii_str(id_),
                                      alias=self.repository.alias)
             else:
                 obj = self.repository._repo.get_object(id_)
                 if isinstance(obj, objects.Tree):
                     if path == '':
                         node = RootNode(changeset=self)
                     else:
                         node = DirNode(path, changeset=self)
                     node._tree = obj
                 elif isinstance(obj, objects.Blob):
                     node = FileNode(path, changeset=self)
                     node._blob = obj
                 else:
                     raise NodeDoesNotExistError("There is no file nor directory "
                         "at the given path: '%s' at revision %s"
                         % (path, self.short_id))
             # cache node
             self.nodes[path] = node
         return self.nodes[path]
     @LazyProperty
     def affected_files(self):
         """
         Gets a fast accessible file changes for given changeset
         """
         added, modified, deleted = self._changes_cache
         return list(added.union(modified).union(deleted))
     @LazyProperty
     def _changes_cache(self):
         added = set()
         modified = set()

kallithea/lib/vcs/backends/git/repository.py

➞

Show inline comments

 # -*- coding: utf-8 -*-
 """
     vcs.backends.git.repository
     ~~~~~~~~~~~~~~~~~~~~~~~~~~~
     Git repository implementation.
     :created_on: Apr 8, 2010
     :copyright: (c) 2010-2011 by Marcin Kuzminski, Lukasz Balcerzak.
 """
 import errno
 import logging
 import os
 import re
 import time
 import urllib.error
 import urllib.parse
 import urllib.request
 from collections import OrderedDict
 import mercurial.url  # import httpbasicauthhandler, httpdigestauthhandler
 import mercurial.util  # import url as hg_url
 from dulwich.config import ConfigFile
 from dulwich.objects import Tag
 from dulwich.repo import NotGitRepository, Repo
 from kallithea.lib.vcs import subprocessio
 from kallithea.lib.vcs.backends.base import BaseRepository, CollectionGenerator
 from kallithea.lib.vcs.conf import settings
 from kallithea.lib.vcs.exceptions import (
     BranchDoesNotExistError, ChangesetDoesNotExistError, EmptyRepositoryError, RepositoryError, TagAlreadyExistError, TagDoesNotExistError)
-from kallithea.lib.vcs.utils import ascii_str, date_fromtimestamp, makedate, safe_bytes, safe_str, safe_unicode
 from kallithea.lib.vcs.utils import ascii_str, date_fromtimestamp, makedate, safe_bytes, safe_unicode
 from kallithea.lib.vcs.utils.lazy import LazyProperty
 from kallithea.lib.vcs.utils.paths import abspath, get_user_home
 from .changeset import GitChangeset
 from .inmemory import GitInMemoryChangeset
 from .workdir import GitWorkdir
 SHA_PATTERN = re.compile(r'^([0-9a-fA-F]{12}|[0-9a-fA-F]{40})$')
 log = logging.getLogger(__name__)
 class GitRepository(BaseRepository):
     """
     Git repository backend.
     """
     DEFAULT_BRANCH_NAME = 'master'
     scm = 'git'
     def __init__(self, repo_path, create=False, src_url=None,
                  update_after_clone=False, bare=False):
         self.path = abspath(repo_path)
         self.repo = self._get_repo(create, src_url, update_after_clone, bare)
         self.bare = self.repo.bare
     @property
     def _config_files(self):
         return [
             self.bare and abspath(self.path, 'config')
                       or abspath(self.path, '.git', 'config'),
              abspath(get_user_home(), '.gitconfig'),
+         ]
     @property
     def _repo(self):
         return self.repo
     @property
     def head(self):
         try:
             return self._repo.head()
         except KeyError:
             return None
     @property
     def _empty(self):
         """
         Checks if repository is empty ie. without any changesets
         """
         try:
             self.revisions[0]
         except (KeyError, IndexError):
             return True
         return False
     @LazyProperty
     def revisions(self):
         """
         Returns list of revisions' ids, in ascending order.  Being lazy
         attribute allows external tools to inject shas from cache.
         """
         return self._get_all_revisions()
     @classmethod
     def _run_git_command(cls, cmd, cwd=None):
         """
         Runs given ``cmd`` as git command and returns output bytes in a tuple
         (stdout, stderr) ... or raise RepositoryError.
         :param cmd: git command to be executed
         :param cwd: passed directly to subprocess
         """
         # need to clean fix GIT_DIR !
         gitenv = dict(os.environ)
         gitenv.pop('GIT_DIR', None)
         gitenv['GIT_CONFIG_NOGLOBAL'] = '1'
         assert isinstance(cmd, list), cmd
         cmd = [settings.GIT_EXECUTABLE_PATH, '-c', 'core.quotepath=false'] + cmd
         try:
             p = subprocessio.SubprocessIOChunker(cmd, cwd=cwd, env=gitenv, shell=False)
         except (EnvironmentError, OSError) as err:
             # output from the failing process is in str(EnvironmentError)
             msg = ("Couldn't run git command %s.\n"
                    "Subprocess failed with '%s': %s\n" %
                    (cmd, type(err).__name__, err)
             ).strip()
             log.error(msg)
             raise RepositoryError(msg)
         try:
             stdout = b''.join(p.output)
             stderr = b''.join(p.error)
         finally:
             p.close()
         # TODO: introduce option to make commands fail if they have any stderr output?
         if stderr:
             log.debug('stderr from %s:\n%s', cmd, stderr)
         else:
             log.debug('stderr from %s: None', cmd)
         return stdout, stderr
     def run_git_command(self, cmd):
         """
         Runs given ``cmd`` as git command with cwd set to current repo.
         Returns stdout as unicode str ... or raise RepositoryError.
         """
         cwd = None
         if os.path.isdir(self.path):
             cwd = self.path
         stdout, _stderr = self._run_git_command(cmd, cwd=cwd)
         return safe_unicode(stdout)
     @classmethod
     def _check_url(cls, url):
         """
         Function will check given url and try to verify if it's a valid
         link. Sometimes it may happened that git will issue basic
         auth request that can cause whole API to hang when used from python
         or other external calls.
         On failures it'll raise urllib2.HTTPError, exception is also thrown
         when the return code is non 200
         """
         # check first if it's not an local url
         if os.path.isdir(url) or url.startswith('file:'):
             return True
         if url.startswith('git://'):
             return True
         if '+' in url[:url.find('://')]:
             url = url[url.find('+') + 1:]
         handlers = []
         url_obj = mercurial.util.url(safe_bytes(url))
         test_uri, authinfo = url_obj.authinfo()
         if not test_uri.endswith('info/refs'):
             test_uri = test_uri.rstrip('/') + '/info/refs'
         url_obj.passwd = b'*****'
         cleaned_uri = str(url_obj)
         if authinfo:
             # create a password manager
             passmgr = urllib.request.HTTPPasswordMgrWithDefaultRealm()
             passmgr.add_password(*authinfo)
             handlers.extend((mercurial.url.httpbasicauthhandler(passmgr),
                              mercurial.url.httpdigestauthhandler(passmgr)))
         o = urllib.request.build_opener(*handlers)
         o.addheaders = [('User-Agent', 'git/1.7.8.0')]  # fake some git
         req = urllib.request.Request(
             "%s?%s" % (
                 test_uri,
                 urllib.parse.urlencode({"service": 'git-upload-pack'})
             ))
         try:
             resp = o.open(req)
             if resp.code != 200:
                 raise Exception('Return Code is not 200')
         except Exception as e:
             # means it cannot be cloned
             raise urllib.error.URLError("[%s] org_exc: %s" % (cleaned_uri, e))
         # now detect if it's proper git repo
         gitdata = resp.read()
         if 'service=git-upload-pack' not in gitdata:
             raise urllib.error.URLError(
                 "url [%s] does not look like an git" % cleaned_uri)
         return True
     def _get_repo(self, create, src_url=None, update_after_clone=False,
                   bare=False):
         if create and os.path.exists(self.path):
             raise RepositoryError("Location already exist")
         if src_url and not create:
             raise RepositoryError("Create should be set to True if src_url is "
                                   "given (clone operation creates repository)")
         try:
             if create and src_url:
                 GitRepository._check_url(src_url)
                 self.clone(src_url, update_after_clone, bare)
                 return Repo(self.path)
             elif create:
                 os.makedirs(self.path)
                 if bare:
                     return Repo.init_bare(self.path)
                 else:
                     return Repo.init(self.path)
             else:
                 return Repo(self.path)
         except (NotGitRepository, OSError) as err:
             raise RepositoryError(err)
     def _get_all_revisions(self):
         # we must check if this repo is not empty, since later command
         # fails if it is. And it's cheaper to ask than throw the subprocess
         # errors
         try:
             self._repo.head()
         except KeyError:
             return []
         rev_filter = settings.GIT_REV_FILTER
         cmd = ['rev-list', rev_filter, '--reverse', '--date-order']
         try:
             so = self.run_git_command(cmd)
         except RepositoryError:
             # Can be raised for empty repositories
             return []
         return so.splitlines()
     def _get_all_revisions2(self):
         # alternate implementation using dulwich
         includes = [ascii_str(sha) for key, (sha, type_) in self._parsed_refs.items()
                     if type_ != b'T']
         return [c.commit.id for c in self._repo.get_walker(include=includes)]
     def _get_revision(self, revision):
         """
         Given any revision identifier, returns a 40 char string with revision hash.
         """
         if self._empty:
             raise EmptyRepositoryError("There are no changesets yet")
         if revision in (None, '', 'tip', 'HEAD', 'head', -1):
             revision = -1
         if isinstance(revision, int):
             try:
                 return self.revisions[revision]
             except IndexError:
                 msg = "Revision %r does not exist for %s" % (revision, self.name)
                 raise ChangesetDoesNotExistError(msg)
         if isinstance(revision, (str, unicode)):
             if revision.isdigit() and (len(revision) < 12 or len(revision) == revision.count('0')):
                 try:
                     return self.revisions[int(revision)]
                 except IndexError:
                     msg = "Revision %r does not exist for %s" % (revision, self)
                     raise ChangesetDoesNotExistError(msg)
             # get by branch/tag name
             _ref_revision = self._parsed_refs.get(safe_bytes(revision))
             if _ref_revision:  # and _ref_revision[1] in [b'H', b'RH', b'T']:
                 return ascii_str(_ref_revision[0])
             if revision in self.revisions:
                 return revision
             # maybe it's a tag ? we don't have them in self.revisions
             if revision in self.tags.values():
                 return revision
             if SHA_PATTERN.match(revision):
                 msg = "Revision %r does not exist for %s" % (revision, self.name)
                 raise ChangesetDoesNotExistError(msg)
         raise ChangesetDoesNotExistError("Given revision %r not recognized" % revision)
     def get_ref_revision(self, ref_type, ref_name):
         """
         Returns ``GitChangeset`` object representing repository's
         changeset at the given ``revision``.
         """
         return self._get_revision(ref_name)
     def _get_archives(self, archive_name='tip'):
         for i in [('zip', '.zip'), ('gz', '.tar.gz'), ('bz2', '.tar.bz2')]:
             yield {"type": i[0], "extension": i[1], "node": archive_name}
     def _get_url(self, url):
         """
         Returns normalized url. If schema is not given, would fall to
         filesystem (``file:///``) schema.
         """
         url = safe_str(url)
         if url != 'default' and '://' not in url:
             url = ':///'.join(('file', url))
         return url
     @LazyProperty
     def name(self):
         return os.path.basename(self.path)
     @LazyProperty
     def last_change(self):
         """
         Returns last change made on this repository as datetime object
         """
         return date_fromtimestamp(self._get_mtime(), makedate()[1])
     def _get_mtime(self):
         try:
             return time.mktime(self.get_changeset().date.timetuple())
         except RepositoryError:
             idx_loc = '' if self.bare else '.git'
             # fallback to filesystem
             in_path = os.path.join(self.path, idx_loc, "index")
             he_path = os.path.join(self.path, idx_loc, "HEAD")
             if os.path.exists(in_path):
                 return os.stat(in_path).st_mtime
             else:
                 return os.stat(he_path).st_mtime
     @LazyProperty
     def description(self):
         return safe_unicode(self._repo.get_description() or b'unknown')
     @LazyProperty
     def contact(self):
         undefined_contact = u'Unknown'
         return undefined_contact
     @property
     def branches(self):
         if not self.revisions:
             return {}
         sortkey = lambda ctx: ctx[0]
         _branches = [(key, ascii_str(sha))
                      for key, (sha, type_) in self._parsed_refs.items() if type_ == b'H']
         return OrderedDict(sorted(_branches, key=sortkey, reverse=False))
     @LazyProperty
     def closed_branches(self):
         return {}
     @LazyProperty
     def tags(self):
         return self._get_tags()
     def _get_tags(self):
         if not self.revisions:
             return {}
         sortkey = lambda ctx: ctx[0]
         _tags = [(key, ascii_str(sha))
                  for key, (sha, type_) in self._parsed_refs.items() if type_ == b'T']
         return OrderedDict(sorted(_tags, key=sortkey, reverse=True))
     def tag(self, name, user, revision=None, message=None, date=None,
             **kwargs):
         """
         Creates and returns a tag for the given ``revision``.
         :param name: name for new tag
         :param user: full username, i.e.: "Joe Doe <joe.doe@example.com>"
         :param revision: changeset id for which new tag would be created
         :param message: message of the tag's commit
         :param date: date of tag's commit
         :raises TagAlreadyExistError: if tag with same name already exists
         """
         if name in self.tags:
             raise TagAlreadyExistError("Tag %s already exists" % name)
         changeset = self.get_changeset(revision)
         message = message or "Added tag %s for commit %s" % (name,
             changeset.raw_id)
         self._repo.refs[b"refs/tags/%s" % safe_bytes(name)] = changeset._commit.id
         self._parsed_refs = self._get_parsed_refs()
         self.tags = self._get_tags()
         return changeset
     def remove_tag(self, name, user, message=None, date=None):
         """
         Removes tag with the given ``name``.
         :param name: name of the tag to be removed
         :param user: full username, i.e.: "Joe Doe <joe.doe@example.com>"
         :param message: message of the tag's removal commit
         :param date: date of tag's removal commit
         :raises TagDoesNotExistError: if tag with given name does not exists
         """
         if name not in self.tags:
             raise TagDoesNotExistError("Tag %s does not exist" % name)
         # self._repo.refs is a DiskRefsContainer, and .path gives the full absolute path of '.git'
         tagpath = os.path.join(self._repo.refs.path, 'refs', 'tags', name)
         try:
             os.remove(tagpath)
             self._parsed_refs = self._get_parsed_refs()
             self.tags = self._get_tags()
         except OSError as e:
             raise RepositoryError(e.strerror)
     @LazyProperty
     def bookmarks(self):
         """
         Gets bookmarks for this repository
         """
         return {}
     @LazyProperty
     def _parsed_refs(self):
         return self._get_parsed_refs()
     def _get_parsed_refs(self):
         """Return refs as a dict, like:
         { b'v0.2.0': [b'599ba911aa24d2981225f3966eb659dfae9e9f30', b'T'] }
         """
         _repo = self._repo
         refs = _repo.get_refs()
         keys = [(b'refs/heads/', b'H'),
                 (b'refs/remotes/origin/', b'RH'),
                 (b'refs/tags/', b'T')]
         _refs = {}
         for ref, sha in refs.items():
             for k, type_ in keys:
                 if ref.startswith(k):
                     _key = ref[len(k):]
                     if type_ == b'T':
                         obj = _repo.get_object(sha)
                         if isinstance(obj, Tag):
                             sha = _repo.get_object(sha).object[1]
                     _refs[_key] = [sha, type_]
                     break
         return _refs
     def _heads(self, reverse=False):
         refs = self._repo.get_refs()
         heads = {}
         for key, val in refs.items():
             for ref_key in [b'refs/heads/', b'refs/remotes/origin/']:
                 if key.startswith(ref_key):
                     n = key[len(ref_key):]
                     if n not in [b'HEAD']:
                         heads[n] = val
         return heads if reverse else dict((y, x) for x, y in heads.items())
     def get_changeset(self, revision=None):
         """
         Returns ``GitChangeset`` object representing commit from git repository
         at the given revision or head (most recent commit) if None given.
         """
         if isinstance(revision, GitChangeset):
             return revision
         return GitChangeset(repository=self, revision=self._get_revision(revision))
     def get_changesets(self, start=None, end=None, start_date=None,
            end_date=None, branch_name=None, reverse=False, max_revisions=None):
         """
         Returns iterator of ``GitChangeset`` objects from start to end (both
         are inclusive), in ascending date order (unless ``reverse`` is set).
         :param start: changeset ID, as str; first returned changeset
         :param end: changeset ID, as str; last returned changeset
         :param start_date: if specified, changesets with commit date less than
           ``start_date`` would be filtered out from returned set
         :param end_date: if specified, changesets with commit date greater than
           ``end_date`` would be filtered out from returned set
         :param branch_name: if specified, changesets not reachable from given
           branch would be filtered out from returned set
         :param reverse: if ``True``, returned generator would be reversed
           (meaning that returned changesets would have descending date order)
         :raise BranchDoesNotExistError: If given ``branch_name`` does not
             exist.
         :raise ChangesetDoesNotExistError: If changeset for given ``start`` or
           ``end`` could not be found.
         """
         if branch_name and branch_name not in self.branches:
             raise BranchDoesNotExistError("Branch '%s' not found"
                                           % branch_name)
         # actually we should check now if it's not an empty repo to not spaw
         # subprocess commands

kallithea/lib/vcs/backends/git/ssh.py

➞

Show inline comments

 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
+#
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 import logging
 import os
 from kallithea.lib.hooks import log_pull_action
 from kallithea.lib.utils import make_ui
 from kallithea.lib.vcs.backends.ssh import BaseSshHandler
 from kallithea.lib.vcs.utils import safe_str
 log = logging.getLogger(__name__)
 class GitSshHandler(BaseSshHandler):
     vcs_type = 'git'
     @classmethod
     def make(cls, ssh_command_parts):
         r"""
         >>> import shlex
         >>> GitSshHandler.make(shlex.split("git-upload-pack '/foo bar'")).repo_name
         u'foo bar'
         >>> GitSshHandler.make(shlex.split("git-upload-pack '/foo bar'")).verb
         'git-upload-pack'
         >>> GitSshHandler.make(shlex.split(" git-upload-pack /blåbærgrød ")).repo_name # might not be necessary to support no quoting ... but we can
         u'bl\xe5b\xe6rgr\xf8d'
         >>> GitSshHandler.make(shlex.split('''git-upload-pack "/foo'bar"''')).repo_name
         u"foo'bar"
         >>> GitSshHandler.make(shlex.split("git-receive-pack '/foo'")).repo_name
         u'foo'
         >>> GitSshHandler.make(shlex.split("git-receive-pack '/foo'")).verb
         'git-receive-pack'
         >>> GitSshHandler.make(shlex.split("/bin/git-upload-pack '/foo'")) # ssh-serve will report 'SSH command %r is not supported'
         >>> GitSshHandler.make(shlex.split('''git-upload-pack /foo bar''')) # ssh-serve will report 'SSH command %r is not supported'
         >>> shlex.split("git-upload-pack '/foo'bar' x") # ssh-serve will report: Error parsing SSH command "...": No closing quotation
         Traceback (most recent call last):
         ValueError: No closing quotation
         >>> GitSshHandler.make(shlex.split('hg -R foo serve --stdio')) # not handled here
         """
         if (len(ssh_command_parts) == 2 and
             ssh_command_parts[0] in ['git-upload-pack', 'git-receive-pack'] and
             ssh_command_parts[1].startswith('/')
         ):
             return cls(ssh_command_parts[1][1:], ssh_command_parts[0])
         return None
     def __init__(self, repo_name, verb):
         BaseSshHandler.__init__(self, repo_name)
         self.verb = verb
     def _serve(self):
         if self.verb == 'git-upload-pack': # action 'pull'
             # base class called set_hook_environment - action is hardcoded to 'pull'
             log_pull_action(ui=make_ui(), repo=self.db_repo.scm_instance._repo)
         else: # probably verb 'git-receive-pack', action 'push'
             if not self.allow_push:
-                self.exit('Push access to %r denied' % safe_str(self.repo_name))
                 self.exit('Push access to %r denied' % self.repo_name)
             # Note: push logging is handled by Git post-receive hook
         # git shell is not a real shell but use shell inspired quoting *inside* the argument.
         # Per https://github.com/git/git/blob/v2.22.0/quote.c#L12 :
         # The path must be "'" quoted, but "'" and "!" must exit the quoting and be "\" escaped
         quoted_abspath = "'%s'" % self.db_repo.repo_full_path.replace("'", r"'\''").replace("!", r"'\!'")
         newcmd = ['git', 'shell', '-c', "%s %s" % (self.verb, quoted_abspath)]
         log.debug('Serving: %s', newcmd)
         os.execvp(newcmd[0], newcmd)
         self.exit("Failed to exec 'git' as %s" % newcmd)

kallithea/lib/vcs/backends/hg/changeset.py

➞

Show inline comments

 import os
 import posixpath
 import mercurial.archival
 import mercurial.node
 import mercurial.obsutil
 from kallithea.lib.vcs.backends.base import BaseChangeset
 from kallithea.lib.vcs.conf import settings
 from kallithea.lib.vcs.exceptions import ChangesetDoesNotExistError, ChangesetError, ImproperArchiveTypeError, NodeDoesNotExistError, VCSError
 from kallithea.lib.vcs.nodes import (
     AddedFileNodesGenerator, ChangedFileNodesGenerator, DirNode, FileNode, NodeKind, RemovedFileNodesGenerator, RootNode, SubModuleNode)
-from kallithea.lib.vcs.utils import ascii_bytes, ascii_str, date_fromtimestamp, safe_bytes, safe_str, safe_unicode
 from kallithea.lib.vcs.utils import ascii_bytes, ascii_str, date_fromtimestamp, safe_bytes, safe_unicode
 from kallithea.lib.vcs.utils.lazy import LazyProperty
 from kallithea.lib.vcs.utils.paths import get_dirs_for_path
 class MercurialChangeset(BaseChangeset):
     """
     Represents state of the repository at a revision.
     """
     def __init__(self, repository, revision):
         self.repository = repository
         assert isinstance(revision, str), repr(revision)
         self._ctx = repository._repo[ascii_bytes(revision)]
         self.raw_id = ascii_str(self._ctx.hex())
         self.revision = self._ctx._rev
         self.nodes = {}
     @LazyProperty
     def tags(self):
         return [safe_unicode(tag) for tag in self._ctx.tags()]
     @LazyProperty
     def branch(self):
         return safe_unicode(self._ctx.branch())
     @LazyProperty
     def branches(self):
         return [safe_unicode(self._ctx.branch())]
     @LazyProperty
     def closesbranch(self):
         return self._ctx.closesbranch()
     @LazyProperty
     def obsolete(self):
         return self._ctx.obsolete()
     @LazyProperty
     def bumped(self):
         return self._ctx.phasedivergent()
     @LazyProperty
     def divergent(self):
         return self._ctx.contentdivergent()
     @LazyProperty
     def extinct(self):
         return self._ctx.extinct()
     @LazyProperty
     def unstable(self):
         return self._ctx.orphan()
     @LazyProperty
     def phase(self):
         if(self._ctx.phase() == 1):
             return 'Draft'
         elif(self._ctx.phase() == 2):
             return 'Secret'
         else:
             return ''
     @LazyProperty
     def successors(self):
         successors = mercurial.obsutil.successorssets(self._ctx._repo, self._ctx.node(), closest=True)
         if successors:
             # flatten the list here handles both divergent (len > 1)
             # and the usual case (len = 1)
             successors = [mercurial.node.hex(n)[:12] for sub in successors for n in sub if n != self._ctx.node()]
         return successors
     @LazyProperty
     def predecessors(self):
         return [mercurial.node.hex(n)[:12] for n in mercurial.obsutil.closestpredecessors(self._ctx._repo, self._ctx.node())]
     @LazyProperty
     def bookmarks(self):
         return [safe_unicode(bookmark) for bookmark in self._ctx.bookmarks()]
     @LazyProperty
     def message(self):
         return safe_unicode(self._ctx.description())
     @LazyProperty
     def committer(self):
         return safe_unicode(self.author)
     @LazyProperty
     def author(self):
         return safe_unicode(self._ctx.user())
     @LazyProperty
     def date(self):
         return date_fromtimestamp(*self._ctx.date())
     @LazyProperty
     def _timestamp(self):
         return self._ctx.date()[0]
     @LazyProperty
     def status(self):
         """
         Returns modified, added, removed, deleted files for current changeset
         """
         return self.repository._repo.status(self._ctx.p1().node(),
                                             self._ctx.node())
     @LazyProperty
     def _file_paths(self):
         return list(self._ctx)
     @LazyProperty
     def _dir_paths(self):
         p = list(set(get_dirs_for_path(*self._file_paths)))
         p.insert(0, '')
         return p
     @LazyProperty
     def _paths(self):
         return self._dir_paths + self._file_paths
     @LazyProperty
     def short_id(self):
         return self.raw_id[:12]
     @LazyProperty
     def parents(self):
         """
         Returns list of parents changesets.
         """
         return [self.repository.get_changeset(parent.rev())
                 for parent in self._ctx.parents() if parent.rev() >= 0]
     @LazyProperty
     def children(self):
         """
         Returns list of children changesets.
         """
         return [self.repository.get_changeset(child.rev())
                 for child in self._ctx.children() if child.rev() >= 0]
     def next(self, branch=None):
         if branch and self.branch != branch:
             raise VCSError('Branch option used on changeset not belonging '
                            'to that branch')
         cs = self
         while True:
             try:
                 next_ = cs.repository.revisions.index(cs.raw_id) + 1
                 next_rev = cs.repository.revisions[next_]
             except IndexError:
                 raise ChangesetDoesNotExistError
             cs = cs.repository.get_changeset(next_rev)
             if not branch or branch == cs.branch:
                 return cs
     def prev(self, branch=None):
         if branch and self.branch != branch:
             raise VCSError('Branch option used on changeset not belonging '
                            'to that branch')
         cs = self
         while True:
             try:
                 prev_ = cs.repository.revisions.index(cs.raw_id) - 1
                 if prev_ < 0:
                     raise IndexError
                 prev_rev = cs.repository.revisions[prev_]
             except IndexError:
                 raise ChangesetDoesNotExistError
             cs = cs.repository.get_changeset(prev_rev)
             if not branch or branch == cs.branch:
                 return cs
     def diff(self):
         # Only used to feed diffstat
         return b''.join(self._ctx.diff())
     def _fix_path(self, path):
         """
         Paths are stored without trailing slash so we need to get rid off it if
         needed. Also mercurial keeps filenodes as str so we need to decode
         from unicode to str
         """
         if path.endswith('/'):
             path = path.rstrip('/')
-        return safe_str(path)
         return path
     def _get_kind(self, path):
         path = self._fix_path(path)
         if path in self._file_paths:
             return NodeKind.FILE
         elif path in self._dir_paths:
             return NodeKind.DIR
         else:
             raise ChangesetError("Node does not exist at the given path '%s'"
                 % (path))
     def _get_filectx(self, path):
         path = self._fix_path(path)
         if self._get_kind(path) != NodeKind.FILE:
             raise ChangesetError("File does not exist for revision %s at "
                 " '%s'" % (self.raw_id, path))
         return self._ctx.filectx(safe_bytes(path))
     def _extract_submodules(self):
         """
         returns a dictionary with submodule information from substate file
         of hg repository
         """
         return self._ctx.substate
     def get_file_mode(self, path):
         """
         Returns stat mode of the file at the given ``path``.
         """
         fctx = self._get_filectx(path)
         if b'x' in fctx.flags():
             return 0o100755
         else:
             return 0o100644
     def get_file_content(self, path):
         """
         Returns content of the file at given ``path``.
         """
         fctx = self._get_filectx(path)
         return fctx.data()
     def get_file_size(self, path):
         """
         Returns size of the file at given ``path``.
         """
         fctx = self._get_filectx(path)
         return fctx.size()
     def get_file_changeset(self, path):
         """
         Returns last commit of the file at the given ``path``.
         """
         return self.get_file_history(path, limit=1)[0]
     def get_file_history(self, path, limit=None):
         """
         Returns history of file as reversed list of ``Changeset`` objects for
         which file at given ``path`` has been modified.
         """
         fctx = self._get_filectx(path)
         hist = []
         cnt = 0
         for cs in reversed([x for x in fctx.filelog()]):
             cnt += 1
             hist.append(mercurial.node.hex(fctx.filectx(cs).node()))
             if limit is not None and cnt == limit:
                 break
         return [self.repository.get_changeset(node) for node in hist]
     def get_file_annotate(self, path):
         """
         Returns a generator of four element tuples with
             lineno, sha, changeset lazy loader and line
         """
         annotations = self._get_filectx(path).annotate()
         annotation_lines = [(annotateline.fctx, annotateline.text) for annotateline in annotations]
         for i, (fctx, line) in enumerate(annotation_lines):
             sha = ascii_str(fctx.hex())
             yield (i + 1, sha, lambda sha=sha: self.repository.get_changeset(sha), line)
     def fill_archive(self, stream=None, kind='tgz', prefix=None,
                      subrepos=False):
         """
         Fills up given stream.
         :param stream: file like object.
         :param kind: one of following: ``zip``, ``tgz`` or ``tbz2``.
             Default: ``tgz``.
         :param prefix: name of root directory in archive.
             Default is repository name and changeset's raw_id joined with dash
             (``repo-tip.<KIND>``).
         :param subrepos: include subrepos in this archive.
         :raise ImproperArchiveTypeError: If given kind is wrong.
         :raise VcsError: If given stream is None
         """
         allowed_kinds = settings.ARCHIVE_SPECS
         if kind not in allowed_kinds:
             raise ImproperArchiveTypeError('Archive kind not supported use one'
                 'of %s' % ' '.join(allowed_kinds))
         if stream is None:
             raise VCSError('You need to pass in a valid stream for filling'
                            ' with archival data')
         if prefix is None:
             prefix = '%s-%s' % (self.repository.name, self.short_id)
         elif prefix.startswith('/'):
             raise VCSError("Prefix cannot start with leading slash")
         elif prefix.strip() == '':
             raise VCSError("Prefix cannot be empty")
         mercurial.archival.archive(self.repository._repo, stream, ascii_bytes(self.raw_id),
                          safe_bytes(kind), prefix=safe_bytes(prefix), subrepos=subrepos)
     def get_nodes(self, path):
         """
         Returns combined ``DirNode`` and ``FileNode`` objects list representing
         state of changeset at the given ``path``. If node at the given ``path``
         is not instance of ``DirNode``, ChangesetError would be raised.
         """
         if self._get_kind(path) != NodeKind.DIR:
             raise ChangesetError("Directory does not exist for revision %s at "
                 " '%s'" % (self.revision, path))
         path = self._fix_path(path)
         filenodes = [FileNode(f, changeset=self) for f in self._file_paths
             if os.path.dirname(f) == path]
         dirs = path == '' and '' or [d for d in self._dir_paths
             if d and posixpath.dirname(d) == path]
         dirnodes = [DirNode(d, changeset=self) for d in dirs
             if os.path.dirname(d) == path]
         als = self.repository.alias
         for k, vals in self._extract_submodules().items():
             #vals = url,rev,type
             loc = vals[0]
             cs = vals[1]
             dirnodes.append(SubModuleNode(k, url=loc, changeset=cs,
                                           alias=als))
         nodes = dirnodes + filenodes
         for node in nodes:
             self.nodes[node.path] = node
         nodes.sort()
         return nodes
     def get_node(self, path):
         """
         Returns ``Node`` object from the given ``path``. If there is no node at
         the given ``path``, ``ChangesetError`` would be raised.
         """
         path = self._fix_path(path)
         if path not in self.nodes:
             if path in self._file_paths:
                 node = FileNode(path, changeset=self)
             elif path in self._dir_paths or path in self._dir_paths:
                 if path == '':
                     node = RootNode(changeset=self)
                 else:
                     node = DirNode(path, changeset=self)
             else:
                 raise NodeDoesNotExistError("There is no file nor directory "
                     "at the given path: '%s' at revision %s"
                     % (path, self.short_id))
             # cache node
             self.nodes[path] = node
         return self.nodes[path]
     @LazyProperty
     def affected_files(self):
         """
         Gets a fast accessible file changes for given changeset
         """
         return self._ctx.files()
     @property
     def added(self):
         """
         Returns list of added ``FileNode`` objects.
         """
         return AddedFileNodesGenerator([n for n in self.status.added], self)
     @property
     def changed(self):
         """
         Returns list of modified ``FileNode`` objects.
         """
         return ChangedFileNodesGenerator([n for n in self.status.modified], self)

kallithea/lib/vcs/backends/hg/repository.py

➞

Show inline comments

 # -*- coding: utf-8 -*-
 """
     vcs.backends.hg.repository
     ~~~~~~~~~~~~~~~~~~~~~~~~~~
     Mercurial repository implementation.
     :created_on: Apr 8, 2010
     :copyright: (c) 2010-2011 by Marcin Kuzminski, Lukasz Balcerzak.
 """
 import datetime
 import logging
 import os
 import time
 import urllib.error
 import urllib.parse
 import urllib.request
 from collections import OrderedDict
 import mercurial.commands
 import mercurial.error
 import mercurial.exchange
 import mercurial.hg
 import mercurial.hgweb
 import mercurial.httppeer
 import mercurial.localrepo
 import mercurial.match
 import mercurial.mdiff
 import mercurial.node
 import mercurial.patch
 import mercurial.scmutil
 import mercurial.sshpeer
 import mercurial.tags
 import mercurial.ui
 import mercurial.url
 import mercurial.util
 from kallithea.lib.vcs.backends.base import BaseRepository, CollectionGenerator
 from kallithea.lib.vcs.exceptions import (
     BranchDoesNotExistError, ChangesetDoesNotExistError, EmptyRepositoryError, RepositoryError, TagAlreadyExistError, TagDoesNotExistError, VCSError)
-from kallithea.lib.vcs.utils import ascii_str, author_email, author_name, date_fromtimestamp, makedate, safe_bytes, safe_str, safe_unicode
 from kallithea.lib.vcs.utils import ascii_str, author_email, author_name, date_fromtimestamp, makedate, safe_bytes, safe_unicode
 from kallithea.lib.vcs.utils.lazy import LazyProperty
 from kallithea.lib.vcs.utils.paths import abspath
 from .changeset import MercurialChangeset
 from .inmemory import MercurialInMemoryChangeset
 from .workdir import MercurialWorkdir
 log = logging.getLogger(__name__)
 class MercurialRepository(BaseRepository):
     """
     Mercurial repository backend
     """
     DEFAULT_BRANCH_NAME = 'default'
     scm = 'hg'
     def __init__(self, repo_path, create=False, baseui=None, src_url=None,
                  update_after_clone=False):
         """
         Raises RepositoryError if repository could not be find at the given
         ``repo_path``.
         :param repo_path: local path of the repository
         :param create=False: if set to True, would try to create repository if
            it does not exist rather than raising exception
         :param baseui=None: user data
         :param src_url=None: would try to clone repository from given location
         :param update_after_clone=False: sets update of working copy after
           making a clone
         """
         if not isinstance(repo_path, str):
             raise VCSError('Mercurial backend requires repository path to '
                            'be instance of <str> got %s instead' %
                            type(repo_path))
         self.path = abspath(repo_path)
         self.baseui = baseui or mercurial.ui.ui()
         # We've set path and ui, now we can set _repo itself
         self._repo = self._get_repo(create, src_url, update_after_clone)
     @property
     def _empty(self):
         """
         Checks if repository is empty ie. without any changesets
         """
         # TODO: Following raises errors when using InMemoryChangeset...
         # return len(self._repo.changelog) == 0
         return len(self.revisions) == 0
     @LazyProperty
     def revisions(self):
         """
         Returns list of revisions' ids, in ascending order.  Being lazy
         attribute allows external tools to inject shas from cache.
         """
         return self._get_all_revisions()
     @LazyProperty
     def name(self):
         return os.path.basename(self.path)
     @LazyProperty
     def branches(self):
         return self._get_branches()
     @LazyProperty
     def closed_branches(self):
         return self._get_branches(normal=False, closed=True)
     @LazyProperty
     def allbranches(self):
         """
         List all branches, including closed branches.
         """
         return self._get_branches(closed=True)
     def _get_branches(self, normal=True, closed=False):
         """
         Gets branches for this repository
         Returns only not closed branches by default
         :param closed: return also closed branches for mercurial
         :param normal: return also normal branches
         """
         if self._empty:
             return {}
         bt = OrderedDict()
         for bn, _heads, node, isclosed in sorted(self._repo.branchmap().iterbranches()):
             if isclosed:
                 if closed:
                     bt[safe_unicode(bn)] = ascii_str(mercurial.node.hex(node))
             else:
                 if normal:
                     bt[safe_unicode(bn)] = ascii_str(mercurial.node.hex(node))
         return bt
     @LazyProperty
     def tags(self):
         """
         Gets tags for this repository
         """
         return self._get_tags()
     def _get_tags(self):
         if self._empty:
             return {}
         return OrderedDict(sorted(
             ((safe_unicode(n), ascii_str(mercurial.node.hex(h))) for n, h in self._repo.tags().items()),
             reverse=True,
             key=lambda x: x[0],  # sort by name
         ))
     def tag(self, name, user, revision=None, message=None, date=None,
             **kwargs):
         """
         Creates and returns a tag for the given ``revision``.
         :param name: name for new tag
         :param user: full username, i.e.: "Joe Doe <joe.doe@example.com>"
         :param revision: changeset id for which new tag would be created
         :param message: message of the tag's commit
         :param date: date of tag's commit
         :raises TagAlreadyExistError: if tag with same name already exists
         """
         if name in self.tags:
             raise TagAlreadyExistError("Tag %s already exists" % name)
         changeset = self.get_changeset(revision)
         local = kwargs.setdefault('local', False)
         if message is None:
             message = "Added tag %s for changeset %s" % (name,
                 changeset.short_id)
         if date is None:
             date = safe_bytes(datetime.datetime.now().strftime('%a, %d %b %Y %H:%M:%S'))
         try:
             mercurial.tags.tag(self._repo, safe_bytes(name), changeset._ctx.node(), safe_bytes(message), local, safe_bytes(user), date)
         except mercurial.error.Abort as e:
             raise RepositoryError(e.args[0])
         # Reinitialize tags
         self.tags = self._get_tags()
         tag_id = self.tags[name]
         return self.get_changeset(revision=tag_id)
     def remove_tag(self, name, user, message=None, date=None):
         """
         Removes tag with the given ``name``.
         :param name: name of the tag to be removed
         :param user: full username, i.e.: "Joe Doe <joe.doe@example.com>"
         :param message: message of the tag's removal commit
         :param date: date of tag's removal commit
         :raises TagDoesNotExistError: if tag with given name does not exists
         """
         if name not in self.tags:
             raise TagDoesNotExistError("Tag %s does not exist" % name)
         if message is None:
             message = "Removed tag %s" % name
         if date is None:
             date = safe_bytes(datetime.datetime.now().strftime('%a, %d %b %Y %H:%M:%S'))
         local = False
         try:
             mercurial.tags.tag(self._repo, safe_bytes(name), mercurial.commands.nullid, safe_bytes(message), local, safe_bytes(user), date)
             self.tags = self._get_tags()
         except mercurial.error.Abort as e:
             raise RepositoryError(e.args[0])
     @LazyProperty
     def bookmarks(self):
         """
         Gets bookmarks for this repository
         """
         return self._get_bookmarks()
     def _get_bookmarks(self):
         if self._empty:
             return {}
         return OrderedDict(sorted(
             ((safe_unicode(n), ascii_str(h)) for n, h in self._repo._bookmarks.items()),
             reverse=True,
@@ @@ -257,363 +257,360 @@ class MercurialRepository(BaseRepository @@
         # Negative context values make no sense, and will result in
         # errors. Ensure this does not happen.
         if context < 0:
             context = 0
         if hasattr(rev1, 'raw_id'):
             rev1 = getattr(rev1, 'raw_id')
         if hasattr(rev2, 'raw_id'):
             rev2 = getattr(rev2, 'raw_id')
         # Check if given revisions are present at repository (may raise
         # ChangesetDoesNotExistError)
         if rev1 != self.EMPTY_CHANGESET:
             self.get_changeset(rev1)
         self.get_changeset(rev2)
         if path:
             file_filter = mercurial.match.exact(path)
         else:
             file_filter = None
         return b''.join(mercurial.patch.diff(self._repo, rev1, rev2, match=file_filter,
                           opts=mercurial.mdiff.diffopts(git=True,
                                         showfunc=True,
                                         ignorews=ignore_whitespace,
                                         context=context)))
     @classmethod
     def _check_url(cls, url, repoui=None):
         """
         Function will check given url and try to verify if it's a valid
         link. Sometimes it may happened that mercurial will issue basic
         auth request that can cause whole API to hang when used from python
         or other external calls.
         On failures it'll raise urllib2.HTTPError, exception is also thrown
         when the return code is non 200
         """
         # check first if it's not an local url
         if os.path.isdir(url) or url.startswith(b'file:'):
             return True
         if url.startswith(b'ssh:'):
             # in case of invalid uri or authentication issues, sshpeer will
             # throw an exception.
             mercurial.sshpeer.instance(repoui or mercurial.ui.ui(), url, False).lookup(b'tip')
             return True
         url_prefix = None
         if b'+' in url[:url.find(b'://')]:
             url_prefix, url = url.split(b'+', 1)
         handlers = []
         url_obj = mercurial.util.url(url)
         test_uri, authinfo = url_obj.authinfo()
         url_obj.passwd = b'*****'
         cleaned_uri = str(url_obj)
         if authinfo:
             # create a password manager
             passmgr = urllib.request.HTTPPasswordMgrWithDefaultRealm()
             passmgr.add_password(*authinfo)
             handlers.extend((mercurial.url.httpbasicauthhandler(passmgr),
                              mercurial.url.httpdigestauthhandler(passmgr)))
         o = urllib.request.build_opener(*handlers)
         o.addheaders = [('Content-Type', 'application/mercurial-0.1'),
                         ('Accept', 'application/mercurial-0.1')]
         req = urllib.request.Request(
             "%s?%s" % (
                 test_uri,
                 urllib.parse.urlencode({
                     'cmd': 'between',
                     'pairs': "%s-%s" % ('0' * 40, '0' * 40),
                 })
             ))
         try:
             resp = o.open(req)
             if resp.code != 200:
                 raise Exception('Return Code is not 200')
         except Exception as e:
             # means it cannot be cloned
             raise urllib.error.URLError("[%s] org_exc: %s" % (cleaned_uri, e))
         if not url_prefix: # skip svn+http://... (and git+... too)
             # now check if it's a proper hg repo
             try:
                 mercurial.httppeer.instance(repoui or mercurial.ui.ui(), url, False).lookup(b'tip')
             except Exception as e:
                 raise urllib.error.URLError(
                     "url [%s] does not look like an hg repo org_exc: %s"
                     % (cleaned_uri, e))
         return True
     def _get_repo(self, create, src_url=None, update_after_clone=False):
         """
         Function will check for mercurial repository in given path and return
         a localrepo object. If there is no repository in that path it will
         raise an exception unless ``create`` parameter is set to True - in
         that case repository would be created and returned.
         If ``src_url`` is given, would try to clone repository from the
         location at given clone_point. Additionally it'll make update to
         working copy accordingly to ``update_after_clone`` flag
         """
         try:
             if src_url:
                 url = safe_bytes(self._get_url(src_url))
                 opts = {}
                 if not update_after_clone:
                     opts.update({'noupdate': True})
                 MercurialRepository._check_url(url, self.baseui)
                 mercurial.commands.clone(self.baseui, url, safe_bytes(self.path), **opts)
                 # Don't try to create if we've already cloned repo
                 create = False
             return mercurial.localrepo.instance(self.baseui, safe_bytes(self.path), create=create)
         except (mercurial.error.Abort, mercurial.error.RepoError) as err:
             if create:
                 msg = "Cannot create repository at %s. Original error was %s" \
                     % (self.name, err)
             else:
                 msg = "Not valid repository at %s. Original error was %s" \
                     % (self.name, err)
             raise RepositoryError(msg)
     @LazyProperty
     def in_memory_changeset(self):
         return MercurialInMemoryChangeset(self)
     @LazyProperty
     def description(self):
         _desc = self._repo.ui.config(b'web', b'description', None, untrusted=True)
         return safe_unicode(_desc or b'unknown')
     @LazyProperty
     def contact(self):
         return safe_unicode(mercurial.hgweb.common.get_contact(self._repo.ui.config)
                             or b'Unknown')
     @LazyProperty
     def last_change(self):
         """
         Returns last change made on this repository as datetime object
         """
         return date_fromtimestamp(self._get_mtime(), makedate()[1])
     def _get_mtime(self):
         try:
             return time.mktime(self.get_changeset().date.timetuple())
         except RepositoryError:
             # fallback to filesystem
             cl_path = os.path.join(self.path, '.hg', "00changelog.i")
             st_path = os.path.join(self.path, '.hg', "store")
             if os.path.exists(cl_path):
                 return os.stat(cl_path).st_mtime
             else:
                 return os.stat(st_path).st_mtime
     def _get_revision(self, revision):
         """
         Given any revision identifier, returns a 40 char string with revision hash.
         :param revision: str or int or None
         """
         if self._empty:
             raise EmptyRepositoryError("There are no changesets yet")
         if revision in [-1, None]:
             revision = b'tip'
         elif isinstance(revision, unicode):
             revision = safe_bytes(revision)
         try:
             if isinstance(revision, int):
                 return ascii_str(self._repo[revision].hex())
             return ascii_str(mercurial.scmutil.revsymbol(self._repo, revision).hex())
         except (IndexError, ValueError, mercurial.error.RepoLookupError, TypeError):
             msg = "Revision %r does not exist for %s" % (safe_unicode(revision), self.name)
             raise ChangesetDoesNotExistError(msg)
         except (LookupError, ):
             msg = "Ambiguous identifier `%s` for %s" % (safe_unicode(revision), self.name)
             raise ChangesetDoesNotExistError(msg)
     def get_ref_revision(self, ref_type, ref_name):
         """
         Returns revision number for the given reference.
         """
         ref_name = safe_str(ref_name)
         if ref_type == 'rev' and not ref_name.strip('0'):
             return self.EMPTY_CHANGESET
         # lookup up the exact node id
         _revset_predicates = {
                 'branch': 'branch',
                 'book': 'bookmark',
                 'tag': 'tag',
                 'rev': 'id',
+            }
         # avoid expensive branch(x) iteration over whole repo
         rev_spec = "%%s & %s(%%s)" % _revset_predicates[ref_type]
         try:
             revs = self._repo.revs(rev_spec, ref_name, ref_name)
         except LookupError:
             msg = "Ambiguous identifier %s:%s for %s" % (ref_type, ref_name, self.name)
             raise ChangesetDoesNotExistError(msg)
         except mercurial.error.RepoLookupError:
             msg = "Revision %s:%s does not exist for %s" % (ref_type, ref_name, self.name)
             raise ChangesetDoesNotExistError(msg)
         if revs:
             revision = revs.last()
         else:
             # TODO: just report 'not found'?
             revision = ref_name
         return self._get_revision(revision)
     def _get_archives(self, archive_name='tip'):
         allowed = self.baseui.configlist(b"web", b"allow_archive",
                                          untrusted=True)
         for name, ext in [(b'zip', '.zip'), (b'gz', '.tar.gz'), (b'bz2', '.tar.bz2')]:
             if name in allowed or self._repo.ui.configbool(b"web",
                                                            b"allow" + name,
                                                            untrusted=True):
                 yield {"type": name, "extension": ext, "node": archive_name}
     def _get_url(self, url):
         """
         Returns normalized url. If schema is not given, would fall
         to filesystem
         (``file:///``) schema.
         Returns normalized url. If schema is not given, fall back to
         filesystem (``file:///``) schema.
         """
         url = safe_str(url)
         if url != 'default' and '://' not in url:
             url = "file:" + urllib.request.pathname2url(url)
         return url
     def get_changeset(self, revision=None):
         """
         Returns ``MercurialChangeset`` object representing repository's
         changeset at the given ``revision``.
         """
         return MercurialChangeset(repository=self, revision=self._get_revision(revision))
     def get_changesets(self, start=None, end=None, start_date=None,
                        end_date=None, branch_name=None, reverse=False, max_revisions=None):
         """
         Returns iterator of ``MercurialChangeset`` objects from start to end
         (both are inclusive)
         :param start: None, str, int or mercurial lookup format
         :param end:  None, str, int or mercurial lookup format
         :param start_date:
         :param end_date:
         :param branch_name:
         :param reversed: return changesets in reversed order
         """
         start_raw_id = self._get_revision(start)
         start_pos = None if start is None else self.revisions.index(start_raw_id)
         end_raw_id = self._get_revision(end)
         end_pos = None if end is None else self.revisions.index(end_raw_id)
         if start_pos is not None and end_pos is not None and start_pos > end_pos:
             raise RepositoryError("Start revision '%s' cannot be "
                                   "after end revision '%s'" % (start, end))
         if branch_name and branch_name not in self.allbranches:
             msg = "Branch %r not found in %s" % (branch_name, self.name)
             raise BranchDoesNotExistError(msg)
         if end_pos is not None:
             end_pos += 1
         # filter branches
         filter_ = []
         if branch_name:
             filter_.append(b'branch("%s")' % safe_bytes(branch_name))
         if start_date:
             filter_.append(b'date(">%s")' % safe_bytes(str(start_date)))
         if end_date:
             filter_.append(b'date("<%s")' % safe_bytes(str(end_date)))
         if filter_ or max_revisions:
             if filter_:
                 revspec = b' and '.join(filter_)
             else:
                 revspec = b'all()'
             if max_revisions:
                 revspec = b'limit(%s, %d)' % (revspec, max_revisions)
             revisions = mercurial.scmutil.revrange(self._repo, [revspec])
         else:
             revisions = self.revisions
         # this is very much a hack to turn this into a list; a better solution
         # would be to get rid of this function entirely and use revsets
         revs = list(revisions)[start_pos:end_pos]
         if reverse:
             revs.reverse()
         return CollectionGenerator(self, revs)
     def pull(self, url):
         """
         Tries to pull changes from external location.
         """
         other = mercurial.hg.peer(self._repo, {}, safe_bytes(self._get_url(url)))
         try:
             mercurial.exchange.pull(self._repo, other, heads=None, force=None)
         except mercurial.error.Abort as err:
             # Propagate error but with vcs's type
             raise RepositoryError(str(err))
     @LazyProperty
     def workdir(self):
         """
         Returns ``Workdir`` instance for this repository.
         """
         return MercurialWorkdir(self)
     def get_config_value(self, section, name=None, config_file=None):
         """
         Returns configuration value for a given [``section``] and ``name``.
         :param section: Section we want to retrieve value from
         :param name: Name of configuration we want to retrieve
         :param config_file: A path to file which should be used to retrieve
           configuration from (might also be a list of file paths)
         """
         if config_file is None:
             config_file = []
         elif isinstance(config_file, str):
             config_file = [config_file]
         config = self._repo.ui
         if config_file:
             config = mercurial.ui.ui()
             for path in config_file:
                 config.readconfig(safe_bytes(path))
         return config.config(safe_bytes(section), safe_bytes(name))
     def get_user_name(self, config_file=None):
         """
         Returns user's name from global configuration file.
         :param config_file: A path to file which should be used to retrieve
           configuration from (might also be a list of file paths)
         """
         username = self.get_config_value('ui', 'username', config_file=config_file)
         if username:
             return author_name(username)
         return None
     def get_user_email(self, config_file=None):
         """
         Returns user's email from global configuration file.
         :param config_file: A path to file which should be used to retrieve
           configuration from (might also be a list of file paths)
         """
         username = self.get_config_value('ui', 'username', config_file=config_file)
         if username:
             return author_email(username)
         return None

kallithea/lib/vcs/backends/ssh.py

➞

Show inline comments

 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
+#
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 vcs.backends.ssh
 ~~~~~~~~~~~~~~~~~
 SSH backend for all available SCMs
 """
 import datetime
 import logging
 import sys
 from kallithea.lib.auth import AuthUser, HasPermissionAnyMiddleware
 from kallithea.lib.utils2 import set_hook_environment
 from kallithea.lib.vcs.utils import safe_str
 from kallithea.model.db import Repository, User, UserSshKeys
 from kallithea.model.meta import Session
 log = logging.getLogger(__name__)
 class BaseSshHandler(object):
     # Protocol for setting properties:
     # Set by sub class:
     #   vcs_type: 'hg' or 'git'
     # Set by make() / __init__():
     #   repo_name: requested repo name - only validated by serve()
     # Set by serve() - must not be accessed before:
     #   db_repo: repository db object
     #   authuser: user that has been authenticated - like request.authuser ... which isn't used here
     #   allow_push: false for read-only access to the repo
     # Set defaults, in case .exit should be called early
     vcs_type = None
     repo_name = None
     @staticmethod
     def make(ssh_command):
         """Factory function. Given a command as invoked over SSH (and preserved
         in SSH_ORIGINAL_COMMAND when run as authorized_keys command), return a
         handler if the command looks ok, else return None.
         """
         raise NotImplementedError
     def __init__(self, repo_name):
         self.repo_name = repo_name.rstrip('/')
     def serve(self, user_id, key_id, client_ip):
         """Verify basic sanity of the repository, and that the user is
         valid and has access - then serve the native VCS protocol for
         repository access."""
         dbuser = User.get(user_id)
         if dbuser is None:
             self.exit('User %r not found' % user_id)
         self.authuser = AuthUser.make(dbuser=dbuser, ip_addr=client_ip)
         log.info('Authorized user %s from SSH %s trusting user id %s and key id %s for %r', dbuser, client_ip, user_id, key_id, self.repo_name)
         if self.authuser is None: # not ok ... but already kind of authenticated by SSH ... but not really not authorized ...
             self.exit('User %s from %s cannot be authorized' % (dbuser.username, client_ip))
         ssh_key = UserSshKeys.get(key_id)
         if ssh_key is None:
             self.exit('SSH key %r not found' % key_id)
         ssh_key.last_seen = datetime.datetime.now()
         Session().commit()
         if HasPermissionAnyMiddleware('repository.write',
                                       'repository.admin')(self.authuser, self.repo_name):
             self.allow_push = True
         elif HasPermissionAnyMiddleware('repository.read')(self.authuser, self.repo_name):
             self.allow_push = False
         else:
-            self.exit('Access to %r denied' % safe_str(self.repo_name))
             self.exit('Access to %r denied' % self.repo_name)
         self.db_repo = Repository.get_by_repo_name(self.repo_name)
         if self.db_repo is None:
             self.exit("Repository '%s' not found" % self.repo_name)
         assert self.db_repo.repo_name == self.repo_name
         # Set global hook environment up for 'push' actions.
         # If pull actions should be served, the actual hook invocation will be
         # hardcoded to 'pull' when log_pull_action is invoked (directly on Git,
         # or through the Mercurial 'outgoing' hook).
         # For push actions, the action in global hook environment is used (in
         # handle_git_post_receive when it is called as Git post-receive hook,
         # or in log_push_action through the Mercurial 'changegroup' hook).
         set_hook_environment(self.authuser.username, client_ip, self.repo_name, self.vcs_type, 'push')
         return self._serve()
     def _serve(self):
         """Serve the native protocol for repository access."""
         raise NotImplementedError
     def exit(self, error):
         log.info('abort serving %s %s: %s', self.vcs_type, self.repo_name, error)
         sys.stderr.write('abort: %s\n' % error)
         sys.exit(1)

kallithea/lib/vcs/nodes.py

➞

Show inline comments

 # -*- coding: utf-8 -*-
 """
     vcs.nodes
     ~~~~~~~~~
     Module holding everything related to vcs nodes.
     :created_on: Apr 8, 2010
     :copyright: (c) 2010-2011 by Marcin Kuzminski, Lukasz Balcerzak.
 """
 import functools
 import mimetypes
 import posixpath
 import stat
 from kallithea.lib.vcs.backends.base import EmptyChangeset
 from kallithea.lib.vcs.exceptions import NodeError, RemovedFileNodeError
-from kallithea.lib.vcs.utils import safe_bytes, safe_str, safe_unicode
 from kallithea.lib.vcs.utils import safe_bytes, safe_unicode
 from kallithea.lib.vcs.utils.lazy import LazyProperty
 class NodeKind:
     SUBMODULE = -1
     DIR = 1
     FILE = 2
 class NodeState:
     ADDED = u'added'
     CHANGED = u'changed'
     NOT_CHANGED = u'not changed'
     REMOVED = u'removed'
 class NodeGeneratorBase(object):
     """
     Base class for removed added and changed filenodes, it's a lazy generator
     class that will create filenodes only on iteration or call
     The len method doesn't need to create filenodes at all
     """
     def __init__(self, current_paths, cs):
         self.cs = cs
         self.current_paths = current_paths
     def __getitem__(self, key):
         assert isinstance(key, slice), key
         for p in self.current_paths[key]:
             yield self.cs.get_node(p)
     def __len__(self):
         return len(self.current_paths)
     def __iter__(self):
         for p in self.current_paths:
             yield self.cs.get_node(p)
 class AddedFileNodesGenerator(NodeGeneratorBase):
     """
     Class holding Added files for current changeset
     """
     pass
 class ChangedFileNodesGenerator(NodeGeneratorBase):
     """
     Class holding Changed files for current changeset
     """
     pass
 class RemovedFileNodesGenerator(NodeGeneratorBase):
     """
     Class holding removed files for current changeset
     """
     def __iter__(self):
         for p in self.current_paths:
             yield RemovedFileNode(path=p)
     def __getitem__(self, key):
         assert isinstance(key, slice), key
         for p in self.current_paths[key]:
             yield RemovedFileNode(path=p)
 @functools.total_ordering
 class Node(object):
     """
     Simplest class representing file or directory on repository.  SCM backends
     should use ``FileNode`` and ``DirNode`` subclasses rather than ``Node``
     directly.
     Node's ``path`` cannot start with slash as we operate on *relative* paths
     only. Moreover, every single node is identified by the ``path`` attribute,
     so it cannot end with slash, too. Otherwise, path could lead to mistakes.
     """
     def __init__(self, path, kind):
         if path.startswith('/'):
             raise NodeError("Cannot initialize Node objects with slash at "
                             "the beginning as only relative paths are supported")
-        self.path = safe_str(path.rstrip('/'))  # we store paths as str
         self.path = path.rstrip('/')
         if path == '' and kind != NodeKind.DIR:
             raise NodeError("Only DirNode and its subclasses may be "
                             "initialized with empty path")
         self.kind = kind
         #self.dirs, self.files = [], []
         if self.is_root() and not self.is_dir():
             raise NodeError("Root node cannot be FILE kind")
     @LazyProperty
     def parent(self):
         parent_path = self.get_parent_path()
         if parent_path:
             if self.changeset:
                 return self.changeset.get_node(parent_path)
             return DirNode(parent_path)
         return None
     @LazyProperty
     def name(self):
         """
         Returns name of the node so if its path
         then only last part is returned.
         """
         return self.path.rstrip('/').split('/')[-1]
     def _get_kind(self):
         return self._kind
     def _set_kind(self, kind):
         if hasattr(self, '_kind'):
             raise NodeError("Cannot change node's kind")
         else:
             self._kind = kind
             # Post setter check (path's trailing slash)
             if self.path.endswith('/'):
                 raise NodeError("Node's path cannot end with slash")
     kind = property(_get_kind, _set_kind)
     def __eq__(self, other):
         if type(self) is not type(other):
             return False
         if self._kind != other._kind:
             return False
         if self.path != other.path:
             return False
         if self.is_file():
             return self.content == other.content
         else:
             # For DirNode's check without entering each dir
             self_nodes_paths = list(sorted(n.path for n in self.nodes))
             other_nodes_paths = list(sorted(n.path for n in self.nodes))
             return self_nodes_paths == other_nodes_paths
     def __lt__(self, other):
         if self._kind < other._kind:
             return True
         if self._kind > other._kind:
             return False
         if self.path < other.path:
             return True
         if self.path > other.path:
             return False
         if self.is_file():
             return self.content < other.content
         else:
             # For DirNode's check without entering each dir
             self_nodes_paths = list(sorted(n.path for n in self.nodes))
             other_nodes_paths = list(sorted(n.path for n in self.nodes))
             return self_nodes_paths < other_nodes_paths
     def __repr__(self):
         return '<%s %r>' % (self.__class__.__name__, self.path)
     def get_parent_path(self):
         """
         Returns node's parent path or empty string if node is root.
         """
         if self.is_root():
             return ''
         return posixpath.dirname(self.path.rstrip('/')) + '/'
     def is_file(self):
         """
         Returns ``True`` if node's kind is ``NodeKind.FILE``, ``False``
         otherwise.
         """
         return self.kind == NodeKind.FILE
     def is_dir(self):
         """
         Returns ``True`` if node's kind is ``NodeKind.DIR``, ``False``
         otherwise.
         """
         return self.kind == NodeKind.DIR
     def is_root(self):
         """
         Returns ``True`` if node is a root node and ``False`` otherwise.
         """
         return self.kind == NodeKind.DIR and self.path == ''
     def is_submodule(self):
         """
         Returns ``True`` if node's kind is ``NodeKind.SUBMODULE``, ``False``
         otherwise.
         """
         return self.kind == NodeKind.SUBMODULE
     @LazyProperty
     def added(self):
         return self.state is NodeState.ADDED
     @LazyProperty
     def changed(self):
         return self.state is NodeState.CHANGED
     @LazyProperty
     def not_changed(self):
         return self.state is NodeState.NOT_CHANGED
     @LazyProperty
     def removed(self):
         return self.state is NodeState.REMOVED
 class FileNode(Node):
     """
     Class representing file nodes.
     :attribute: path: path to the node, relative to repository's root
     :attribute: content: if given arbitrary sets content of the file
     :attribute: changeset: if given, first time content is accessed, callback
     :attribute: mode: octal stat mode for a node. Default is 0100644.
     """
     def __init__(self, path, content=None, changeset=None, mode=None):
         """
         Only one of ``content`` and ``changeset`` may be given. Passing both
         would raise ``NodeError`` exception.
         :param path: relative path to the node
         :param content: content may be passed to constructor
         :param changeset: if given, will use it to lazily fetch content
         :param mode: octal representation of ST_MODE (i.e. 0100644)
         """
         if content and changeset:
             raise NodeError("Cannot use both content and changeset")
         super(FileNode, self).__init__(path, kind=NodeKind.FILE)
         self.changeset = changeset
         if not isinstance(content, bytes) and content is not None:
             # File content is one thing that inherently must be bytes ... but
             # VCS module tries to be "user friendly" and support unicode ...
             content = safe_bytes(content)
         self._content = content
         self._mode = mode or 0o100644
     @LazyProperty
     def mode(self):
         """
         Returns lazily mode of the FileNode. If ``changeset`` is not set, would
         use value given at initialization or 0100644 (default).
         """
         if self.changeset:
             mode = self.changeset.get_file_mode(self.path)
         else:
             mode = self._mode
         return mode
     @property
     def content(self):
         """
         Returns lazily byte content of the FileNode.
         """
         if self.changeset:
             content = self.changeset.get_file_content(self.path)
         else:
             content = self._content
         return content
     @LazyProperty
     def size(self):
         if self.changeset:
             return self.changeset.get_file_size(self.path)
         raise NodeError("Cannot retrieve size of the file without related "
             "changeset attribute")
     @LazyProperty
     def message(self):
         if self.changeset:
             return self.last_changeset.message
@@ @@ -403,207 +403,207 @@ class FileNode(Node): @@
         """
         return b'\0' in self.content
     def is_browser_compatible_image(self):
         return self.mimetype in [
             "image/gif",
             "image/jpeg",
             "image/png",
             "image/bmp"
+        ]
     @LazyProperty
     def extension(self):
         """Returns filenode extension"""
         return self.name.split('.')[-1]
     @property
     def is_executable(self):
         """
         Returns ``True`` if file has executable flag turned on.
         """
         return bool(self.mode & stat.S_IXUSR)
     def __repr__(self):
         return '<%s %r @ %s>' % (self.__class__.__name__, self.path,
                                  getattr(self.changeset, 'short_id', ''))
 class RemovedFileNode(FileNode):
     """
     Dummy FileNode class - trying to access any public attribute except path,
     name, kind or state (or methods/attributes checking those two) would raise
     RemovedFileNodeError.
     """
     ALLOWED_ATTRIBUTES = [
         'name', 'path', 'state', 'is_root', 'is_file', 'is_dir', 'kind',
         'added', 'changed', 'not_changed', 'removed'
+    ]
     def __init__(self, path):
         """
         :param path: relative path to the node
         """
         super(RemovedFileNode, self).__init__(path=path)
     def __getattribute__(self, attr):
         if attr.startswith('_') or attr in RemovedFileNode.ALLOWED_ATTRIBUTES:
             return super(RemovedFileNode, self).__getattribute__(attr)
         raise RemovedFileNodeError("Cannot access attribute %s on "
             "RemovedFileNode" % attr)
     @LazyProperty
     def state(self):
         return NodeState.REMOVED
 class DirNode(Node):
     """
     DirNode stores list of files and directories within this node.
     Nodes may be used standalone but within repository context they
     lazily fetch data within same repository's changeset.
     """
     def __init__(self, path, nodes=(), changeset=None):
         """
         Only one of ``nodes`` and ``changeset`` may be given. Passing both
         would raise ``NodeError`` exception.
         :param path: relative path to the node
         :param nodes: content may be passed to constructor
         :param changeset: if given, will use it to lazily fetch content
         :param size: always 0 for ``DirNode``
         """
         if nodes and changeset:
             raise NodeError("Cannot use both nodes and changeset")
         super(DirNode, self).__init__(path, NodeKind.DIR)
         self.changeset = changeset
         self._nodes = nodes
     @LazyProperty
     def content(self):
         raise NodeError("%s represents a dir and has no ``content`` attribute"
             % self)
     @LazyProperty
     def nodes(self):
         if self.changeset:
             nodes = self.changeset.get_nodes(self.path)
         else:
             nodes = self._nodes
         self._nodes_dict = dict((node.path, node) for node in nodes)
         return sorted(nodes)
     @LazyProperty
     def files(self):
         return sorted((node for node in self.nodes if node.is_file()))
     @LazyProperty
     def dirs(self):
         return sorted((node for node in self.nodes if node.is_dir()))
     def __iter__(self):
         for node in self.nodes:
             yield node
     def get_node(self, path):
         """
         Returns node from within this particular ``DirNode``, so it is now
         allowed to fetch, i.e. node located at 'docs/api/index.rst' from node
         'docs'. In order to access deeper nodes one must fetch nodes between
         them first - this would work::
            docs = root.get_node('docs')
            docs.get_node('api').get_node('index.rst')
         :param: path - relative to the current node
         .. note::
            To access lazily (as in example above) node have to be initialized
            with related changeset object - without it node is out of
            context and may know nothing about anything else than nearest
            (located at same level) nodes.
         """
         try:
             path = path.rstrip('/')
             if path == '':
                 raise NodeError("Cannot retrieve node without path")
             self.nodes  # access nodes first in order to set _nodes_dict
             paths = path.split('/')
             if len(paths) == 1:
                 if not self.is_root():
                     path = '/'.join((self.path, paths[0]))
                 else:
                     path = paths[0]
                 return self._nodes_dict[path]
             elif len(paths) > 1:
                 if self.changeset is None:
                     raise NodeError("Cannot access deeper "
                                     "nodes without changeset")
                 else:
                     path1, path2 = paths[0], '/'.join(paths[1:])
                     return self.get_node(path1).get_node(path2)
             else:
                 raise KeyError
         except KeyError:
             raise NodeError("Node does not exist at %s" % path)
     @LazyProperty
     def state(self):
         raise NodeError("Cannot access state of DirNode")
     @LazyProperty
     def size(self):
         size = 0
         for root, dirs, files in self.changeset.walk(self.path):
             for f in files:
                 size += f.size
         return size
     def __repr__(self):
         return '<%s %r @ %s>' % (self.__class__.__name__, self.path,
                                  getattr(self.changeset, 'short_id', ''))
 class RootNode(DirNode):
     """
     DirNode being the root node of the repository.
     """
     def __init__(self, nodes=(), changeset=None):
         super(RootNode, self).__init__(path='', nodes=nodes,
             changeset=changeset)
     def __repr__(self):
         return '<%s>' % self.__class__.__name__
 class SubModuleNode(Node):
     """
     represents a SubModule of Git or SubRepo of Mercurial
     """
     is_binary = False
     size = 0
     def __init__(self, name, url, changeset=None, alias=None):
         # Note: Doesn't call Node.__init__!
         self.path = name
         self.kind = NodeKind.SUBMODULE
         self.alias = alias
         # we have to use emptyChangeset here since this can point to svn/git/hg
         # submodules we cannot get from repository
-        self.changeset = EmptyChangeset(str(changeset), alias=alias)
         self.changeset = EmptyChangeset(changeset, alias=alias)
         self.url = url
     def __repr__(self):
         return '<%s %r @ %s>' % (self.__class__.__name__, self.path,
                                  getattr(self.changeset, 'short_id', ''))
     @LazyProperty
     def name(self):
         """
         Returns name of the node so if its path
         then only last part is returned.
         """
         org = self.path.rstrip('/').rsplit('/', 1)[-1]
         return u'%s @ %s' % (org, self.changeset.short_id)

kallithea/lib/vcs/utils/__init__.py

➞

Show inline comments

@@ @@ -15,207 +15,207 @@ def makedate(): @@
     if lt[8] == 1 and time.daylight:
         tz = time.altzone
     else:
         tz = time.timezone
     return time.mktime(lt), tz
 def aslist(obj, sep=None, strip=True):
     """
     Returns given string separated by sep as list
     :param obj:
     :param sep:
     :param strip:
     """
     if isinstance(obj, str):
         lst = obj.split(sep)
         if strip:
             lst = [v.strip() for v in lst]
         return lst
     elif isinstance(obj, (list, tuple)):
         return obj
     elif obj is None:
         return []
     else:
         return [obj]
 def date_fromtimestamp(unixts, tzoffset=0):
     """
     Makes a local datetime object out of unix timestamp
     :param unixts:
     :param tzoffset:
     """
     return datetime.datetime.fromtimestamp(float(unixts))
 def safe_int(val, default=None):
     """
     Returns int() of val if val is not convertible to int use default
     instead
     :param val:
     :param default:
     """
     try:
         val = int(val)
     except (ValueError, TypeError):
         val = default
     return val
 def safe_unicode(s):
     """
     Safe unicode function. Use a few tricks to turn s into unicode string:
     In case of UnicodeDecodeError with configured default encodings, try to
     detect encoding with chardet library, then fall back to first encoding with
     errors replaced.
     """
     if isinstance(s, unicode):
         return s
     if not isinstance(s, bytes):  # use __str__ / __unicode__ and don't expect UnicodeDecodeError
         return unicode(s)
     from kallithea.lib.vcs.conf import settings
     for enc in settings.DEFAULT_ENCODINGS:
         try:
             return unicode(s, enc)
         except UnicodeDecodeError:
             pass
     try:
         import chardet
         encoding = chardet.detect(s)['encoding']
         if encoding is not None:
             return s.decode(encoding)
     except (ImportError, UnicodeDecodeError):
         pass
     return unicode(s, settings.DEFAULT_ENCODINGS[0], 'replace')
 def safe_bytes(s):
     """
     Safe bytes function. Use a few tricks to turn s into bytes string:
     In case of UnicodeEncodeError with configured default encodings, fall back
     to first configured encoding with errors replaced.
     """
     if isinstance(s, bytes):
         return s
     assert isinstance(s, unicode), repr(s)  # bytes cannot coerse with __str__ or handle None or int
     from kallithea.lib.vcs.conf import settings
     for enc in settings.DEFAULT_ENCODINGS:
         try:
             return s.encode(enc)
         except UnicodeEncodeError:
             pass
     return s.encode(settings.DEFAULT_ENCODINGS[0], 'replace')
 safe_str = safe_bytes  # safe_str is deprecated - it will be redefined when changing to py3
 def ascii_bytes(s):
     """
     Simple conversion from unicode/str to bytes, *assuming* all codepoints are
 -bit and it thus is pure ASCII.
     Will fail badly with UnicodeError on invalid input.
     This should be used where enocding and "safe" ambiguity should be avoided.
     Where strings already have been encoded in other ways but still are unicode
     string - for example to hex, base64, json, urlencoding, or are known to be
     identifiers.
     >>> ascii_bytes('a')
     'a'
     >>> ascii_bytes(u'a')
     'a'
     >>> ascii_bytes('å')
     Traceback (most recent call last):
     UnicodeDecodeError: 'ascii' codec can't decode byte 0xc3 in position 0: ordinal not in range(128)
     >>> ascii_bytes(u'å')
     Traceback (most recent call last):
     UnicodeEncodeError: 'ascii' codec can't encode characters in position 0-1: ordinal not in range(128)
     """
     assert isinstance(s, (unicode, str)), repr(s)
     return s.encode('ascii')
 def ascii_str(s):
     r"""
     Simple conversion from bytes to str, *assuming* all codepoints are
 -bit and it thus is pure ASCII.
     Will fail badly with UnicodeError on invalid input.
     This should be used where enocding and "safe" ambiguity should be avoided.
     Where strings are encoded but also in other ways are known to be ASCII, and
     where a unicode string is wanted without caring about encoding. For example
     to hex, base64, urlencoding, or are known to be identifiers.
     >>> ascii_str('a')
     'a'
     >>> ascii_str(u'a')
     Traceback (most recent call last):
     AssertionError: u'a'
     >>> ascii_str('å')
     Traceback (most recent call last):
     UnicodeDecodeError: 'ascii' codec can't decode byte 0xc3 in position 0: ordinal not in range(128)
     >>> ascii_str(u'å')
     Traceback (most recent call last):
     AssertionError: u'\xc3\xa5'
     """
     assert isinstance(s, bytes), repr(s)
     # Note: we use "encode", even though we really *should* use "decode". But
     # we are in py2 and don't want py2, and encode is doing what we need for the
     # ascii subset.
     return s.encode('ascii')
 # Regex taken from http://www.regular-expressions.info/email.html
 email_re = re.compile(
     r"""[a-z0-9!#$%&'*+/=?^_`{|}~-]+(?:\.[a-z0-9!#$%&'*+/=?^_`{|}~-]+)*@"""
     r"""(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?""",
     re.IGNORECASE)
 def author_email(author):
     """
     Returns email address of given author string.
     If author contains <> brackets, only look inside that.
     If any RFC valid email address is found, return that.
     Else, return empty string.
     """
     if not author:
         return ''
     l = author.find('<') + 1
     if l != 0:
         r = author.find('>', l)
         if r != -1:
             author = author[l:r]
     m = email_re.search(author)
     if m is None:
         return ''
-    return safe_str(m.group(0))
     return m.group(0)
 def author_name(author):
     """
     get name of author, or else username.
     It'll try to find an email in the author string and just cut it off
     to get the username
     """
     if not author:
         return ''
     if '@' not in author:
         return author
     return author.replace(author_email(author), '').replace('<', '') \
         .replace('>', '').strip()

kallithea/model/db.py

➞

Show inline comments

 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
+#
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.model.db
 ~~~~~~~~~~~~~~~~~~
 Database Models for Kallithea
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Apr 08, 2010
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 import base64
 import collections
 import datetime
 import functools
 import hashlib
 import logging
 import os
 import time
 import traceback
 import ipaddr
 import sqlalchemy
 from beaker.cache import cache_region, region_invalidate
 from sqlalchemy import Boolean, Column, DateTime, Float, ForeignKey, Index, Integer, LargeBinary, String, Unicode, UnicodeText, UniqueConstraint
 from sqlalchemy.ext.hybrid import hybrid_property
 from sqlalchemy.orm import class_mapper, joinedload, relationship, validates
 from tg.i18n import lazy_ugettext as _
 from webob.exc import HTTPNotFound
 import kallithea
 from kallithea.lib import ext_json
 from kallithea.lib.caching_query import FromCache
 from kallithea.lib.exceptions import DefaultUserException
 from kallithea.lib.utils2 import (
-    Optional, ascii_bytes, aslist, get_changeset_safe, get_clone_url, remove_prefix, safe_bytes, safe_int, safe_str, safe_unicode, str2bool, urlreadable)
     Optional, ascii_bytes, aslist, get_changeset_safe, get_clone_url, remove_prefix, safe_bytes, safe_int, safe_unicode, str2bool, urlreadable)
 from kallithea.lib.vcs import get_backend
 from kallithea.lib.vcs.backends.base import EmptyChangeset
 from kallithea.lib.vcs.utils.helpers import get_scm
 from kallithea.lib.vcs.utils.lazy import LazyProperty
 from kallithea.model.meta import Base, Session
 URL_SEP = '/'
 log = logging.getLogger(__name__)
 #==============================================================================
 # BASE CLASSES
 #==============================================================================
 def _hash_key(k):
     return hashlib.md5(safe_bytes(k)).hexdigest()
 class BaseDbModel(object):
     """
     Base Model for all classes
     """
     @classmethod
     def _get_keys(cls):
         """return column names for this model """
         # Note: not a normal dict - iterator gives "users.firstname", but keys gives "firstname"
         return class_mapper(cls).c.keys()
     def get_dict(self):
         """
         return dict with keys and values corresponding
         to this model data """
         d = {}
         for k in self._get_keys():
             d[k] = getattr(self, k)
         # also use __json__() if present to get additional fields
         _json_attr = getattr(self, '__json__', None)
         if _json_attr:
             # update with attributes from __json__
             if callable(_json_attr):
                 _json_attr = _json_attr()
             for k, val in _json_attr.items():
                 d[k] = val
         return d
     def get_appstruct(self):
         """return list with keys and values tuples corresponding
         to this model data """
         return [
             (k, getattr(self, k))
             for k in self._get_keys()
+        ]
     def populate_obj(self, populate_dict):
         """populate model with data from given populate_dict"""
         for k in self._get_keys():
             if k in populate_dict:
                 setattr(self, k, populate_dict[k])
     @classmethod
     def query(cls):
         return Session().query(cls)
     @classmethod
     def get(cls, id_):
         if id_:
             return cls.query().get(id_)
     @classmethod
     def guess_instance(cls, value, callback=None):
         """Haphazardly attempt to convert `value` to a `cls` instance.
         If `value` is None or already a `cls` instance, return it. If `value`
         is a number (or looks like one if you squint just right), assume it's
         a database primary key and let SQLAlchemy sort things out. Otherwise,
         fall back to resolving it using `callback` (if specified); this could
         e.g. be a function that looks up instances by name (though that won't
         work if the name begins with a digit). Otherwise, raise Exception.
         """
         if value is None:
             return None
         if isinstance(value, cls):
             return value
         if isinstance(value, int):
             return cls.get(value)
         if isinstance(value, str) and value.isdigit():
             return cls.get(int(value))
         if callback is not None:
             return callback(value)
         raise Exception(
             'given object must be int, long or Instance of %s '
             'got %s, no callback provided' % (cls, type(value))
+        )
     @classmethod
     def get_or_404(cls, id_):
         try:
             id_ = int(id_)
         except (TypeError, ValueError):
             raise HTTPNotFound
         res = cls.query().get(id_)
         if res is None:
             raise HTTPNotFound
         return res
     @classmethod
     def delete(cls, id_):
         obj = cls.query().get(id_)
         Session().delete(obj)
     def __repr__(self):
         return '<DB:%s>' % (self.__class__.__name__)
 _table_args_default_dict = {'extend_existing': True,
                             'mysql_engine': 'InnoDB',
                             'mysql_charset': 'utf8',
                             'sqlite_autoincrement': True,
+                           }
 class Setting(Base, BaseDbModel):
     __tablename__ = 'settings'
     __table_args__ = (
         _table_args_default_dict,
+    )
     SETTINGS_TYPES = {
         'str': safe_bytes,
         'int': safe_int,
         'unicode': safe_unicode,
         'bool': str2bool,
         'list': functools.partial(aslist, sep=',')
+    }
     DEFAULT_UPDATE_URL = ''
     app_settings_id = Column(Integer(), primary_key=True)
     app_settings_name = Column(String(255), nullable=False, unique=True)
     _app_settings_value = Column("app_settings_value", Unicode(4096), nullable=False)
     _app_settings_type = Column("app_settings_type", String(255), nullable=True) # FIXME: not nullable?
     def __init__(self, key='', val='', type='unicode'):
         self.app_settings_name = key
         self.app_settings_value = val
         self.app_settings_type = type
     @validates('_app_settings_value')
     def validate_settings_value(self, key, val):
         assert isinstance(val, unicode)
         return val
     @hybrid_property
     def app_settings_value(self):
         v = self._app_settings_value
         _type = self.app_settings_type
         converter = self.SETTINGS_TYPES.get(_type) or self.SETTINGS_TYPES['unicode']
         return converter(v)
     @app_settings_value.setter
     def app_settings_value(self, val):
         """
         Setter that will always make sure we use unicode in app_settings_value
         :param val:
         """
         self._app_settings_value = safe_unicode(val)
     @hybrid_property
     def app_settings_type(self):
         return self._app_settings_type
     @app_settings_type.setter
     def app_settings_type(self, val):
         if val not in self.SETTINGS_TYPES:
             raise Exception('type must be one of %s got %s'
                             % (list(self.SETTINGS_TYPES), val))
         self._app_settings_type = val
     def __repr__(self):
         return "<%s %s.%s=%r>" % (
             self.__class__.__name__,
             self.app_settings_name, self.app_settings_type, self.app_settings_value
+        )
     @classmethod
@@ @@ -1235,385 +1235,385 @@ class Repository(Base, BaseDbModel): @@
         if with_pullrequests:
             data['pull_requests'] = repo.pull_requests_other
         rc_config = Setting.get_app_settings()
         repository_fields = str2bool(rc_config.get('repository_fields'))
         if repository_fields:
             for f in self.extra_fields:
                 data[f.field_key_prefixed] = f.field_value
         return data
     @property
     def last_db_change(self):
         return self.updated_on
     @property
     def clone_uri_hidden(self):
         clone_uri = self.clone_uri
         if clone_uri:
             import urlobject
             url_obj = urlobject.URLObject(self.clone_uri)
             if url_obj.password:
                 clone_uri = url_obj.with_password('*****')
         return clone_uri
     def clone_url(self, clone_uri_tmpl, with_id=False, username=None):
         if '{repo}' not in clone_uri_tmpl and '_{repoid}' not in clone_uri_tmpl:
             log.error("Configured clone_uri_tmpl %r has no '{repo}' or '_{repoid}' and cannot toggle to use repo id URLs", clone_uri_tmpl)
         elif with_id:
             clone_uri_tmpl = clone_uri_tmpl.replace('{repo}', '_{repoid}')
         else:
             clone_uri_tmpl = clone_uri_tmpl.replace('_{repoid}', '{repo}')
         import kallithea.lib.helpers as h
         prefix_url = h.canonical_url('home')
         return get_clone_url(clone_uri_tmpl=clone_uri_tmpl,
                              prefix_url=prefix_url,
                              repo_name=self.repo_name,
                              repo_id=self.repo_id,
                              username=username)
     def set_state(self, state):
         self.repo_state = state
     #==========================================================================
     # SCM PROPERTIES
     #==========================================================================
     def get_changeset(self, rev=None):
         return get_changeset_safe(self.scm_instance, rev)
     def get_landing_changeset(self):
         """
         Returns landing changeset, or if that doesn't exist returns the tip
         """
         _rev_type, _rev = self.landing_rev
         cs = self.get_changeset(_rev)
         if isinstance(cs, EmptyChangeset):
             return self.get_changeset()
         return cs
     def update_changeset_cache(self, cs_cache=None):
         """
         Update cache of last changeset for repository, keys should be::
             short_id
             raw_id
             revision
             message
             date
             author
         :param cs_cache:
         """
         from kallithea.lib.vcs.backends.base import BaseChangeset
         if cs_cache is None:
             cs_cache = EmptyChangeset()
             # use no-cache version here
             scm_repo = self.scm_instance_no_cache()
             if scm_repo:
                 cs_cache = scm_repo.get_changeset()
         if isinstance(cs_cache, BaseChangeset):
             cs_cache = cs_cache.__json__()
         if (not self.changeset_cache or cs_cache['raw_id'] != self.changeset_cache['raw_id']):
             _default = datetime.datetime.fromtimestamp(0)
             last_change = cs_cache.get('date') or _default
             log.debug('updated repo %s with new cs cache %s',
                       self.repo_name, cs_cache)
             self.updated_on = last_change
             self.changeset_cache = cs_cache
             Session().commit()
         else:
             log.debug('changeset_cache for %s already up to date with %s',
                       self.repo_name, cs_cache['raw_id'])
     @property
     def tip(self):
         return self.get_changeset('tip')
     @property
     def author(self):
         return self.tip.author
     @property
     def last_change(self):
         return self.scm_instance.last_change
     def get_comments(self, revisions=None):
         """
         Returns comments for this repository grouped by revisions
         :param revisions: filter query by revisions only
         """
         cmts = ChangesetComment.query() \
             .filter(ChangesetComment.repo == self)
         if revisions is not None:
             if not revisions:
                 return {} # don't use sql 'in' on empty set
             cmts = cmts.filter(ChangesetComment.revision.in_(revisions))
         grouped = collections.defaultdict(list)
         for cmt in cmts.all():
             grouped[cmt.revision].append(cmt)
         return grouped
     def statuses(self, revisions):
         """
         Returns statuses for this repository.
         PRs without any votes do _not_ show up as unreviewed.
         :param revisions: list of revisions to get statuses for
         """
         if not revisions:
             return {}
         statuses = ChangesetStatus.query() \
             .filter(ChangesetStatus.repo == self) \
             .filter(ChangesetStatus.version == 0) \
             .filter(ChangesetStatus.revision.in_(revisions))
         grouped = {}
         for stat in statuses.all():
             pr_id = pr_nice_id = pr_repo = None
             if stat.pull_request:
                 pr_id = stat.pull_request.pull_request_id
                 pr_nice_id = PullRequest.make_nice_id(pr_id)
                 pr_repo = stat.pull_request.other_repo.repo_name
             grouped[stat.revision] = [str(stat.status), stat.status_lbl,
                                       pr_id, pr_repo, pr_nice_id,
                                       stat.author]
         return grouped
     def _repo_size(self):
         from kallithea.lib import helpers as h
         log.debug('calculating repository size...')
         return h.format_byte_size(self.scm_instance.size)
     #==========================================================================
     # SCM CACHE INSTANCE
     #==========================================================================
     def set_invalidate(self):
         """
         Mark caches of this repo as invalid.
         """
         CacheInvalidation.set_invalidate(self.repo_name)
     _scm_instance = None
     @property
     def scm_instance(self):
         if self._scm_instance is None:
             self._scm_instance = self.scm_instance_cached()
         return self._scm_instance
     def scm_instance_cached(self, valid_cache_keys=None):
         @cache_region('long_term', 'scm_instance_cached')
         def _c(repo_name): # repo_name is just for the cache key
             log.debug('Creating new %s scm_instance and populating cache', repo_name)
             return self.scm_instance_no_cache()
         rn = self.repo_name
         valid = CacheInvalidation.test_and_set_valid(rn, None, valid_cache_keys=valid_cache_keys)
         if not valid:
             log.debug('Cache for %s invalidated, getting new object', rn)
             region_invalidate(_c, None, 'scm_instance_cached', rn)
         else:
             log.debug('Trying to get scm_instance of %s from cache', rn)
         return _c(rn)
     def scm_instance_no_cache(self):
-        repo_full_path = safe_str(self.repo_full_path)
         repo_full_path = self.repo_full_path
         alias = get_scm(repo_full_path)[0]
         log.debug('Creating instance of %s repository from %s',
                   alias, self.repo_full_path)
         backend = get_backend(alias)
         if alias == 'hg':
             repo = backend(repo_full_path, create=False,
                            baseui=self._ui)
         else:
             repo = backend(repo_full_path, create=False)
         return repo
     def __json__(self):
         return dict(
             repo_id=self.repo_id,
             repo_name=self.repo_name,
             landing_rev=self.landing_rev,
+        )
 class RepoGroup(Base, BaseDbModel):
     __tablename__ = 'groups'
     __table_args__ = (
         _table_args_default_dict,
+    )
     SEP = ' &raquo; '
     group_id = Column(Integer(), primary_key=True)
     group_name = Column(Unicode(255), nullable=False, unique=True) # full path
     parent_group_id = Column('group_parent_id', Integer(), ForeignKey('groups.group_id'), nullable=True)
     group_description = Column(Unicode(10000), nullable=False)
     owner_id = Column('user_id', Integer(), ForeignKey('users.user_id'), nullable=False)
     created_on = Column(DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
     repo_group_to_perm = relationship('UserRepoGroupToPerm', cascade='all', order_by='UserRepoGroupToPerm.group_to_perm_id')
     users_group_to_perm = relationship('UserGroupRepoGroupToPerm', cascade='all')
     parent_group = relationship('RepoGroup', remote_side=group_id)
     owner = relationship('User')
     @classmethod
     def query(cls, sorted=False):
         """Add RepoGroup-specific helpers for common query constructs.
         sorted: if True, apply the default ordering (name, case insensitive).
         """
         q = super(RepoGroup, cls).query()
         if sorted:
             q = q.order_by(sqlalchemy.func.lower(RepoGroup.group_name))
         return q
     def __init__(self, group_name='', parent_group=None):
         self.group_name = group_name
         self.parent_group = parent_group
     def __repr__(self):
         return "<%s %s: %s>" % (self.__class__.__name__,
                                 self.group_id, self.group_name)
     @classmethod
     def _generate_choice(cls, repo_group):
         """Return tuple with group_id and name as html literal"""
         from webhelpers2.html import literal
         if repo_group is None:
             return (-1, u'-- %s --' % _('top level'))
         return repo_group.group_id, literal(cls.SEP.join(repo_group.full_path_splitted))
     @classmethod
     def groups_choices(cls, groups):
         """Return tuples with group_id and name as html literal."""
         return sorted((cls._generate_choice(g) for g in groups),
                       key=lambda c: c[1].split(cls.SEP))
     @classmethod
     def url_sep(cls):
         return URL_SEP
     @classmethod
     def guess_instance(cls, value):
         return super(RepoGroup, cls).guess_instance(value, RepoGroup.get_by_group_name)
     @classmethod
     def get_by_group_name(cls, group_name, cache=False, case_insensitive=False):
         group_name = group_name.rstrip('/')
         if case_insensitive:
             gr = cls.query() \
                 .filter(sqlalchemy.func.lower(cls.group_name) == sqlalchemy.func.lower(group_name))
         else:
             gr = cls.query() \
                 .filter(cls.group_name == group_name)
         if cache:
             gr = gr.options(FromCache(
                             "sql_cache_short",
                             "get_group_%s" % _hash_key(group_name)
+                            )
+            )
         return gr.scalar()
     @property
     def parents(self):
         groups = []
         group = self.parent_group
         while group is not None:
             groups.append(group)
             group = group.parent_group
             assert group not in groups, group # avoid recursion on bad db content
         groups.reverse()
         return groups
     @property
     def children(self):
         return RepoGroup.query().filter(RepoGroup.parent_group == self)
     @property
     def name(self):
         return self.group_name.split(RepoGroup.url_sep())[-1]
     @property
     def full_path(self):
         return self.group_name
     @property
     def full_path_splitted(self):
         return self.group_name.split(RepoGroup.url_sep())
     @property
     def repositories(self):
         return Repository.query(sorted=True).filter_by(group=self)
     @property
     def repositories_recursive_count(self):
         cnt = self.repositories.count()
         def children_count(group):
             cnt = 0
             for child in group.children:
                 cnt += child.repositories.count()
                 cnt += children_count(child)
             return cnt
         return cnt + children_count(self)
     def _recursive_objects(self, include_repos=True):
         all_ = []
         def _get_members(root_gr):
             if include_repos:
                 for r in root_gr.repositories:
                     all_.append(r)
             childs = root_gr.children.all()
             if childs:
                 for gr in childs:
                     all_.append(gr)
                     _get_members(gr)
         _get_members(self)
         return [self] + all_
     def recursive_groups_and_repos(self):
         """
         Recursive return all groups, with repositories in those groups
         """
         return self._recursive_objects()
     def recursive_groups(self):
         """
         Returns all children groups for this group including children of children
         """
         return self._recursive_objects(include_repos=False)
     def get_new_name(self, group_name):
         """
         returns new full group name based on parent and new name
         :param group_name:
         """
         path_prefix = (self.parent_group.full_path_splitted if
                        self.parent_group else [])
         return RepoGroup.url_sep().join(path_prefix + [group_name])
     def get_api_data(self):
         """
         Common function for generating api data
         """
         group = self
         data = dict(
             group_id=group.group_id,
             group_name=group.group_name,
@@ @@ -1902,389 +1902,389 @@ class UserGroupUserGroupToPerm(Base, Bas @@
         n.target_user_group = target_user_group
         n.user_group = user_group
         n.permission = permission
         Session().add(n)
         return n
     def __repr__(self):
         return '<%s %s at %s: %s>' % (
             self.__class__.__name__, self.user_group, self.target_user_group, self.permission)
 class UserGroupToPerm(Base, BaseDbModel):
     __tablename__ = 'users_group_to_perm'
     __table_args__ = (
         UniqueConstraint('users_group_id', 'permission_id',),
         _table_args_default_dict,
+    )
     users_group_to_perm_id = Column(Integer(), primary_key=True)
     users_group_id = Column(Integer(), ForeignKey('users_groups.users_group_id'), nullable=False)
     permission_id = Column(Integer(), ForeignKey('permissions.permission_id'), nullable=False)
     users_group = relationship('UserGroup')
     permission = relationship('Permission')
 class UserRepoGroupToPerm(Base, BaseDbModel):
     __tablename__ = 'user_repo_group_to_perm'
     __table_args__ = (
         UniqueConstraint('user_id', 'group_id', 'permission_id'),
         _table_args_default_dict,
+    )
     group_to_perm_id = Column(Integer(), primary_key=True)
     user_id = Column(Integer(), ForeignKey('users.user_id'), nullable=False)
     group_id = Column(Integer(), ForeignKey('groups.group_id'), nullable=False)
     permission_id = Column(Integer(), ForeignKey('permissions.permission_id'), nullable=False)
     user = relationship('User')
     group = relationship('RepoGroup')
     permission = relationship('Permission')
     @classmethod
     def create(cls, user, repository_group, permission):
         n = cls()
         n.user = user
         n.group = repository_group
         n.permission = permission
         Session().add(n)
         return n
 class UserGroupRepoGroupToPerm(Base, BaseDbModel):
     __tablename__ = 'users_group_repo_group_to_perm'
     __table_args__ = (
         UniqueConstraint('users_group_id', 'group_id'),
         _table_args_default_dict,
+    )
     users_group_repo_group_to_perm_id = Column(Integer(), primary_key=True)
     users_group_id = Column(Integer(), ForeignKey('users_groups.users_group_id'), nullable=False)
     group_id = Column(Integer(), ForeignKey('groups.group_id'), nullable=False)
     permission_id = Column(Integer(), ForeignKey('permissions.permission_id'), nullable=False)
     users_group = relationship('UserGroup')
     permission = relationship('Permission')
     group = relationship('RepoGroup')
     @classmethod
     def create(cls, user_group, repository_group, permission):
         n = cls()
         n.users_group = user_group
         n.group = repository_group
         n.permission = permission
         Session().add(n)
         return n
 class Statistics(Base, BaseDbModel):
     __tablename__ = 'statistics'
     __table_args__ = (
          _table_args_default_dict,
+    )
     stat_id = Column(Integer(), primary_key=True)
     repository_id = Column(Integer(), ForeignKey('repositories.repo_id'), nullable=False, unique=True)
     stat_on_revision = Column(Integer(), nullable=False)
     commit_activity = Column(LargeBinary(1000000), nullable=False) # JSON data
     commit_activity_combined = Column(LargeBinary(), nullable=False) # JSON data
     languages = Column(LargeBinary(1000000), nullable=False) # JSON data
     repository = relationship('Repository', single_parent=True)
 class UserFollowing(Base, BaseDbModel):
     __tablename__ = 'user_followings'
     __table_args__ = (
         UniqueConstraint('user_id', 'follows_repository_id', name='uq_user_followings_user_repo'),
         UniqueConstraint('user_id', 'follows_user_id', name='uq_user_followings_user_user'),
         _table_args_default_dict,
+    )
     user_following_id = Column(Integer(), primary_key=True)
     user_id = Column(Integer(), ForeignKey('users.user_id'), nullable=False)
     follows_repository_id = Column(Integer(), ForeignKey('repositories.repo_id'), nullable=True)
     follows_user_id = Column(Integer(), ForeignKey('users.user_id'), nullable=True)
     follows_from = Column(DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
     user = relationship('User', primaryjoin='User.user_id==UserFollowing.user_id')
     follows_user = relationship('User', primaryjoin='User.user_id==UserFollowing.follows_user_id')
     follows_repository = relationship('Repository', order_by=lambda: sqlalchemy.func.lower(Repository.repo_name))
     @classmethod
     def get_repo_followers(cls, repo_id):
         return cls.query().filter(cls.follows_repository_id == repo_id)
 class CacheInvalidation(Base, BaseDbModel):
     __tablename__ = 'cache_invalidation'
     __table_args__ = (
         Index('key_idx', 'cache_key'),
         _table_args_default_dict,
+    )
     # cache_id, not used
     cache_id = Column(Integer(), primary_key=True)
     # cache_key as created by _get_cache_key
     cache_key = Column(Unicode(255), nullable=False, unique=True)
     # cache_args is a repo_name
     cache_args = Column(Unicode(255), nullable=False)
     # instance sets cache_active True when it is caching, other instances set
     # cache_active to False to indicate that this cache is invalid
     cache_active = Column(Boolean(), nullable=False, default=False)
     def __init__(self, cache_key, repo_name=''):
         self.cache_key = cache_key
         self.cache_args = repo_name
         self.cache_active = False
     def __repr__(self):
         return "<%s %s: %s=%s" % (
             self.__class__.__name__,
             self.cache_id, self.cache_key, self.cache_active)
     def _cache_key_partition(self):
         prefix, repo_name, suffix = self.cache_key.partition(self.cache_args)
         return prefix, repo_name, suffix
     def get_prefix(self):
         """
         get prefix that might have been used in _get_cache_key to
         generate self.cache_key. Only used for informational purposes
         in repo_edit.html.
         """
         # prefix, repo_name, suffix
         return self._cache_key_partition()[0]
     def get_suffix(self):
         """
         get suffix that might have been used in _get_cache_key to
         generate self.cache_key. Only used for informational purposes
         in repo_edit.html.
         """
         # prefix, repo_name, suffix
         return self._cache_key_partition()[2]
     @classmethod
     def clear_cache(cls):
         """
         Delete all cache keys from database.
         Should only be run when all instances are down and all entries thus stale.
         """
         cls.query().delete()
         Session().commit()
     @classmethod
     def _get_cache_key(cls, key):
         """
         Wrapper for generating a unique cache key for this instance and "key".
         key must / will start with a repo_name which will be stored in .cache_args .
         """
         prefix = kallithea.CONFIG.get('instance_id', '')
         return "%s%s" % (prefix, key)
     @classmethod
     def set_invalidate(cls, repo_name):
         """
         Mark all caches of a repo as invalid in the database.
         """
         inv_objs = Session().query(cls).filter(cls.cache_args == repo_name).all()
         log.debug('for repo %s got %s invalidation objects',
-                  safe_str(repo_name), inv_objs)
                   repo_name, inv_objs)
         for inv_obj in inv_objs:
             log.debug('marking %s key for invalidation based on repo_name=%s',
-                      inv_obj, safe_str(repo_name))
                       inv_obj, repo_name)
             Session().delete(inv_obj)
         Session().commit()
     @classmethod
     def test_and_set_valid(cls, repo_name, kind, valid_cache_keys=None):
         """
         Mark this cache key as active and currently cached.
         Return True if the existing cache registration still was valid.
         Return False to indicate that it had been invalidated and caches should be refreshed.
         """
         key = (repo_name + '_' + kind) if kind else repo_name
         cache_key = cls._get_cache_key(key)
         if valid_cache_keys and cache_key in valid_cache_keys:
             return True
         inv_obj = cls.query().filter(cls.cache_key == cache_key).scalar()
         if inv_obj is None:
             inv_obj = cls(cache_key, repo_name)
             Session().add(inv_obj)
         elif inv_obj.cache_active:
             return True
         inv_obj.cache_active = True
         try:
             Session().commit()
         except sqlalchemy.exc.IntegrityError:
             log.error('commit of CacheInvalidation failed - retrying')
             Session().rollback()
             inv_obj = cls.query().filter(cls.cache_key == cache_key).scalar()
             if inv_obj is None:
                 log.error('failed to create CacheInvalidation entry')
                 # TODO: fail badly?
             # else: TOCTOU - another thread added the key at the same time; no further action required
         return False
     @classmethod
     def get_valid_cache_keys(cls):
         """
         Return opaque object with information of which caches still are valid
         and can be used without checking for invalidation.
         """
         return set(inv_obj.cache_key for inv_obj in cls.query().filter(cls.cache_active).all())
 class ChangesetComment(Base, BaseDbModel):
     __tablename__ = 'changeset_comments'
     __table_args__ = (
         Index('cc_revision_idx', 'revision'),
         Index('cc_pull_request_id_idx', 'pull_request_id'),
         _table_args_default_dict,
+    )
     comment_id = Column(Integer(), primary_key=True)
     repo_id = Column(Integer(), ForeignKey('repositories.repo_id'), nullable=False)
     revision = Column(String(40), nullable=True)
     pull_request_id = Column(Integer(), ForeignKey('pull_requests.pull_request_id'), nullable=True)
     line_no = Column(Unicode(10), nullable=True)
     f_path = Column(Unicode(1000), nullable=True)
     author_id = Column('user_id', Integer(), ForeignKey('users.user_id'), nullable=False)
     text = Column(UnicodeText(), nullable=False)
     created_on = Column(DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
     modified_at = Column(DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
     author = relationship('User')
     repo = relationship('Repository')
     # status_change is frequently used directly in templates - make it a lazy
     # join to avoid fetching each related ChangesetStatus on demand.
     # There will only be one ChangesetStatus referencing each comment so the join will not explode.
     status_change = relationship('ChangesetStatus',
                                  cascade="all, delete-orphan", lazy='joined')
     pull_request = relationship('PullRequest')
     def url(self):
         anchor = "comment-%s" % self.comment_id
         import kallithea.lib.helpers as h
         if self.revision:
             return h.url('changeset_home', repo_name=self.repo.repo_name, revision=self.revision, anchor=anchor)
         elif self.pull_request_id is not None:
             return self.pull_request.url(anchor=anchor)
     def __json__(self):
         return dict(
             comment_id=self.comment_id,
             username=self.author.username,
             text=self.text,
+        )
     def deletable(self):
         return self.created_on > datetime.datetime.now() - datetime.timedelta(minutes=5)
 class ChangesetStatus(Base, BaseDbModel):
     __tablename__ = 'changeset_statuses'
     __table_args__ = (
         Index('cs_revision_idx', 'revision'),
         Index('cs_version_idx', 'version'),
         Index('cs_pull_request_id_idx', 'pull_request_id'),
         Index('cs_changeset_comment_id_idx', 'changeset_comment_id'),
         Index('cs_pull_request_id_user_id_version_idx', 'pull_request_id', 'user_id', 'version'),
         Index('cs_repo_id_pull_request_id_idx', 'repo_id', 'pull_request_id'),
         UniqueConstraint('repo_id', 'revision', 'version'),
         _table_args_default_dict,
+    )
     STATUS_NOT_REVIEWED = DEFAULT = 'not_reviewed'
     STATUS_APPROVED = 'approved'
     STATUS_REJECTED = 'rejected' # is shown as "Not approved" - TODO: change database content / scheme
     STATUS_UNDER_REVIEW = 'under_review'
     STATUSES = [
         (STATUS_NOT_REVIEWED, _("Not reviewed")),  # (no icon) and default
         (STATUS_UNDER_REVIEW, _("Under review")),
         (STATUS_REJECTED, _("Not approved")),
         (STATUS_APPROVED, _("Approved")),
+    ]
     STATUSES_DICT = dict(STATUSES)
     changeset_status_id = Column(Integer(), primary_key=True)
     repo_id = Column(Integer(), ForeignKey('repositories.repo_id'), nullable=False)
     user_id = Column(Integer(), ForeignKey('users.user_id'), nullable=False)
     revision = Column(String(40), nullable=True)
     status = Column(String(128), nullable=False, default=DEFAULT)
     comment_id = Column('changeset_comment_id', Integer(), ForeignKey('changeset_comments.comment_id'), nullable=False)
     modified_at = Column(DateTime(), nullable=False, default=datetime.datetime.now)
     version = Column(Integer(), nullable=False, default=0)
     pull_request_id = Column(Integer(), ForeignKey('pull_requests.pull_request_id'), nullable=True)
     author = relationship('User')
     repo = relationship('Repository')
     comment = relationship('ChangesetComment')
     pull_request = relationship('PullRequest')
     def __repr__(self):
         return "<%s %r by %r>" % (
             self.__class__.__name__,
             self.status, self.author
+        )
     @classmethod
     def get_status_lbl(cls, value):
         return cls.STATUSES_DICT.get(value)
     @property
     def status_lbl(self):
         return ChangesetStatus.get_status_lbl(self.status)
     def __json__(self):
         return dict(
             status=self.status,
             modified_at=self.modified_at.replace(microsecond=0),
             reviewer=self.author.username,
+            )
 class PullRequest(Base, BaseDbModel):
     __tablename__ = 'pull_requests'
     __table_args__ = (
         Index('pr_org_repo_id_idx', 'org_repo_id'),
         Index('pr_other_repo_id_idx', 'other_repo_id'),
         _table_args_default_dict,
+    )
     # values for .status
     STATUS_NEW = u'new'
     STATUS_CLOSED = u'closed'
     pull_request_id = Column(Integer(), primary_key=True)
     title = Column(Unicode(255), nullable=False)
     description = Column(UnicodeText(), nullable=False)
     status = Column(Unicode(255), nullable=False, default=STATUS_NEW) # only for closedness, not approve/reject/etc
     created_on = Column(DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
     updated_on = Column(DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
     owner_id = Column('user_id', Integer(), ForeignKey('users.user_id'), nullable=False)
     _revisions = Column('revisions', UnicodeText(), nullable=False)
     org_repo_id = Column(Integer(), ForeignKey('repositories.repo_id'), nullable=False)
     org_ref = Column(Unicode(255), nullable=False)
     other_repo_id = Column(Integer(), ForeignKey('repositories.repo_id'), nullable=False)
     other_ref = Column(Unicode(255), nullable=False)
     @hybrid_property
     def revisions(self):
         return self._revisions.split(':')
     @revisions.setter
     def revisions(self, val):
         self._revisions = ':'.join(val)
     @property
     def org_ref_parts(self):
         return self.org_ref.split(':')
@@ @@ -2328,225 +2328,225 @@ class PullRequest(Base, BaseDbModel): @@
         """Like .reviewers, but actually returning the users"""
         return User.query() \
             .join(PullRequestReviewer) \
             .filter(PullRequestReviewer.pull_request == self) \
             .order_by(PullRequestReviewer.pull_request_reviewers_id) \
             .all()
     def is_closed(self):
         return self.status == self.STATUS_CLOSED
     def user_review_status(self, user_id):
         """Return the user's latest status votes on PR"""
         # note: no filtering on repo - that would be redundant
         status = ChangesetStatus.query() \
             .filter(ChangesetStatus.pull_request == self) \
             .filter(ChangesetStatus.user_id == user_id) \
             .order_by(ChangesetStatus.version) \
             .first()
         return str(status.status) if status else ''
     @classmethod
     def make_nice_id(cls, pull_request_id):
         '''Return pull request id nicely formatted for displaying'''
         return '#%s' % pull_request_id
     def nice_id(self):
         '''Return the id of this pull request, nicely formatted for displaying'''
         return self.make_nice_id(self.pull_request_id)
     def get_api_data(self):
         return self.__json__()
     def __json__(self):
         clone_uri_tmpl = kallithea.CONFIG.get('clone_uri_tmpl') or Repository.DEFAULT_CLONE_URI
         return dict(
             pull_request_id=self.pull_request_id,
             url=self.url(),
             reviewers=self.reviewers,
             revisions=self.revisions,
             owner=self.owner.username,
             title=self.title,
             description=self.description,
             org_repo_url=self.org_repo.clone_url(clone_uri_tmpl=clone_uri_tmpl),
             org_ref_parts=self.org_ref_parts,
             other_ref_parts=self.other_ref_parts,
             status=self.status,
             comments=self.comments,
             statuses=self.statuses,
+        )
     def url(self, **kwargs):
         canonical = kwargs.pop('canonical', None)
         import kallithea.lib.helpers as h
         b = self.org_ref_parts[1]
         if b != self.other_ref_parts[1]:
             s = '/_/' + b
         else:
             s = '/_/' + self.title
         kwargs['extra'] = urlreadable(s)
         if canonical:
             return h.canonical_url('pullrequest_show', repo_name=self.other_repo.repo_name,
                                    pull_request_id=self.pull_request_id, **kwargs)
         return h.url('pullrequest_show', repo_name=self.other_repo.repo_name,
                      pull_request_id=self.pull_request_id, **kwargs)
 class PullRequestReviewer(Base, BaseDbModel):
     __tablename__ = 'pull_request_reviewers'
     __table_args__ = (
         Index('pull_request_reviewers_user_id_idx', 'user_id'),
         _table_args_default_dict,
+    )
     def __init__(self, user=None, pull_request=None):
         self.user = user
         self.pull_request = pull_request
     pull_request_reviewers_id = Column('pull_requests_reviewers_id', Integer(), primary_key=True)
     pull_request_id = Column(Integer(), ForeignKey('pull_requests.pull_request_id'), nullable=False)
     user_id = Column(Integer(), ForeignKey('users.user_id'), nullable=False)
     user = relationship('User')
     pull_request = relationship('PullRequest')
     def __json__(self):
         return dict(
             username=self.user.username if self.user else None,
+        )
 class Notification(object):
     __tablename__ = 'notifications'
 class UserNotification(object):
     __tablename__ = 'user_to_notification'
 class Gist(Base, BaseDbModel):
     __tablename__ = 'gists'
     __table_args__ = (
         Index('g_gist_access_id_idx', 'gist_access_id'),
         Index('g_created_on_idx', 'created_on'),
         _table_args_default_dict,
+    )
     GIST_PUBLIC = u'public'
     GIST_PRIVATE = u'private'
     DEFAULT_FILENAME = u'gistfile1.txt'
     gist_id = Column(Integer(), primary_key=True)
     gist_access_id = Column(Unicode(250), nullable=False)
     gist_description = Column(UnicodeText(), nullable=False)
     owner_id = Column('user_id', Integer(), ForeignKey('users.user_id'), nullable=False)
     gist_expires = Column(Float(53), nullable=False)
     gist_type = Column(Unicode(128), nullable=False)
     created_on = Column(DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
     modified_at = Column(DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
     owner = relationship('User')
     @hybrid_property
     def is_expired(self):
         return (self.gist_expires != -1) & (time.time() > self.gist_expires)
     def __repr__(self):
         return "<%s %s %s>" % (
             self.__class__.__name__,
             self.gist_type, self.gist_access_id)
     @classmethod
     def guess_instance(cls, value):
         return super(Gist, cls).guess_instance(value, Gist.get_by_access_id)
     @classmethod
     def get_or_404(cls, id_):
         res = cls.query().filter(cls.gist_access_id == id_).scalar()
         if res is None:
             raise HTTPNotFound
         return res
     @classmethod
     def get_by_access_id(cls, gist_access_id):
         return cls.query().filter(cls.gist_access_id == gist_access_id).scalar()
     def gist_url(self):
         alias_url = kallithea.CONFIG.get('gist_alias_url')
         if alias_url:
             return alias_url.replace('{gistid}', self.gist_access_id)
         import kallithea.lib.helpers as h
         return h.canonical_url('gist', gist_id=self.gist_access_id)
     @classmethod
     def base_path(cls):
         """
         Returns base path where all gists are stored
         :param cls:
         """
         from kallithea.model.gist import GIST_STORE_LOC
         q = Session().query(Ui) \
             .filter(Ui.ui_key == URL_SEP)
         q = q.options(FromCache("sql_cache_short", "repository_repo_path"))
         return os.path.join(q.one().ui_value, GIST_STORE_LOC)
     def get_api_data(self):
         """
         Common function for generating gist related data for API
         """
         gist = self
         data = dict(
             gist_id=gist.gist_id,
             type=gist.gist_type,
             access_id=gist.gist_access_id,
             description=gist.gist_description,
             url=gist.gist_url(),
             expires=gist.gist_expires,
             created_on=gist.created_on,
+        )
         return data
     def __json__(self):
         data = dict(
+        )
         data.update(self.get_api_data())
         return data
     ## SCM functions
     @property
     def scm_instance(self):
         from kallithea.lib.vcs import get_repo
         base_path = self.base_path()
-        return get_repo(os.path.join(safe_str(base_path), safe_str(self.gist_access_id)))
         return get_repo(os.path.join(base_path, self.gist_access_id))
 class UserSshKeys(Base, BaseDbModel):
     __tablename__ = 'user_ssh_keys'
     __table_args__ = (
         Index('usk_fingerprint_idx', 'fingerprint'),
         UniqueConstraint('fingerprint'),
         _table_args_default_dict
+    )
     __mapper_args__ = {}
     user_ssh_key_id = Column(Integer(), primary_key=True)
     user_id = Column(Integer(), ForeignKey('users.user_id'), nullable=False)
     _public_key = Column('public_key', UnicodeText(), nullable=False)
     description = Column(UnicodeText(), nullable=False)
     fingerprint = Column(String(255), nullable=False, unique=True)
     created_on = Column(DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
     last_seen = Column(DateTime(timezone=False), nullable=True)
     user = relationship('User')
     @property
     def public_key(self):
         return self._public_key
     @public_key.setter
     def public_key(self, full_key):
         # the full public key is too long to be suitable as database key - instead,
         # use fingerprints similar to 'ssh-keygen -E sha256 -lf ~/.ssh/id_rsa.pub'
         self._public_key = full_key
         enc_key = safe_bytes(full_key.split(" ")[1])
         self.fingerprint = base64.b64encode(hashlib.sha256(base64.b64decode(enc_key)).digest()).replace(b'\n', b'').rstrip(b'=').decode()

kallithea/model/repo.py

➞

Show inline comments

 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
+#
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.model.repo
 ~~~~~~~~~~~~~~~~~~~~
 Repository model for kallithea
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Jun 5, 2010
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 import logging
 import os
 import shutil
 import traceback
 from datetime import datetime
 import kallithea.lib.utils2
 from kallithea.lib import helpers as h
 from kallithea.lib.auth import HasRepoPermissionLevel, HasUserGroupPermissionLevel
 from kallithea.lib.caching_query import FromCache
 from kallithea.lib.exceptions import AttachedForksError
 from kallithea.lib.hooks import log_delete_repository
 from kallithea.lib.utils import is_valid_repo_uri, make_ui
-from kallithea.lib.utils2 import LazyProperty, get_current_authuser, obfuscate_url_pw, remove_prefix, safe_str
 from kallithea.lib.utils2 import LazyProperty, get_current_authuser, obfuscate_url_pw, remove_prefix
 from kallithea.lib.vcs.backends import get_backend
 from kallithea.model.db import (
     Permission, RepoGroup, Repository, RepositoryField, Session, Statistics, Ui, User, UserGroup, UserGroupRepoGroupToPerm, UserGroupRepoToPerm, UserRepoGroupToPerm, UserRepoToPerm)
 log = logging.getLogger(__name__)
 class RepoModel(object):
     URL_SEPARATOR = Repository.url_sep()
     def _create_default_perms(self, repository, private):
         # create default permission
         default = 'repository.read'
         def_user = User.get_default_user()
         for p in def_user.user_perms:
             if p.permission.permission_name.startswith('repository.'):
                 default = p.permission.permission_name
                 break
         default_perm = 'repository.none' if private else default
         repo_to_perm = UserRepoToPerm()
         repo_to_perm.permission = Permission.get_by_key(default_perm)
         repo_to_perm.repository = repository
         repo_to_perm.user_id = def_user.user_id
         Session().add(repo_to_perm)
         return repo_to_perm
     @LazyProperty
     def repos_path(self):
         """
         Gets the repositories root path from database
         """
         q = Ui.query().filter(Ui.ui_key == '/').one()
         return q.ui_value
     def get(self, repo_id, cache=False):
         repo = Repository.query() \
             .filter(Repository.repo_id == repo_id)
         if cache:
             repo = repo.options(FromCache("sql_cache_short",
                                           "get_repo_%s" % repo_id))
         return repo.scalar()
     def get_repo(self, repository):
         return Repository.guess_instance(repository)
     def get_by_repo_name(self, repo_name, cache=False):
         repo = Repository.query() \
             .filter(Repository.repo_name == repo_name)
         if cache:
             repo = repo.options(FromCache("sql_cache_short",
                                           "get_repo_%s" % repo_name))
         return repo.scalar()
     def get_all_user_repos(self, user):
         """
         Gets all repositories that user have at least read access
         :param user:
         """
         from kallithea.lib.auth import AuthUser
         user = User.guess_instance(user)
         repos = AuthUser(dbuser=user).permissions['repositories']
         access_check = lambda r: r[1] in ['repository.read',
                                           'repository.write',
                                           'repository.admin']
         repos = [x[0] for x in filter(access_check, repos.items())]
         return Repository.query().filter(Repository.repo_name.in_(repos))
     @classmethod
     def _render_datatable(cls, tmpl, *args, **kwargs):
         from tg import tmpl_context as c, request, app_globals
         from tg.i18n import ugettext as _
         _tmpl_lookup = app_globals.mako_lookup
         template = _tmpl_lookup.get_template('data_table/_dt_elements.html')
         tmpl = template.get_def(tmpl)
         kwargs.update(dict(_=_, h=h, c=c, request=request))
         return tmpl.render_unicode(*args, **kwargs)
     def get_repos_as_dict(self, repos_list, repo_groups_list=None,
                           admin=False,
                           short_name=False):
         """Return repository list for use by DataTable.
         repos_list: list of repositories - but will be filtered for read permission.
         repo_groups_list: added at top of list without permission check.
         admin: return data for action column.
         """
         _render = self._render_datatable
         from tg import tmpl_context as c
         def repo_lnk(name, rtype, rstate, private, fork_of):
             return _render('repo_name', name, rtype, rstate, private, fork_of,
                            short_name=short_name)
         def last_change(last_change):
             return _render("last_change", last_change)
         def rss_lnk(repo_name):
             return _render("rss", repo_name)
         def atom_lnk(repo_name):
             return _render("atom", repo_name)
         def last_rev(repo_name, cs_cache):
             return _render('revision', repo_name, cs_cache.get('revision'),
                            cs_cache.get('raw_id'), cs_cache.get('author'),
                            cs_cache.get('message'))
         def desc(desc):
             return h.urlify_text(desc, truncate=80, stylize=c.visual.stylify_metalabels)
         def state(repo_state):
             return _render("repo_state", repo_state)
         def repo_actions(repo_name):
             return _render('repo_actions', repo_name)
         def owner_actions(owner_id, username):
             return _render('user_name', owner_id, username)
         repos_data = []
         for gr in repo_groups_list or []:
             repos_data.append(dict(
                 raw_name='\0' + gr.name, # sort before repositories
                 just_name=gr.name,
                 name=_render('group_name_html', group_name=gr.group_name, name=gr.name),
                 desc=gr.group_description))
         for repo in repos_list:
             if not HasRepoPermissionLevel('read')(repo.repo_name, 'get_repos_as_dict check'):
                 continue
             cs_cache = repo.changeset_cache
             row = {
                 "raw_name": repo.repo_name,
                 "just_name": repo.just_name,
                 "name": repo_lnk(repo.repo_name, repo.repo_type,
                                  repo.repo_state, repo.private, repo.fork),
                 "last_change_iso": repo.last_db_change.isoformat(),
                 "last_change": last_change(repo.last_db_change),
                 "last_changeset": last_rev(repo.repo_name, cs_cache),
                 "last_rev_raw": cs_cache.get('revision'),
                 "desc": desc(repo.description),
                 "owner": h.person(repo.owner),
                 "state": state(repo.repo_state),
                 "rss": rss_lnk(repo.repo_name),
                 "atom": atom_lnk(repo.repo_name),
+            }
             if admin:
                 row.update({
                     "action": repo_actions(repo.repo_name),
                     "owner": owner_actions(repo.owner_id,
                                            h.person(repo.owner))
                 })
             repos_data.append(row)
         return {
             "sort": "name",
             "dir": "asc",
             "records": repos_data
+        }
     def _get_defaults(self, repo_name):
         """
         Gets information about repository, and returns a dict for
         usage in forms
         :param repo_name:
         """
         repo_info = Repository.get_by_repo_name(repo_name)
         if repo_info is None:
             return None
         defaults = repo_info.get_dict()
         defaults['repo_name'] = repo_info.just_name
         defaults['repo_group'] = repo_info.group_id
         for strip, k in [(0, 'repo_type'), (1, 'repo_enable_downloads'),
                          (1, 'repo_description'),
                          (1, 'repo_landing_rev'), (0, 'clone_uri'),
@@ @@ -452,267 +452,266 @@ class RepoModel(object): @@
                 self.grant_user_permission(
                     repo=repo, user=member, perm=perm
+                )
             else:
                 # check if we have permissions to alter this usergroup's access
                 if not check_perms or HasUserGroupPermissionLevel('read')(member):
                     self.grant_user_group_permission(
                         repo=repo, group_name=member, perm=perm
+                    )
     def create_fork(self, form_data, cur_user):
         """
         Simple wrapper into executing celery task for fork creation
         :param form_data:
         :param cur_user:
         """
         from kallithea.lib.celerylib import tasks
         return tasks.create_repo_fork(form_data, cur_user)
     def delete(self, repo, forks=None, fs_remove=True, cur_user=None):
         """
         Delete given repository, forks parameter defines what do do with
         attached forks. Throws AttachedForksError if deleted repo has attached
         forks
         :param repo:
         :param forks: str 'delete' or 'detach'
         :param fs_remove: remove(archive) repo from filesystem
         """
         if not cur_user:
             cur_user = getattr(get_current_authuser(), 'username', None)
         repo = Repository.guess_instance(repo)
         if repo is not None:
             if forks == 'detach':
                 for r in repo.forks:
                     r.fork = None
             elif forks == 'delete':
                 for r in repo.forks:
                     self.delete(r, forks='delete')
             elif [f for f in repo.forks]:
                 raise AttachedForksError()
             old_repo_dict = repo.get_dict()
             try:
                 Session().delete(repo)
                 if fs_remove:
                     self._delete_filesystem_repo(repo)
                 else:
                     log.debug('skipping removal from filesystem')
                 log_delete_repository(old_repo_dict,
                                       deleted_by=cur_user)
             except Exception:
                 log.error(traceback.format_exc())
                 raise
     def grant_user_permission(self, repo, user, perm):
         """
         Grant permission for user on given repository, or update existing one
         if found
         :param repo: Instance of Repository, repository_id, or repository name
         :param user: Instance of User, user_id or username
         :param perm: Instance of Permission, or permission_name
         """
         user = User.guess_instance(user)
         repo = Repository.guess_instance(repo)
         permission = Permission.guess_instance(perm)
         # check if we have that permission already
         obj = UserRepoToPerm.query() \
             .filter(UserRepoToPerm.user == user) \
             .filter(UserRepoToPerm.repository == repo) \
             .scalar()
         if obj is None:
             # create new !
             obj = UserRepoToPerm()
             Session().add(obj)
         obj.repository = repo
         obj.user = user
         obj.permission = permission
         log.debug('Granted perm %s to %s on %s', perm, user, repo)
         return obj
     def revoke_user_permission(self, repo, user):
         """
         Revoke permission for user on given repository
         :param repo: Instance of Repository, repository_id, or repository name
         :param user: Instance of User, user_id or username
         """
         user = User.guess_instance(user)
         repo = Repository.guess_instance(repo)
         obj = UserRepoToPerm.query() \
             .filter(UserRepoToPerm.repository == repo) \
             .filter(UserRepoToPerm.user == user) \
             .scalar()
         if obj is not None:
             Session().delete(obj)
             log.debug('Revoked perm on %s on %s', repo, user)
     def grant_user_group_permission(self, repo, group_name, perm):
         """
         Grant permission for user group on given repository, or update
         existing one if found
         :param repo: Instance of Repository, repository_id, or repository name
         :param group_name: Instance of UserGroup, users_group_id,
             or user group name
         :param perm: Instance of Permission, or permission_name
         """
         repo = Repository.guess_instance(repo)
         group_name = UserGroup.guess_instance(group_name)
         permission = Permission.guess_instance(perm)
         # check if we have that permission already
         obj = UserGroupRepoToPerm.query() \
             .filter(UserGroupRepoToPerm.users_group == group_name) \
             .filter(UserGroupRepoToPerm.repository == repo) \
             .scalar()
         if obj is None:
             # create new
             obj = UserGroupRepoToPerm()
             Session().add(obj)
         obj.repository = repo
         obj.users_group = group_name
         obj.permission = permission
         log.debug('Granted perm %s to %s on %s', perm, group_name, repo)
         return obj
     def revoke_user_group_permission(self, repo, group_name):
         """
         Revoke permission for user group on given repository
         :param repo: Instance of Repository, repository_id, or repository name
         :param group_name: Instance of UserGroup, users_group_id,
             or user group name
         """
         repo = Repository.guess_instance(repo)
         group_name = UserGroup.guess_instance(group_name)
         obj = UserGroupRepoToPerm.query() \
             .filter(UserGroupRepoToPerm.repository == repo) \
             .filter(UserGroupRepoToPerm.users_group == group_name) \
             .scalar()
         if obj is not None:
             Session().delete(obj)
             log.debug('Revoked perm to %s on %s', repo, group_name)
     def delete_stats(self, repo_name):
         """
         removes stats for given repo
         :param repo_name:
         """
         repo = Repository.guess_instance(repo_name)
         try:
             obj = Statistics.query() \
                 .filter(Statistics.repository == repo).scalar()
             if obj is not None:
                 Session().delete(obj)
         except Exception:
             log.error(traceback.format_exc())
             raise
     def _create_filesystem_repo(self, repo_name, repo_type, repo_group,
                                 clone_uri=None, repo_store_location=None):
         """
         Makes repository on filesystem. Operation is group aware, meaning that it will create
         a repository within a group, and alter the paths accordingly to the group location.
         Note: clone_uri is low level and not validated - it might be a file system path used for validated cloning
         """
         from kallithea.lib.utils import is_valid_repo, is_valid_repo_group
         from kallithea.model.scm import ScmModel
         if '/' in repo_name:
             raise ValueError('repo_name must not contain groups got `%s`' % repo_name)
         if isinstance(repo_group, RepoGroup):
             new_parent_path = os.sep.join(repo_group.full_path_splitted)
         else:
             new_parent_path = repo_group or ''
         if repo_store_location:
             _paths = [repo_store_location]
         else:
             _paths = [self.repos_path, new_parent_path, repo_name]
             # we need to make it str for mercurial
         repo_path = os.path.join(*(safe_str(x) for x in _paths))
         repo_path = os.path.join(*_paths)
         # check if this path is not a repository
         if is_valid_repo(repo_path, self.repos_path):
             raise Exception('This path %s is a valid repository' % repo_path)
         # check if this path is a group
         if is_valid_repo_group(repo_path, self.repos_path):
             raise Exception('This path %s is a valid group' % repo_path)
         log.info('creating repo %s in %s from url: `%s`',
             repo_name, repo_path,
             obfuscate_url_pw(clone_uri))
         backend = get_backend(repo_type)
         if repo_type == 'hg':
             baseui = make_ui()
             # patch and reset hooks section of UI config to not run any
             # hooks on creating remote repo
             for k, v in baseui.configitems('hooks'):
                 baseui.setconfig('hooks', k, None)
             repo = backend(repo_path, create=True, src_url=clone_uri, baseui=baseui)
         elif repo_type == 'git':
             repo = backend(repo_path, create=True, src_url=clone_uri, bare=True)
             # add kallithea hook into this repo
             ScmModel().install_git_hooks(repo=repo)
         else:
             raise Exception('Not supported repo_type %s expected hg/git' % repo_type)
         log.debug('Created repo %s with %s backend',
                   repo_name, repo_type)
         return repo
     def _rename_filesystem_repo(self, old, new):
         """
         renames repository on filesystem
         :param old: old name
         :param new: new name
         """
         log.info('renaming repo from %s to %s', old, new)
         old_path = safe_str(os.path.join(self.repos_path, old))
         new_path = safe_str(os.path.join(self.repos_path, new))
         old_path = os.path.join(self.repos_path, old)
         new_path = os.path.join(self.repos_path, new)
         if os.path.isdir(new_path):
             raise Exception(
                 'Was trying to rename to already existing dir %s' % new_path
+            )
         shutil.move(old_path, new_path)
     def _delete_filesystem_repo(self, repo):
         """
         removes repo from filesystem, the removal is actually done by
         renaming dir to a 'rm__*' prefix which Kallithea will skip.
         It can be undeleted later by reverting the rename.
         :param repo: repo object
         """
-        rm_path = safe_str(os.path.join(self.repos_path, repo.repo_name))
         rm_path = os.path.join(self.repos_path, repo.repo_name)
         log.info("Removing %s", rm_path)
         _now = datetime.now()
         _ms = str(_now.microsecond).rjust(6, '0')
         _d = 'rm__%s__%s' % (_now.strftime('%Y%m%d_%H%M%S_' + _ms),
                              repo.just_name)
         if repo.group:
             args = repo.group.full_path_splitted + [_d]
             _d = os.path.join(*args)
         if os.path.exists(rm_path):
-            shutil.move(rm_path, safe_str(os.path.join(self.repos_path, _d)))
             shutil.move(rm_path, os.path.join(self.repos_path, _d))
         else:
             log.error("Can't find repo to delete in %r", rm_path)

kallithea/model/scm.py

➞

Show inline comments

 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
+#
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.model.scm
 ~~~~~~~~~~~~~~~~~~~
 Scm model for Kallithea
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Apr 9, 2010
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 import logging
 import os
 import posixpath
 import re
 import sys
 import traceback
 import pkg_resources
 from tg.i18n import ugettext as _
 import kallithea
 from kallithea import BACKENDS
 from kallithea.lib.auth import HasPermissionAny, HasRepoGroupPermissionLevel, HasRepoPermissionLevel, HasUserGroupPermissionLevel
 from kallithea.lib.exceptions import IMCCommitError, NonRelativePathError
 from kallithea.lib.hooks import process_pushed_raw_ids
 from kallithea.lib.utils import action_logger, get_filesystem_repos, make_ui
-from kallithea.lib.utils2 import safe_bytes, safe_str, set_hook_environment
 from kallithea.lib.utils2 import safe_bytes, set_hook_environment
 from kallithea.lib.vcs import get_backend
 from kallithea.lib.vcs.backends.base import EmptyChangeset
 from kallithea.lib.vcs.exceptions import RepositoryError
 from kallithea.lib.vcs.nodes import FileNode
 from kallithea.lib.vcs.utils.lazy import LazyProperty
 from kallithea.model.db import PullRequest, RepoGroup, Repository, Session, Ui, User, UserFollowing, UserLog
 log = logging.getLogger(__name__)
 class UserTemp(object):
     def __init__(self, user_id):
         self.user_id = user_id
     def __repr__(self):
         return "<%s('id:%s')>" % (self.__class__.__name__, self.user_id)
 class RepoTemp(object):
     def __init__(self, repo_id):
         self.repo_id = repo_id
     def __repr__(self):
         return "<%s('id:%s')>" % (self.__class__.__name__, self.repo_id)
 class _PermCheckIterator(object):
     def __init__(self, obj_list, obj_attr, perm_set, perm_checker, extra_kwargs=None):
         """
         Creates iterator from given list of objects, additionally
         checking permission for them from perm_set var
         :param obj_list: list of db objects
         :param obj_attr: attribute of object to pass into perm_checker
         :param perm_set: list of permissions to check
         :param perm_checker: callable to check permissions against
         """
         self.obj_list = obj_list
         self.obj_attr = obj_attr
         self.perm_set = perm_set
         self.perm_checker = perm_checker
         self.extra_kwargs = extra_kwargs or {}
     def __len__(self):
         return len(self.obj_list)
     def __repr__(self):
         return '<%s (%s)>' % (self.__class__.__name__, self.__len__())
     def __iter__(self):
         for db_obj in self.obj_list:
             # check permission at this level
             name = getattr(db_obj, self.obj_attr, None)
             if not self.perm_checker(*self.perm_set)(
                     name, self.__class__.__name__, **self.extra_kwargs):
                 continue
             yield db_obj
 class RepoList(_PermCheckIterator):
     def __init__(self, db_repo_list, perm_level, extra_kwargs=None):
         super(RepoList, self).__init__(obj_list=db_repo_list,
                     obj_attr='repo_name', perm_set=[perm_level],
                     perm_checker=HasRepoPermissionLevel,
                     extra_kwargs=extra_kwargs)
 class RepoGroupList(_PermCheckIterator):
     def __init__(self, db_repo_group_list, perm_level, extra_kwargs=None):
         super(RepoGroupList, self).__init__(obj_list=db_repo_group_list,
                     obj_attr='group_name', perm_set=[perm_level],
                     perm_checker=HasRepoGroupPermissionLevel,
                     extra_kwargs=extra_kwargs)
 class UserGroupList(_PermCheckIterator):
     def __init__(self, db_user_group_list, perm_level, extra_kwargs=None):
         super(UserGroupList, self).__init__(obj_list=db_user_group_list,
                     obj_attr='users_group_name', perm_set=[perm_level],
                     perm_checker=HasUserGroupPermissionLevel,
                     extra_kwargs=extra_kwargs)
 class ScmModel(object):
     """
     Generic Scm Model
     """
     def __get_repo(self, instance):
         cls = Repository
         if isinstance(instance, cls):
             return instance
         elif isinstance(instance, int):
             return cls.get(instance)
         elif isinstance(instance, str):
             if instance.isdigit():
                 return cls.get(int(instance))
             return cls.get_by_repo_name(instance)
         elif instance is not None:
             raise Exception('given object must be int, basestr or Instance'
                             ' of %s got %s' % (type(cls), type(instance)))
     @LazyProperty
     def repos_path(self):
         """
         Gets the repositories root path from database
         """
         q = Ui.query().filter(Ui.ui_key == '/').one()
         return q.ui_value
     def repo_scan(self, repos_path=None):
         """
         Listing of repositories in given path. This path should not be a
         repository itself. Return a dictionary of repository objects mapping to
         vcs instances.
         :param repos_path: path to directory containing repositories
         """
         if repos_path is None:
             repos_path = self.repos_path
         log.info('scanning for repositories in %s', repos_path)
         baseui = make_ui()
         repos = {}
         for name, path in get_filesystem_repos(repos_path):
             # name need to be decomposed and put back together using the /
             # since this is internal storage separator for kallithea
             name = Repository.normalize_repo_name(name)
             try:
                 if name in repos:
                     raise RepositoryError('Duplicate repository name %s '
                                           'found in %s' % (name, path))
                 else:
                     klass = get_backend(path[0])
                     if path[0] == 'hg' and path[0] in BACKENDS:
-                        repos[name] = klass(safe_str(path[1]), baseui=baseui)
                         repos[name] = klass(path[1], baseui=baseui)
                     if path[0] == 'git' and path[0] in BACKENDS:
                         repos[name] = klass(path[1])
             except OSError:
                 continue
         log.debug('found %s paths with repositories', len(repos))
         return repos
     def get_repos(self, repos):
         """Return the repos the user has access to"""
         return RepoList(repos, perm_level='read')
     def get_repo_groups(self, groups=None):
         """Return the repo groups the user has access to
         If no groups are specified, use top level groups.
         """
         if groups is None:
             groups = RepoGroup.query() \
                 .filter(RepoGroup.parent_group_id == None).all()
         return RepoGroupList(groups, perm_level='read')
     def mark_for_invalidation(self, repo_name):
         """
         Mark caches of this repo invalid in the database.
         :param repo_name: the repo for which caches should be marked invalid
         """
         log.debug("Marking %s as invalidated and update cache", repo_name)
         repo = Repository.get_by_repo_name(repo_name)
         if repo is not None:
             repo.set_invalidate()
             repo.update_changeset_cache()
     def toggle_following_repo(self, follow_repo_id, user_id):
         f = UserFollowing.query() \
             .filter(UserFollowing.follows_repository_id == follow_repo_id) \
             .filter(UserFollowing.user_id == user_id).scalar()
         if f is not None:
             try:
                 Session().delete(f)
                 action_logger(UserTemp(user_id),
                               'stopped_following_repo',
                               RepoTemp(follow_repo_id))
                 return
             except Exception:
                 log.error(traceback.format_exc())
                 raise
         try:
             f = UserFollowing()
             f.user_id = user_id
             f.follows_repository_id = follow_repo_id
             Session().add(f)
             action_logger(UserTemp(user_id),
                           'started_following_repo',
                           RepoTemp(follow_repo_id))
         except Exception:
             log.error(traceback.format_exc())
             raise
     def toggle_following_user(self, follow_user_id, user_id):
         f = UserFollowing.query() \
             .filter(UserFollowing.follows_user_id == follow_user_id) \
             .filter(UserFollowing.user_id == user_id).scalar()
         if f is not None:
             try:
                 Session().delete(f)
                 return
             except Exception:
                 log.error(traceback.format_exc())
                 raise
         try:
             f = UserFollowing()
             f.user_id = user_id
             f.follows_user_id = follow_user_id
             Session().add(f)
         except Exception:
             log.error(traceback.format_exc())
             raise
     def is_following_repo(self, repo_name, user_id):
         r = Repository.query() \
             .filter(Repository.repo_name == repo_name).scalar()
         f = UserFollowing.query() \
             .filter(UserFollowing.follows_repository == r) \
             .filter(UserFollowing.user_id == user_id).scalar()
         return f is not None
     def is_following_user(self, username, user_id):
         u = User.get_by_username(username)
         f = UserFollowing.query() \
             .filter(UserFollowing.follows_user == u) \
             .filter(UserFollowing.user_id == user_id).scalar()
         return f is not None
     def get_followers(self, repo):
         repo = Repository.guess_instance(repo)
         return UserFollowing.query() \
                 .filter(UserFollowing.follows_repository == repo).count()
     def get_forks(self, repo):
         repo = Repository.guess_instance(repo)
         return Repository.query() \
                 .filter(Repository.fork == repo).count()
     def get_pull_requests(self, repo):
         repo = Repository.guess_instance(repo)
         return PullRequest.query() \
                 .filter(PullRequest.other_repo == repo) \
                 .filter(PullRequest.status != PullRequest.STATUS_CLOSED).count()
     def mark_as_fork(self, repo, fork, user):
         repo = self.__get_repo(repo)
         fork = self.__get_repo(fork)
         if fork and repo.repo_id == fork.repo_id:
             raise Exception("Cannot set repository as fork of itself")
         if fork and repo.repo_type != fork.repo_type:
             raise RepositoryError("Cannot set repository as fork of repository with other type")
         repo.fork = fork
         return repo
     def _handle_push(self, repo, username, ip_addr, action, repo_name, revisions):
         """
         Handle that the repository has changed.
         Adds an action log entry with the new revisions, and the head revision
         cache and in-memory caches are invalidated/updated.
         :param username: username who pushes
         :param action: push/push_local/push_remote
         :param repo_name: name of repo
         :param revisions: list of revisions that we pushed
         """
         set_hook_environment(username, ip_addr, repo_name, repo_alias=repo.alias, action=action)
         process_pushed_raw_ids(revisions) # also calls mark_for_invalidation
     def _get_IMC_module(self, scm_type):
         """
         Returns InMemoryCommit class based on scm_type
         :param scm_type:
         """
         if scm_type == 'hg':
             from kallithea.lib.vcs.backends.hg import MercurialInMemoryChangeset
             return MercurialInMemoryChangeset
         if scm_type == 'git':
             from kallithea.lib.vcs.backends.git import GitInMemoryChangeset
             return GitInMemoryChangeset
         raise Exception('Invalid scm_type, must be one of hg,git got %s'
                         % (scm_type,))
     def pull_changes(self, repo, username, ip_addr, clone_uri=None):
         """
         Pull from "clone URL" or fork origin.
         """
         dbrepo = self.__get_repo(repo)
         if clone_uri is None:
             clone_uri = dbrepo.clone_uri or dbrepo.fork and dbrepo.fork.repo_full_path
         if not clone_uri:
             raise Exception("This repository doesn't have a clone uri")
         repo = dbrepo.scm_instance
         repo_name = dbrepo.repo_name
         try:
             if repo.alias == 'git':
                 repo.fetch(clone_uri)
                 # git doesn't really have something like post-fetch action
                 # we fake that now.
                 # TODO: extract fetched revisions ... somehow ...
                 self._handle_push(repo,
                                   username=username,
                                   ip_addr=ip_addr,
                                   action='push_remote',
                                   repo_name=repo_name,
                                   revisions=[])
             else:
                 set_hook_environment(username, ip_addr, dbrepo.repo_name,
                                            repo.alias, action='push_remote')
                 repo.pull(clone_uri)
         except Exception:
             log.error(traceback.format_exc())
             raise
     def commit_change(self, repo, repo_name, cs, user, ip_addr, author, message,
                       content, f_path):
         """
         Commit a change to a single file
         :param repo: a db_repo.scm_instance
         """
         user = User.guess_instance(user)
         IMC = self._get_IMC_module(repo.alias)
         # decoding here will force that we have proper encoded values
         # in any other case this will throw exceptions and deny commit
         content = safe_str(content)
         path = safe_str(f_path)
         imc = IMC(repo)
-        imc.change(FileNode(path, content, mode=cs.get_file_mode(f_path)))
+        imc.change(FileNode(f_path, content, mode=cs.get_file_mode(f_path)))
         try:
             tip = imc.commit(message=message, author=author,
                              parents=[cs], branch=cs.branch)
         except Exception as e:
             log.error(traceback.format_exc())
             # clear caches - we also want a fresh object if commit fails
             self.mark_for_invalidation(repo_name)
             raise IMCCommitError(str(e))
         self._handle_push(repo,
                           username=user.username,
                           ip_addr=ip_addr,
                           action='push_local',
                           repo_name=repo_name,
                           revisions=[tip.raw_id])
         return tip
     def _sanitize_path(self, f_path):
         if f_path.startswith('/') or f_path.startswith('.') or '../' in f_path:
             raise NonRelativePathError('%s is not an relative path' % f_path)
         if f_path:
             f_path = posixpath.normpath(f_path)
         return f_path
     def get_nodes(self, repo_name, revision, root_path='/', flat=True):
         """
         Recursively walk root dir and return a set of all paths found.
         :param repo_name: name of repository
         :param revision: revision for which to list nodes
         :param root_path: root path to list
         :param flat: return as a list, if False returns a dict with description
         """
         _files = list()
         _dirs = list()
         try:
             _repo = self.__get_repo(repo_name)
             changeset = _repo.scm_instance.get_changeset(revision)
             root_path = root_path.lstrip('/')
             for topnode, dirs, files in changeset.walk(root_path):
                 for f in files:
                     _files.append(f.path if flat else {"name": f.path,
                                                        "type": "file"})
                 for d in dirs:
                     _dirs.append(d.path if flat else {"name": d.path,
                                                       "type": "dir"})
         except RepositoryError:
             log.debug(traceback.format_exc())
             raise
         return _dirs, _files
     def create_nodes(self, user, ip_addr, repo, message, nodes, parent_cs=None,
                      author=None, trigger_push_hook=True):
         """
         Commits specified nodes to repo.
         :param user: Kallithea User object or user_id, the committer
         :param repo: Kallithea Repository object
         :param message: commit message
         :param nodes: mapping {filename:{'content':content},...}
         :param parent_cs: parent changeset, can be empty than it's initial commit
         :param author: author of commit, cna be different that committer only for git
         :param trigger_push_hook: trigger push hooks
         :returns: new committed changeset
         """
         user = User.guess_instance(user)
         scm_instance = repo.scm_instance_no_cache()
         processed_nodes = []
         for f_path in nodes:
             content = nodes[f_path]['content']
             f_path = self._sanitize_path(f_path)
             f_path = safe_str(f_path)
             # decoding here will force that we have proper encoded values
             # in any other case this will throw exceptions and deny commit
             if isinstance(content, (str,)):
                 content = safe_str(content)
             else:
             if not isinstance(content, str) and not isinstance(content, bytes):
                 content = content.read()
             processed_nodes.append((f_path, content))
         message = message
         committer = user.full_contact
         if not author:
             author = committer
         IMC = self._get_IMC_module(scm_instance.alias)
         imc = IMC(scm_instance)
         if not parent_cs:
             parent_cs = EmptyChangeset(alias=scm_instance.alias)
         if isinstance(parent_cs, EmptyChangeset):
             # EmptyChangeset means we we're editing empty repository
             parents = None
         else:
             parents = [parent_cs]
         # add multiple nodes
         for path, content in processed_nodes:
             imc.add(FileNode(path, content=content))
         tip = imc.commit(message=message,
                          author=author,
                          parents=parents,
                          branch=parent_cs.branch)
         if trigger_push_hook:
             self._handle_push(scm_instance,
                               username=user.username,
                               ip_addr=ip_addr,
                               action='push_local',
                               repo_name=repo.repo_name,
                               revisions=[tip.raw_id])
         else:
             self.mark_for_invalidation(repo.repo_name)
         return tip
     def update_nodes(self, user, ip_addr, repo, message, nodes, parent_cs=None,
                      author=None, trigger_push_hook=True):
         """
         Commits specified nodes to repo. Again.
         """
         user = User.guess_instance(user)
         scm_instance = repo.scm_instance_no_cache()
         message = message
         committer = user.full_contact
         if not author:
             author = committer
         imc_class = self._get_IMC_module(scm_instance.alias)
         imc = imc_class(scm_instance)
         if not parent_cs:
             parent_cs = EmptyChangeset(alias=scm_instance.alias)
         if isinstance(parent_cs, EmptyChangeset):
             # EmptyChangeset means we we're editing empty repository
             parents = None
         else:
             parents = [parent_cs]
         # add multiple nodes
         for _filename, data in nodes.items():
             # new filename, can be renamed from the old one
             filename = self._sanitize_path(data['filename'])
             old_filename = self._sanitize_path(_filename)
             content = data['content']
             filenode = FileNode(old_filename, content=content)
             op = data['op']
             if op == 'add':
                 imc.add(filenode)
             elif op == 'del':
                 imc.remove(filenode)
             elif op == 'mod':
                 if filename != old_filename:
                     # TODO: handle renames, needs vcs lib changes
                     imc.remove(filenode)
                     imc.add(FileNode(filename, content=content))
                 else:
                     imc.change(filenode)
         # commit changes
         tip = imc.commit(message=message,
                          author=author,
                          parents=parents,
                          branch=parent_cs.branch)
         if trigger_push_hook:
             self._handle_push(scm_instance,
                               username=user.username,
                               ip_addr=ip_addr,
                               action='push_local',
                               repo_name=repo.repo_name,
                               revisions=[tip.raw_id])
         else:
             self.mark_for_invalidation(repo.repo_name)
     def delete_nodes(self, user, ip_addr, repo, message, nodes, parent_cs=None,
                      author=None, trigger_push_hook=True):
         """
         Deletes specified nodes from repo.
         :param user: Kallithea User object or user_id, the committer
         :param repo: Kallithea Repository object
         :param message: commit message
         :param nodes: mapping {filename:{'content':content},...}
         :param parent_cs: parent changeset, can be empty than it's initial commit
         :param author: author of commit, cna be different that committer only for git
         :param trigger_push_hook: trigger push hooks
         :returns: new committed changeset after deletion
         """
         user = User.guess_instance(user)
         scm_instance = repo.scm_instance_no_cache()
         processed_nodes = []
         for f_path in nodes:
             f_path = self._sanitize_path(f_path)
             # content can be empty but for compatibility it allows same dicts
             # structure as add_nodes
             content = nodes[f_path].get('content')
             processed_nodes.append((f_path, content))
         message = message
         committer = user.full_contact
         if not author:
             author = committer
         IMC = self._get_IMC_module(scm_instance.alias)
         imc = IMC(scm_instance)
         if not parent_cs:
             parent_cs = EmptyChangeset(alias=scm_instance.alias)
         if isinstance(parent_cs, EmptyChangeset):
             # EmptyChangeset means we we're editing empty repository
             parents = None
         else:
             parents = [parent_cs]
         # add multiple nodes
         for path, content in processed_nodes:
             imc.remove(FileNode(path, content=content))
         tip = imc.commit(message=message,
                          author=author,
                          parents=parents,
                          branch=parent_cs.branch)
         if trigger_push_hook:
             self._handle_push(scm_instance,
                               username=user.username,
                               ip_addr=ip_addr,
                               action='push_local',
                               repo_name=repo.repo_name,
                               revisions=[tip.raw_id])
         else:
             self.mark_for_invalidation(repo.repo_name)
         return tip
     def get_unread_journal(self):
         return UserLog.query().count()
     def get_repo_landing_revs(self, repo=None):
         """
         Generates select option with tags branches and bookmarks (for hg only)
         grouped by type
         :param repo:
         """
         hist_l = []
         choices = []
         repo = self.__get_repo(repo)
         hist_l.append(('rev:tip', _('latest tip')))
         choices.append('rev:tip')
         if repo is None:
             return choices, hist_l
         repo = repo.scm_instance
         branches_group = ([(u'branch:%s' % k, k) for k, v in
                            repo.branches.items()], _("Branches"))
         hist_l.append(branches_group)
         choices.extend([x[0] for x in branches_group[0]])
         if repo.alias == 'hg':
             bookmarks_group = ([(u'book:%s' % k, k) for k, v in

kallithea/tests/base.py

➞

Show inline comments

 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
+#
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 import datetime
 import logging
 import os
 import re
 import tempfile
 import time
 import pytest
 from webtest import TestApp
-from kallithea.lib.utils2 import ascii_str, safe_str
 from kallithea.lib.utils2 import ascii_str
 from kallithea.model.db import User
 log = logging.getLogger(__name__)
 skipif = pytest.mark.skipif
 parametrize = pytest.mark.parametrize
 # Hack: These module global values MUST be set to actual values before running any tests. This is currently done by conftest.py.
 url = None
 testapp = None
 __all__ = [
     'skipif', 'parametrize', 'url', 'TestController',
     'ldap_lib_installed', 'pam_lib_installed', 'invalidate_all_caches',
     'TESTS_TMP_PATH', 'HG_REPO', 'GIT_REPO', 'NEW_HG_REPO', 'NEW_GIT_REPO',
     'HG_FORK', 'GIT_FORK', 'TEST_USER_ADMIN_LOGIN', 'TEST_USER_ADMIN_PASS',
     'TEST_USER_ADMIN_EMAIL', 'TEST_USER_REGULAR_LOGIN', 'TEST_USER_REGULAR_PASS',
     'TEST_USER_REGULAR_EMAIL', 'TEST_USER_REGULAR2_LOGIN',
     'TEST_USER_REGULAR2_PASS', 'TEST_USER_REGULAR2_EMAIL', 'IP_ADDR',
     'TEST_HG_REPO', 'TEST_HG_REPO_CLONE', 'TEST_HG_REPO_PULL', 'TEST_GIT_REPO',
     'TEST_GIT_REPO_CLONE', 'TEST_GIT_REPO_PULL', 'HG_REMOTE_REPO',
     'GIT_REMOTE_REPO', 'HG_TEST_REVISION', 'GIT_TEST_REVISION',
+]
 ## SOME GLOBALS FOR TESTS
 TESTS_TMP_PATH = os.environ.get('KALLITHEA_TESTS_TMP_PATH', tempfile.mkdtemp(prefix='kallithea-test-'))
 TEST_USER_ADMIN_LOGIN = 'test_admin'
 TEST_USER_ADMIN_PASS = 'test12'
 TEST_USER_ADMIN_EMAIL = 'test_admin@example.com'
 TEST_USER_REGULAR_LOGIN = 'test_regular'
 TEST_USER_REGULAR_PASS = 'test12'
 TEST_USER_REGULAR_EMAIL = 'test_regular@example.com'
 TEST_USER_REGULAR2_LOGIN = 'test_regular2'
 TEST_USER_REGULAR2_PASS = 'test12'
 TEST_USER_REGULAR2_EMAIL = 'test_regular2@example.com'
 IP_ADDR = '127.0.0.127'
 HG_REPO = u'vcs_test_hg'
 GIT_REPO = u'vcs_test_git'
 NEW_HG_REPO = u'vcs_test_hg_new'
 NEW_GIT_REPO = u'vcs_test_git_new'
 HG_FORK = u'vcs_test_hg_fork'
 GIT_FORK = u'vcs_test_git_fork'
 HG_TEST_REVISION = u"a53d9201d4bc278910d416d94941b7ea007ecd52"
 GIT_TEST_REVISION = u"7ab37bc680b4aa72c34d07b230c866c28e9fc204"
 ## VCS
 uniq_suffix = str(int(time.mktime(datetime.datetime.now().timetuple())))
 GIT_REMOTE_REPO = os.path.join(TESTS_TMP_PATH, GIT_REPO)
 TEST_GIT_REPO = os.path.join(TESTS_TMP_PATH, GIT_REPO)
 TEST_GIT_REPO_CLONE = os.path.join(TESTS_TMP_PATH, 'vcs-git-clone-%s' % uniq_suffix)
 TEST_GIT_REPO_PULL = os.path.join(TESTS_TMP_PATH, 'vcs-git-pull-%s' % uniq_suffix)
 HG_REMOTE_REPO = os.path.join(TESTS_TMP_PATH, HG_REPO)
 TEST_HG_REPO = os.path.join(TESTS_TMP_PATH, HG_REPO)
 TEST_HG_REPO_CLONE = os.path.join(TESTS_TMP_PATH, 'vcs-hg-clone-%s' % uniq_suffix)
 TEST_HG_REPO_PULL = os.path.join(TESTS_TMP_PATH, 'vcs-hg-pull-%s' % uniq_suffix)
 # By default, some of the tests will utilise locally available
 # repositories stored within tar.gz archives as source for
 # cloning. Should you wish to use some other, remote archive, simply
 # uncomment these entries and/or update the URLs to use.
+#
 # GIT_REMOTE_REPO = 'git://github.com/codeinn/vcs.git'
 # HG_REMOTE_REPO = 'http://bitbucket.org/marcinkuzminski/vcs'
 # skip ldap tests if LDAP lib is not installed
 ldap_lib_installed = False
 try:
     import ldap
     ldap.API_VERSION
     ldap_lib_installed = True
 except ImportError:
     # means that python-ldap is not installed
     pass
 try:
     import pam
     pam.PAM_TEXT_INFO
     pam_lib_installed = True
 except ImportError:
     pam_lib_installed = False
 def invalidate_all_caches():
     """Invalidate all beaker caches currently configured.
     Useful when manipulating IP permissions in a test and changes need to take
     effect immediately.
     Note: Any use of this function is probably a workaround - it should be
     replaced with a more specific cache invalidation in code or test."""
     from beaker.cache import cache_managers
     for cache in cache_managers.values():
         cache.clear()
 class NullHandler(logging.Handler):
     def emit(self, record):
         pass
 class TestController(object):
     """Pytest-style test controller"""
     # Note: pytest base classes cannot have an __init__ method
     @pytest.fixture(autouse=True)
     def app_fixture(self):
         h = NullHandler()
         logging.getLogger("kallithea").addHandler(h)
         self.app = TestApp(testapp)
         return self.app
     def log_user(self, username=TEST_USER_ADMIN_LOGIN,
                  password=TEST_USER_ADMIN_PASS):
         self._logged_username = username
         response = self.app.post(url(controller='login', action='index'),
                                  {'username': username,
                                   'password': password,
                                   '_session_csrf_secret_token': self.session_csrf_secret_token()})
         if b'Invalid username or password' in response.body:
             pytest.fail('could not login using %s %s' % (username, password))
         assert response.status == '302 Found'
         self.assert_authenticated_user(response, username)
         response = response.follow()
         return response.session['authuser']
     def _get_logged_user(self):
         return User.get_by_username(self._logged_username)
     def assert_authenticated_user(self, response, expected_username):
         cookie = response.session.get('authuser')
         user = cookie and cookie.get('user_id')
         user = user and User.get(user)
         user = user and user.username
         assert user == expected_username
     def session_csrf_secret_token(self):
         return ascii_str(self.app.get(url('session_csrf_secret_token')).body)
     def checkSessionFlash(self, response, msg=None, skip=0, _matcher=lambda msg, m: msg in m):
         if 'flash' not in response.session:
-            pytest.fail(safe_str(u'msg `%s` not found - session has no flash:\n%s' % (msg, response)))
             pytest.fail(u'msg `%s` not found - session has no flash:\n%s' % (msg, response))
         try:
             level, m = response.session['flash'][-1 - skip]
             if _matcher(msg, m):
                 return
         except IndexError:
             pass
         pytest.fail(safe_str(u'msg `%s` not found in session flash (skipping %s): %s' %
                            (msg, skip,
                             ', '.join('`%s`' % m for level, m in response.session['flash']))))
         pytest.fail(u'msg `%s` not found in session flash (skipping %s): %s' %
                     (msg, skip, ', '.join('`%s`' % m for level, m in response.session['flash'])))
     def checkSessionFlashRegex(self, response, regex, skip=0):
         self.checkSessionFlash(response, regex, skip=skip, _matcher=re.search)

kallithea/tests/functional/test_admin_repos.py

➞

Show inline comments

 # -*- coding: utf-8 -*-
 import os
 import urllib.parse
 import mock
 import pytest
 from kallithea.lib import vcs
 from kallithea.lib.utils2 import safe_str
 from kallithea.model.db import Permission, RepoGroup, Repository, Ui, User, UserRepoToPerm
 from kallithea.model.meta import Session
 from kallithea.model.repo import RepoModel
 from kallithea.model.repo_group import RepoGroupModel
 from kallithea.model.user import UserModel
 from kallithea.tests import base
 from kallithea.tests.fixture import Fixture, error_function
 fixture = Fixture()
 def _get_permission_for_user(user, repo):
     perm = UserRepoToPerm.query() \
                 .filter(UserRepoToPerm.repository ==
                         Repository.get_by_repo_name(repo)) \
                 .filter(UserRepoToPerm.user == User.get_by_username(user)) \
                 .all()
     return perm
 class _BaseTestCase(base.TestController):
     """
     Write all tests here
     """
     REPO = None
     REPO_TYPE = None
     NEW_REPO = None
     OTHER_TYPE_REPO = None
     OTHER_TYPE = None
     def test_index(self):
         self.log_user()
         response = self.app.get(base.url('repos'))
     def test_create(self):
         self.log_user()
         repo_name = self.NEW_REPO
         description = u'description for newly created repo'
         response = self.app.post(base.url('repos'),
                         fixture._get_repo_create_params(repo_private=False,
                                                 repo_name=repo_name,
                                                 repo_type=self.REPO_TYPE,
                                                 repo_description=description,
                                                 _session_csrf_secret_token=self.session_csrf_secret_token()))
         ## run the check page that triggers the flash message
         response = self.app.get(base.url('repo_check_home', repo_name=repo_name))
         assert response.json == {u'result': True}
         self.checkSessionFlash(response,
                                'Created repository <a href="/%s">%s</a>'
                                % (repo_name, repo_name))
         # test if the repo was created in the database
         new_repo = Session().query(Repository) \
             .filter(Repository.repo_name == repo_name).one()
         assert new_repo.repo_name == repo_name
         assert new_repo.description == description
         # test if the repository is visible in the list ?
         response = self.app.get(base.url('summary_home', repo_name=repo_name))
         response.mustcontain(repo_name)
         response.mustcontain(self.REPO_TYPE)
         # test if the repository was created on filesystem
         try:
-            vcs.get_repo(safe_str(os.path.join(Ui.get_by_key('paths', '/').ui_value, repo_name)))
             vcs.get_repo(os.path.join(Ui.get_by_key('paths', '/').ui_value, repo_name))
         except vcs.exceptions.VCSError:
             pytest.fail('no repo %s in filesystem' % repo_name)
         RepoModel().delete(repo_name)
         Session().commit()
     def test_case_insensitivity(self):
         self.log_user()
         repo_name = self.NEW_REPO
         description = u'description for newly created repo'
         response = self.app.post(base.url('repos'),
                                  fixture._get_repo_create_params(repo_private=False,
                                                                  repo_name=repo_name,
                                                                  repo_type=self.REPO_TYPE,
                                                                  repo_description=description,
                                                                  _session_csrf_secret_token=self.session_csrf_secret_token()))
         # try to create repo with swapped case
         swapped_repo_name = repo_name.swapcase()
         response = self.app.post(base.url('repos'),
                                  fixture._get_repo_create_params(repo_private=False,
                                                                  repo_name=swapped_repo_name,
                                                                  repo_type=self.REPO_TYPE,
                                                                  repo_description=description,
                                                                  _session_csrf_secret_token=self.session_csrf_secret_token()))
         response.mustcontain('already exists')
         RepoModel().delete(repo_name)
         Session().commit()
     def test_create_in_group(self):
         self.log_user()
         ## create GROUP
         group_name = u'sometest_%s' % self.REPO_TYPE
         gr = RepoGroupModel().create(group_name=group_name,
                                      group_description=u'test',
                                      owner=base.TEST_USER_ADMIN_LOGIN)
         Session().commit()
         repo_name = u'ingroup'
         repo_name_full = RepoGroup.url_sep().join([group_name, repo_name])
         description = u'description for newly created repo'
         response = self.app.post(base.url('repos'),
                         fixture._get_repo_create_params(repo_private=False,
                                                 repo_name=repo_name,
                                                 repo_type=self.REPO_TYPE,
                                                 repo_description=description,
                                                 repo_group=gr.group_id,
                                                 _session_csrf_secret_token=self.session_csrf_secret_token()))
         ## run the check page that triggers the flash message
         response = self.app.get(base.url('repo_check_home', repo_name=repo_name_full))
         assert response.json == {u'result': True}
         self.checkSessionFlash(response,
                                'Created repository <a href="/%s">%s</a>'
                                % (repo_name_full, repo_name_full))
         # test if the repo was created in the database
         new_repo = Session().query(Repository) \
             .filter(Repository.repo_name == repo_name_full).one()
         new_repo_id = new_repo.repo_id
         assert new_repo.repo_name == repo_name_full
         assert new_repo.description == description
         # test if the repository is visible in the list ?
         response = self.app.get(base.url('summary_home', repo_name=repo_name_full))
         response.mustcontain(repo_name_full)
         response.mustcontain(self.REPO_TYPE)
         inherited_perms = UserRepoToPerm.query() \
             .filter(UserRepoToPerm.repository_id == new_repo_id).all()
         assert len(inherited_perms) == 1
         # test if the repository was created on filesystem
         try:
-            vcs.get_repo(safe_str(os.path.join(Ui.get_by_key('paths', '/').ui_value, repo_name_full)))
             vcs.get_repo(os.path.join(Ui.get_by_key('paths', '/').ui_value, repo_name_full))
         except vcs.exceptions.VCSError:
             RepoGroupModel().delete(group_name)
             Session().commit()
             pytest.fail('no repo %s in filesystem' % repo_name)
         RepoModel().delete(repo_name_full)
         RepoGroupModel().delete(group_name)
         Session().commit()
     def test_create_in_group_without_needed_permissions(self):
         usr = self.log_user(base.TEST_USER_REGULAR_LOGIN, base.TEST_USER_REGULAR_PASS)
         # avoid spurious RepoGroup DetachedInstanceError ...
         session_csrf_secret_token = self.session_csrf_secret_token()
         # revoke
         user_model = UserModel()
         # disable fork and create on default user
         user_model.revoke_perm(User.DEFAULT_USER, 'hg.create.repository')
         user_model.grant_perm(User.DEFAULT_USER, 'hg.create.none')
         user_model.revoke_perm(User.DEFAULT_USER, 'hg.fork.repository')
         user_model.grant_perm(User.DEFAULT_USER, 'hg.fork.none')
         # disable on regular user
         user_model.revoke_perm(base.TEST_USER_REGULAR_LOGIN, 'hg.create.repository')
         user_model.grant_perm(base.TEST_USER_REGULAR_LOGIN, 'hg.create.none')
         user_model.revoke_perm(base.TEST_USER_REGULAR_LOGIN, 'hg.fork.repository')
         user_model.grant_perm(base.TEST_USER_REGULAR_LOGIN, 'hg.fork.none')
         Session().commit()
         ## create GROUP
         group_name = u'reg_sometest_%s' % self.REPO_TYPE
         gr = RepoGroupModel().create(group_name=group_name,
                                      group_description=u'test',
                                      owner=base.TEST_USER_ADMIN_LOGIN)
         Session().commit()
         group_name_allowed = u'reg_sometest_allowed_%s' % self.REPO_TYPE
         gr_allowed = RepoGroupModel().create(group_name=group_name_allowed,
                                      group_description=u'test',
                                      owner=base.TEST_USER_REGULAR_LOGIN)
         Session().commit()
         repo_name = u'ingroup'
         repo_name_full = RepoGroup.url_sep().join([group_name, repo_name])
         description = u'description for newly created repo'
         response = self.app.post(base.url('repos'),
                         fixture._get_repo_create_params(repo_private=False,
                                                 repo_name=repo_name,
                                                 repo_type=self.REPO_TYPE,
                                                 repo_description=description,
                                                 repo_group=gr.group_id,
                                                 _session_csrf_secret_token=session_csrf_secret_token))
         response.mustcontain('Invalid value')
         # user is allowed to create in this group
         repo_name = u'ingroup'
         repo_name_full = RepoGroup.url_sep().join([group_name_allowed, repo_name])
         description = u'description for newly created repo'
         response = self.app.post(base.url('repos'),
                         fixture._get_repo_create_params(repo_private=False,
                                                 repo_name=repo_name,
                                                 repo_type=self.REPO_TYPE,
                                                 repo_description=description,
                                                 repo_group=gr_allowed.group_id,
                                                 _session_csrf_secret_token=session_csrf_secret_token))
         ## run the check page that triggers the flash message
         response = self.app.get(base.url('repo_check_home', repo_name=repo_name_full))
         assert response.json == {u'result': True}
         self.checkSessionFlash(response,
                                'Created repository <a href="/%s">%s</a>'
                                % (repo_name_full, repo_name_full))
         # test if the repo was created in the database
         new_repo = Session().query(Repository) \
             .filter(Repository.repo_name == repo_name_full).one()
         new_repo_id = new_repo.repo_id
         assert new_repo.repo_name == repo_name_full
         assert new_repo.description == description
         # test if the repository is visible in the list ?
         response = self.app.get(base.url('summary_home', repo_name=repo_name_full))
         response.mustcontain(repo_name_full)
         response.mustcontain(self.REPO_TYPE)
         inherited_perms = UserRepoToPerm.query() \
             .filter(UserRepoToPerm.repository_id == new_repo_id).all()
         assert len(inherited_perms) == 1
         # test if the repository was created on filesystem
         try:
-            vcs.get_repo(safe_str(os.path.join(Ui.get_by_key('paths', '/').ui_value, repo_name_full)))
             vcs.get_repo(os.path.join(Ui.get_by_key('paths', '/').ui_value, repo_name_full))
         except vcs.exceptions.VCSError:
             RepoGroupModel().delete(group_name)
             Session().commit()
             pytest.fail('no repo %s in filesystem' % repo_name)
         RepoModel().delete(repo_name_full)
         RepoGroupModel().delete(group_name)
         RepoGroupModel().delete(group_name_allowed)
         Session().commit()
     def test_create_in_group_inherit_permissions(self):
         self.log_user()
         ## create GROUP
         group_name = u'sometest_%s' % self.REPO_TYPE
         gr = RepoGroupModel().create(group_name=group_name,
                                      group_description=u'test',
                                      owner=base.TEST_USER_ADMIN_LOGIN)
         perm = Permission.get_by_key('repository.write')
         RepoGroupModel().grant_user_permission(gr, base.TEST_USER_REGULAR_LOGIN, perm)
         ## add repo permissions
         Session().commit()
         repo_name = u'ingroup_inherited_%s' % self.REPO_TYPE
         repo_name_full = RepoGroup.url_sep().join([group_name, repo_name])
         description = u'description for newly created repo'
         response = self.app.post(base.url('repos'),
                         fixture._get_repo_create_params(repo_private=False,
                                                 repo_name=repo_name,
                                                 repo_type=self.REPO_TYPE,
                                                 repo_description=description,
                                                 repo_group=gr.group_id,
                                                 repo_copy_permissions=True,
                                                 _session_csrf_secret_token=self.session_csrf_secret_token()))
         ## run the check page that triggers the flash message
         response = self.app.get(base.url('repo_check_home', repo_name=repo_name_full))
         self.checkSessionFlash(response,
                                'Created repository <a href="/%s">%s</a>'
                                % (repo_name_full, repo_name_full))
         # test if the repo was created in the database
         new_repo = Session().query(Repository) \
             .filter(Repository.repo_name == repo_name_full).one()
         new_repo_id = new_repo.repo_id
         assert new_repo.repo_name == repo_name_full
         assert new_repo.description == description
         # test if the repository is visible in the list ?
         response = self.app.get(base.url('summary_home', repo_name=repo_name_full))
         response.mustcontain(repo_name_full)
         response.mustcontain(self.REPO_TYPE)
         # test if the repository was created on filesystem
         try:
-            vcs.get_repo(safe_str(os.path.join(Ui.get_by_key('paths', '/').ui_value, repo_name_full)))
             vcs.get_repo(os.path.join(Ui.get_by_key('paths', '/').ui_value, repo_name_full))
         except vcs.exceptions.VCSError:
             RepoGroupModel().delete(group_name)
             Session().commit()
             pytest.fail('no repo %s in filesystem' % repo_name)
         # check if inherited permissiona are applied
         inherited_perms = UserRepoToPerm.query() \
             .filter(UserRepoToPerm.repository_id == new_repo_id).all()
         assert len(inherited_perms) == 2
         assert base.TEST_USER_REGULAR_LOGIN in [x.user.username
                                                     for x in inherited_perms]
         assert 'repository.write' in [x.permission.permission_name
                                                for x in inherited_perms]
         RepoModel().delete(repo_name_full)
         RepoGroupModel().delete(group_name)
         Session().commit()
     def test_create_remote_repo_wrong_clone_uri(self):
         self.log_user()
         repo_name = self.NEW_REPO
         description = u'description for newly created repo'
         response = self.app.post(base.url('repos'),
                         fixture._get_repo_create_params(repo_private=False,
                                                 repo_name=repo_name,
                                                 repo_type=self.REPO_TYPE,
                                                 repo_description=description,
                                                 clone_uri='http://127.0.0.1/repo',
                                                 _session_csrf_secret_token=self.session_csrf_secret_token()))
         response.mustcontain('Invalid repository URL')
     def test_create_remote_repo_wrong_clone_uri_hg_svn(self):
         self.log_user()
         repo_name = self.NEW_REPO
         description = u'description for newly created repo'
         response = self.app.post(base.url('repos'),
                         fixture._get_repo_create_params(repo_private=False,
                                                 repo_name=repo_name,
                                                 repo_type=self.REPO_TYPE,
                                                 repo_description=description,
                                                 clone_uri='svn+http://127.0.0.1/repo',
                                                 _session_csrf_secret_token=self.session_csrf_secret_token()))
         response.mustcontain('Invalid repository URL')
     def test_delete(self):
         self.log_user()
         repo_name = u'vcs_test_new_to_delete_%s' % self.REPO_TYPE
         description = u'description for newly created repo'
         response = self.app.post(base.url('repos'),
                         fixture._get_repo_create_params(repo_private=False,
                                                 repo_type=self.REPO_TYPE,
                                                 repo_name=repo_name,
                                                 repo_description=description,
                                                 _session_csrf_secret_token=self.session_csrf_secret_token()))
         ## run the check page that triggers the flash message
         response = self.app.get(base.url('repo_check_home', repo_name=repo_name))
         self.checkSessionFlash(response,
                                'Created repository <a href="/%s">%s</a>'
                                % (repo_name, repo_name))
         # test if the repo was created in the database
         new_repo = Session().query(Repository) \
             .filter(Repository.repo_name == repo_name).one()
         assert new_repo.repo_name == repo_name
         assert new_repo.description == description
         # test if the repository is visible in the list ?
         response = self.app.get(base.url('summary_home', repo_name=repo_name))
         response.mustcontain(repo_name)
         response.mustcontain(self.REPO_TYPE)
         # test if the repository was created on filesystem
         try:
-            vcs.get_repo(safe_str(os.path.join(Ui.get_by_key('paths', '/').ui_value, repo_name)))
             vcs.get_repo(os.path.join(Ui.get_by_key('paths', '/').ui_value, repo_name))
         except vcs.exceptions.VCSError:
             pytest.fail('no repo %s in filesystem' % repo_name)
         response = self.app.post(base.url('delete_repo', repo_name=repo_name),
             params={'_session_csrf_secret_token': self.session_csrf_secret_token()})
         self.checkSessionFlash(response, 'Deleted repository %s' % (repo_name))
         response.follow()
         # check if repo was deleted from db
         deleted_repo = Session().query(Repository) \
             .filter(Repository.repo_name == repo_name).scalar()
         assert deleted_repo is None
         assert os.path.isdir(os.path.join(Ui.get_by_key('paths', '/').ui_value, repo_name)) == False
     def test_delete_non_ascii(self):
         self.log_user()
         non_ascii = "ąęł"
-        repo_name = "%s%s" % (safe_str(self.NEW_REPO), non_ascii)
         repo_name = "%s%s" % (self.NEW_REPO, non_ascii)
         description = 'description for newly created repo' + non_ascii
         response = self.app.post(base.url('repos'),
                         fixture._get_repo_create_params(repo_private=False,
                                                 repo_name=repo_name,
                                                 repo_type=self.REPO_TYPE,
                                                 repo_description=description,
                                                 _session_csrf_secret_token=self.session_csrf_secret_token()))
         ## run the check page that triggers the flash message
         response = self.app.get(base.url('repo_check_home', repo_name=repo_name))
         assert response.json == {u'result': True}
         self.checkSessionFlash(response,
                                u'Created repository <a href="/%s">%s</a>'
                                % (urllib.parse.quote(repo_name), repo_name))
         # test if the repo was created in the database
         new_repo = Session().query(Repository) \
             .filter(Repository.repo_name == repo_name).one()
         assert new_repo.repo_name == repo_name
         assert new_repo.description == description
         # test if the repository is visible in the list ?
         response = self.app.get(base.url('summary_home', repo_name=repo_name))
         response.mustcontain(repo_name)
         response.mustcontain(self.REPO_TYPE)
         # test if the repository was created on filesystem
         try:
             vcs.get_repo(os.path.join(Ui.get_by_key('paths', '/').ui_value, repo_name))
         except vcs.exceptions.VCSError:
             pytest.fail('no repo %s in filesystem' % repo_name)
         response = self.app.post(base.url('delete_repo', repo_name=repo_name),
             params={'_session_csrf_secret_token': self.session_csrf_secret_token()})
         self.checkSessionFlash(response, 'Deleted repository %s' % (repo_name))
         response.follow()
         # check if repo was deleted from db
         deleted_repo = Session().query(Repository) \
             .filter(Repository.repo_name == repo_name).scalar()
         assert deleted_repo is None
         assert os.path.isdir(os.path.join(Ui.get_by_key('paths', '/').ui_value, repo_name)) == False
     def test_delete_repo_with_group(self):
         # TODO:
         pass
     def test_delete_browser_fakeout(self):
         response = self.app.post(base.url('delete_repo', repo_name=self.REPO),
                                  params=dict(_session_csrf_secret_token=self.session_csrf_secret_token()))
     def test_show(self):
         self.log_user()
         response = self.app.get(base.url('summary_home', repo_name=self.REPO))
     def test_edit(self):
         response = self.app.get(base.url('edit_repo', repo_name=self.REPO))
     def test_set_private_flag_sets_default_to_none(self):
         self.log_user()
         # initially repository perm should be read
         perm = _get_permission_for_user(user='default', repo=self.REPO)
         assert len(perm), 1
         assert perm[0].permission.permission_name == 'repository.read'
         assert Repository.get_by_repo_name(self.REPO).private == False
         response = self.app.post(base.url('update_repo', repo_name=self.REPO),
                         fixture._get_repo_create_params(repo_private=1,
                                                 repo_name=self.REPO,
                                                 repo_type=self.REPO_TYPE,
                                                 owner=base.TEST_USER_ADMIN_LOGIN,
                                                 _session_csrf_secret_token=self.session_csrf_secret_token()))
         self.checkSessionFlash(response,
                                msg='Repository %s updated successfully' % (self.REPO))
         assert Repository.get_by_repo_name(self.REPO).private == True
         # now the repo default permission should be None
         perm = _get_permission_for_user(user='default', repo=self.REPO)
         assert len(perm), 1
         assert perm[0].permission.permission_name == 'repository.none'
         response = self.app.post(base.url('update_repo', repo_name=self.REPO),
                         fixture._get_repo_create_params(repo_private=False,
                                                 repo_name=self.REPO,
                                                 repo_type=self.REPO_TYPE,
                                                 owner=base.TEST_USER_ADMIN_LOGIN,
                                                 _session_csrf_secret_token=self.session_csrf_secret_token()))
         self.checkSessionFlash(response,
                                msg='Repository %s updated successfully' % (self.REPO))
         assert Repository.get_by_repo_name(self.REPO).private == False
         # we turn off private now the repo default permission should stay None
         perm = _get_permission_for_user(user='default', repo=self.REPO)
         assert len(perm), 1
         assert perm[0].permission.permission_name == 'repository.none'
         # update this permission back
         perm[0].permission = Permission.get_by_key('repository.read')
         Session().commit()
     def test_set_repo_fork_has_no_self_id(self):
         self.log_user()
         repo = Repository.get_by_repo_name(self.REPO)
         response = self.app.get(base.url('edit_repo_advanced', repo_name=self.REPO))
         opt = """<option value="%s">%s</option>""" % (repo.repo_id, self.REPO)
         response.mustcontain(no=[opt])
     def test_set_fork_of_other_repo(self):
         self.log_user()
         other_repo = u'other_%s' % self.REPO_TYPE
         fixture.create_repo(other_repo, repo_type=self.REPO_TYPE)
         repo = Repository.get_by_repo_name(self.REPO)
         repo2 = Repository.get_by_repo_name(other_repo)
         response = self.app.post(base.url('edit_repo_advanced_fork', repo_name=self.REPO),
                                 params=dict(id_fork_of=repo2.repo_id, _session_csrf_secret_token=self.session_csrf_secret_token()))
         repo = Repository.get_by_repo_name(self.REPO)
         repo2 = Repository.get_by_repo_name(other_repo)
         self.checkSessionFlash(response,
             'Marked repository %s as fork of %s' % (repo.repo_name, repo2.repo_name))
         assert repo.fork == repo2
         response = response.follow()
         # check if given repo is selected
         opt = """<option value="%s" selected="selected">%s</option>""" % (
                     repo2.repo_id, repo2.repo_name)
         response.mustcontain(opt)
         fixture.destroy_repo(other_repo, forks='detach')
     def test_set_fork_of_other_type_repo(self):
         self.log_user()
         repo = Repository.get_by_repo_name(self.REPO)
         repo2 = Repository.get_by_repo_name(self.OTHER_TYPE_REPO)
         response = self.app.post(base.url('edit_repo_advanced_fork', repo_name=self.REPO),
                                 params=dict(id_fork_of=repo2.repo_id, _session_csrf_secret_token=self.session_csrf_secret_token()))
         repo = Repository.get_by_repo_name(self.REPO)
         repo2 = Repository.get_by_repo_name(self.OTHER_TYPE_REPO)
         self.checkSessionFlash(response,
             'Cannot set repository as fork of repository with other type')
     def test_set_fork_of_none(self):
         self.log_user()
         ## mark it as None
         response = self.app.post(base.url('edit_repo_advanced_fork', repo_name=self.REPO),
                                 params=dict(id_fork_of=None, _session_csrf_secret_token=self.session_csrf_secret_token()))
         repo = Repository.get_by_repo_name(self.REPO)
         repo2 = Repository.get_by_repo_name(self.OTHER_TYPE_REPO)
         self.checkSessionFlash(response,
                                'Marked repository %s as fork of %s'
                                % (repo.repo_name, "Nothing"))
         assert repo.fork is None
     def test_set_fork_of_same_repo(self):
         self.log_user()
         repo = Repository.get_by_repo_name(self.REPO)
         response = self.app.post(base.url('edit_repo_advanced_fork', repo_name=self.REPO),
                                 params=dict(id_fork_of=repo.repo_id, _session_csrf_secret_token=self.session_csrf_secret_token()))
         self.checkSessionFlash(response,
                                'An error occurred during this operation')
     def test_create_on_top_level_without_permissions(self):
         usr = self.log_user(base.TEST_USER_REGULAR_LOGIN, base.TEST_USER_REGULAR_PASS)
         # revoke
         user_model = UserModel()
         # disable fork and create on default user
         user_model.revoke_perm(User.DEFAULT_USER, 'hg.create.repository')
         user_model.grant_perm(User.DEFAULT_USER, 'hg.create.none')
         user_model.revoke_perm(User.DEFAULT_USER, 'hg.fork.repository')
         user_model.grant_perm(User.DEFAULT_USER, 'hg.fork.none')
         # disable on regular user
         user_model.revoke_perm(base.TEST_USER_REGULAR_LOGIN, 'hg.create.repository')
         user_model.grant_perm(base.TEST_USER_REGULAR_LOGIN, 'hg.create.none')
         user_model.revoke_perm(base.TEST_USER_REGULAR_LOGIN, 'hg.fork.repository')
         user_model.grant_perm(base.TEST_USER_REGULAR_LOGIN, 'hg.fork.none')
         Session().commit()
         user = User.get(usr['user_id'])
         repo_name = self.NEW_REPO + u'no_perms'
         description = 'description for newly created repo'
         response = self.app.post(base.url('repos'),
                         fixture._get_repo_create_params(repo_private=False,
                                                 repo_name=repo_name,
                                                 repo_type=self.REPO_TYPE,
                                                 repo_description=description,
                                                 _session_csrf_secret_token=self.session_csrf_secret_token()))
         response.mustcontain('<span class="error-message">Invalid value</span>')

kallithea/tests/functional/test_forks.py

➞

Show inline comments

 # -*- coding: utf-8 -*-
 import urllib.parse
 from kallithea.lib.utils2 import safe_str
 from kallithea.model.db import Repository, User
 from kallithea.model.meta import Session
 from kallithea.model.repo import RepoModel
 from kallithea.model.user import UserModel
 from kallithea.tests import base
 from kallithea.tests.fixture import Fixture
 fixture = Fixture()
 class _BaseTestCase(base.TestController):
     """
     Write all tests here
     """
     REPO = None
     REPO_TYPE = None
     NEW_REPO = None
     REPO_FORK = None
     def setup_method(self, method):
         self.username = u'forkuser'
         self.password = u'qweqwe'
         u1 = fixture.create_user(self.username, password=self.password, email=u'fork_king@example.com')
         self.u1_id = u1.user_id
         Session().commit()
     def teardown_method(self, method):
         fixture.destroy_user(self.u1_id)
         Session().commit()
     def test_index(self):
         self.log_user()
         repo_name = self.REPO
         response = self.app.get(base.url(controller='forks', action='forks',
                                     repo_name=repo_name))
         response.mustcontain("""There are no forks yet""")
     def test_no_permissions_to_fork(self):
         self.log_user(base.TEST_USER_REGULAR_LOGIN, base.TEST_USER_REGULAR_PASS)['user_id']
         try:
             user_model = UserModel()
             usr = User.get_default_user()
             user_model.revoke_perm(usr, 'hg.fork.repository')
             user_model.grant_perm(usr, 'hg.fork.none')
             Session().commit()
             # try create a fork
             repo_name = self.REPO
             self.app.post(base.url(controller='forks', action='fork_create',
                               repo_name=repo_name), {'_session_csrf_secret_token': self.session_csrf_secret_token()}, status=403)
         finally:
             usr = User.get_default_user()
             user_model.revoke_perm(usr, 'hg.fork.none')
             user_model.grant_perm(usr, 'hg.fork.repository')
             Session().commit()
     def test_index_with_fork(self):
         self.log_user()
         # create a fork
         fork_name = self.REPO_FORK
         description = 'fork of vcs test'
         repo_name = self.REPO
         org_repo = Repository.get_by_repo_name(repo_name)
         creation_args = {
             'repo_name': fork_name,
             'repo_group': u'-1',
             'fork_parent_id': org_repo.repo_id,
             'repo_type': self.REPO_TYPE,
             'description': description,
             'private': 'False',
             'landing_rev': 'rev:tip',
             '_session_csrf_secret_token': self.session_csrf_secret_token()}
         self.app.post(base.url(controller='forks', action='fork_create',
                           repo_name=repo_name), creation_args)
         response = self.app.get(base.url(controller='forks', action='forks',
                                     repo_name=repo_name))
         response.mustcontain(
             """<a href="/%s">%s</a>""" % (fork_name, fork_name)
+        )
         # remove this fork
         response = self.app.post(base.url('delete_repo', repo_name=fork_name),
             params={'_session_csrf_secret_token': self.session_csrf_secret_token()})
     def test_fork_create_into_group(self):
         self.log_user()
         group = fixture.create_repo_group(u'vc')
         group_id = group.group_id
         fork_name = self.REPO_FORK
         fork_name_full = 'vc/%s' % fork_name
         description = 'fork of vcs test'
         repo_name = self.REPO
         org_repo = Repository.get_by_repo_name(repo_name)
         creation_args = {
             'repo_name': fork_name,
             'repo_group': group_id,
             'fork_parent_id': org_repo.repo_id,
             'repo_type': self.REPO_TYPE,
             'description': description,
             'private': 'False',
             'landing_rev': 'rev:tip',
             '_session_csrf_secret_token': self.session_csrf_secret_token()}
         self.app.post(base.url(controller='forks', action='fork_create',
                           repo_name=repo_name), creation_args)
         repo = Repository.get_by_repo_name(fork_name_full)
         assert repo.fork.repo_name == self.REPO
         ## run the check page that triggers the flash message
         response = self.app.get(base.url('repo_check_home', repo_name=fork_name_full))
         # test if we have a message that fork is ok
         self.checkSessionFlash(response,
                 'Forked repository %s as <a href="/%s">%s</a>'
                 % (repo_name, fork_name_full, fork_name_full))
         # test if the fork was created in the database
         fork_repo = Session().query(Repository) \
             .filter(Repository.repo_name == fork_name_full).one()
         assert fork_repo.repo_name == fork_name_full
         assert fork_repo.fork.repo_name == repo_name
         # test if the repository is visible in the list ?
         response = self.app.get(base.url('summary_home', repo_name=fork_name_full))
         response.mustcontain(fork_name_full)
         response.mustcontain(self.REPO_TYPE)
         response.mustcontain('Fork of "<a href="/%s">%s</a>"' % (repo_name, repo_name))
         fixture.destroy_repo(fork_name_full)
         fixture.destroy_repo_group(group_id)
     def test_fork_unicode(self):
         self.log_user()
         # create a fork
         repo_name = self.REPO
         org_repo = Repository.get_by_repo_name(repo_name)
-        fork_name = safe_str(self.REPO_FORK + u'-rødgrød')
         fork_name = self.REPO_FORK + u'-rødgrød'
         creation_args = {
             'repo_name': fork_name,
             'repo_group': u'-1',
             'fork_parent_id': org_repo.repo_id,
             'repo_type': self.REPO_TYPE,
             'description': 'unicode repo 1',
             'private': 'False',
             'landing_rev': 'rev:tip',
             '_session_csrf_secret_token': self.session_csrf_secret_token()}
         self.app.post(base.url(controller='forks', action='fork_create',
                           repo_name=repo_name), creation_args)
         response = self.app.get(base.url(controller='forks', action='forks',
                                     repo_name=repo_name))
         response.mustcontain(
             """<a href="/%s">%s</a>""" % (urllib.parse.quote(fork_name), fork_name)
+        )
         fork_repo = Repository.get_by_repo_name(fork_name)
         assert fork_repo
         # fork the fork
-        fork_name_2 = safe_str(self.REPO_FORK + u'-blåbærgrød')
         fork_name_2 = self.REPO_FORK + u'-blåbærgrød'
         creation_args = {
             'repo_name': fork_name_2,
             'repo_group': u'-1',
             'fork_parent_id': fork_repo.repo_id,
             'repo_type': self.REPO_TYPE,
             'description': 'unicode repo 2',
             'private': 'False',
             'landing_rev': 'rev:tip',
             '_session_csrf_secret_token': self.session_csrf_secret_token()}
         self.app.post(base.url(controller='forks', action='fork_create',
                           repo_name=fork_name), creation_args)
         response = self.app.get(base.url(controller='forks', action='forks',
                                     repo_name=fork_name))
         response.mustcontain(
             """<a href="/%s">%s</a>""" % (urllib.parse.quote(fork_name_2), fork_name_2)
+        )
         # remove these forks
         response = self.app.post(base.url('delete_repo', repo_name=fork_name_2),
             params={'_session_csrf_secret_token': self.session_csrf_secret_token()})
         response = self.app.post(base.url('delete_repo', repo_name=fork_name),
             params={'_session_csrf_secret_token': self.session_csrf_secret_token()})
     def test_fork_create_and_permissions(self):
         self.log_user()
         fork_name = self.REPO_FORK
         description = 'fork of vcs test'
         repo_name = self.REPO
         org_repo = Repository.get_by_repo_name(repo_name)
         creation_args = {
             'repo_name': fork_name,
             'repo_group': u'-1',
             'fork_parent_id': org_repo.repo_id,
             'repo_type': self.REPO_TYPE,
             'description': description,
             'private': 'False',
             'landing_rev': 'rev:tip',
             '_session_csrf_secret_token': self.session_csrf_secret_token()}
         self.app.post(base.url(controller='forks', action='fork_create',
                           repo_name=repo_name), creation_args)
         repo = Repository.get_by_repo_name(self.REPO_FORK)
         assert repo.fork.repo_name == self.REPO
         ## run the check page that triggers the flash message
         response = self.app.get(base.url('repo_check_home', repo_name=fork_name))
         # test if we have a message that fork is ok
         self.checkSessionFlash(response,
                 'Forked repository %s as <a href="/%s">%s</a>'
                 % (repo_name, fork_name, fork_name))
         # test if the fork was created in the database
         fork_repo = Session().query(Repository) \
             .filter(Repository.repo_name == fork_name).one()
         assert fork_repo.repo_name == fork_name
         assert fork_repo.fork.repo_name == repo_name
         # test if the repository is visible in the list ?
         response = self.app.get(base.url('summary_home', repo_name=fork_name))
         response.mustcontain(fork_name)
         response.mustcontain(self.REPO_TYPE)
         response.mustcontain('Fork of "<a href="/%s">%s</a>"' % (repo_name, repo_name))
         usr = self.log_user(self.username, self.password)['user_id']
         forks = Repository.query() \
             .filter(Repository.repo_type == self.REPO_TYPE) \
             .filter(Repository.fork_id != None).all()
         assert 1 == len(forks)
         # set read permissions for this
         RepoModel().grant_user_permission(repo=forks[0],
                                           user=usr,
                                           perm='repository.read')
         Session().commit()
         response = self.app.get(base.url(controller='forks', action='forks',
                                     repo_name=repo_name))
         response.mustcontain('<div>fork of vcs test</div>')
         # remove permissions
         default_user = User.get_default_user()
         try:
             RepoModel().grant_user_permission(repo=forks[0],
                                               user=usr, perm='repository.none')
             RepoModel().grant_user_permission(repo=forks[0],
                                               user=default_user, perm='repository.none')
             Session().commit()
             # fork shouldn't be visible
             response = self.app.get(base.url(controller='forks', action='forks',
                                         repo_name=repo_name))
             response.mustcontain('There are no forks yet')
         finally:
             RepoModel().grant_user_permission(repo=forks[0],
                                               user=usr, perm='repository.read')
             RepoModel().grant_user_permission(repo=forks[0],
                                               user=default_user, perm='repository.read')
             RepoModel().delete(repo=forks[0])
 class TestGIT(_BaseTestCase):
     REPO = base.GIT_REPO
     NEW_REPO = base.NEW_GIT_REPO
     REPO_TYPE = 'git'
     REPO_FORK = base.GIT_FORK
 class TestHG(_BaseTestCase):
     REPO = base.HG_REPO
     NEW_REPO = base.NEW_HG_REPO
     REPO_TYPE = 'hg'
     REPO_FORK = base.HG_FORK

kallithea/tests/vcs/test_hg.py

➞

Show inline comments

 import os
 import mock
 import pytest
 from kallithea.lib.vcs.backends.hg import MercurialChangeset, MercurialRepository
 from kallithea.lib.vcs.exceptions import NodeDoesNotExistError, RepositoryError, VCSError
 from kallithea.lib.vcs.nodes import NodeKind, NodeState
 from kallithea.lib.vcs.utils import safe_str
 from kallithea.tests.vcs.conf import TEST_HG_REPO, TEST_HG_REPO_CLONE, TEST_HG_REPO_PULL, TESTS_TMP_PATH
 class TestMercurialRepository(object):
     def __check_for_existing_repo(self):
         if os.path.exists(TEST_HG_REPO_CLONE):
             pytest.fail('Cannot test mercurial clone repo as location %s already '
                       'exists. You should manually remove it first.'
                       % TEST_HG_REPO_CLONE)
     def setup_method(self):
-        self.repo = MercurialRepository(safe_str(TEST_HG_REPO))
         self.repo = MercurialRepository(TEST_HG_REPO)
     def test_wrong_repo_path(self):
         wrong_repo_path = os.path.join(TESTS_TMP_PATH, 'errorrepo')
         with pytest.raises(RepositoryError):
             MercurialRepository(wrong_repo_path)
     def test_unicode_path_repo(self):
         with pytest.raises(VCSError):
             MercurialRepository(u'iShouldFail')
     def test_repo_clone(self):
         self.__check_for_existing_repo()
-        repo = MercurialRepository(safe_str(TEST_HG_REPO))
         repo = MercurialRepository(TEST_HG_REPO)
         repo_clone = MercurialRepository(TEST_HG_REPO_CLONE,
             src_url=TEST_HG_REPO, update_after_clone=True)
         assert len(repo.revisions) == len(repo_clone.revisions)
         # Checking hashes of changesets should be enough
         for changeset in repo.get_changesets():
             raw_id = changeset.raw_id
             assert raw_id == repo_clone.get_changeset(raw_id).raw_id
     def test_repo_clone_with_update(self):
-        repo = MercurialRepository(safe_str(TEST_HG_REPO))
         repo = MercurialRepository(TEST_HG_REPO)
         repo_clone = MercurialRepository(TEST_HG_REPO_CLONE + '_w_update',
             src_url=TEST_HG_REPO, update_after_clone=True)
         assert len(repo.revisions) == len(repo_clone.revisions)
         # check if current workdir was updated
         assert os.path.isfile(
             os.path.join(
                 TEST_HG_REPO_CLONE + '_w_update', 'MANIFEST.in'
+            )
+        )
     def test_repo_clone_without_update(self):
-        repo = MercurialRepository(safe_str(TEST_HG_REPO))
         repo = MercurialRepository(TEST_HG_REPO)
         repo_clone = MercurialRepository(TEST_HG_REPO_CLONE + '_wo_update',
             src_url=TEST_HG_REPO, update_after_clone=False)
         assert len(repo.revisions) == len(repo_clone.revisions)
         assert not os.path.isfile(
             os.path.join(
                 TEST_HG_REPO_CLONE + '_wo_update', 'MANIFEST.in'
+            )
+        )
     def test_pull(self):
         if os.path.exists(TEST_HG_REPO_PULL):
             pytest.fail('Cannot test mercurial pull command as location %s '
                       'already exists. You should manually remove it first'
                       % TEST_HG_REPO_PULL)
         repo_new = MercurialRepository(TEST_HG_REPO_PULL, create=True)
         assert len(self.repo.revisions) > len(repo_new.revisions)
         repo_new.pull(self.repo.path)
         repo_new = MercurialRepository(TEST_HG_REPO_PULL)
         assert len(self.repo.revisions) == len(repo_new.revisions)
     def test_revisions(self):
         # there are 21 revisions at bitbucket now
         # so we can assume they would be available from now on
         subset = set(['b986218ba1c9b0d6a259fac9b050b1724ed8e545',
                  '3d8f361e72ab303da48d799ff1ac40d5ac37c67e',
                  '6cba7170863a2411822803fa77a0a264f1310b35',
                  '56349e29c2af3ac913b28bde9a2c6154436e615b',
                  '2dda4e345facb0ccff1a191052dd1606dba6781d',
                  '6fff84722075f1607a30f436523403845f84cd9e',
                  '7d4bc8ec6be56c0f10425afb40b6fc315a4c25e7',
                  '3803844fdbd3b711175fc3da9bdacfcd6d29a6fb',
                  'dc5d2c0661b61928834a785d3e64a3f80d3aad9c',
                  'be90031137367893f1c406e0a8683010fd115b79',
                  'db8e58be770518cbb2b1cdfa69146e47cd481481',
                  '84478366594b424af694a6c784cb991a16b87c21',
                  '17f8e105dddb9f339600389c6dc7175d395a535c',
                  '20a662e756499bde3095ffc9bc0643d1def2d0eb',
                  '2e319b85e70a707bba0beff866d9f9de032aa4f9',
                  '786facd2c61deb9cf91e9534735124fb8fc11842',
                  '94593d2128d38210a2fcd1aabff6dda0d6d9edf8',
                  'aa6a0de05b7612707db567078e130a6cd114a9a7',
                  'eada5a770da98ab0dd7325e29d00e0714f228d09'
                 ])
         assert subset.issubset(set(self.repo.revisions))
         # check if we have the proper order of revisions
         org = ['b986218ba1c9b0d6a259fac9b050b1724ed8e545',
                 '3d8f361e72ab303da48d799ff1ac40d5ac37c67e',
                 '6cba7170863a2411822803fa77a0a264f1310b35',
                 '56349e29c2af3ac913b28bde9a2c6154436e615b',
                 '2dda4e345facb0ccff1a191052dd1606dba6781d',
                 '6fff84722075f1607a30f436523403845f84cd9e',
                 '7d4bc8ec6be56c0f10425afb40b6fc315a4c25e7',
                 '3803844fdbd3b711175fc3da9bdacfcd6d29a6fb',
                 'dc5d2c0661b61928834a785d3e64a3f80d3aad9c',
                 'be90031137367893f1c406e0a8683010fd115b79',
                 'db8e58be770518cbb2b1cdfa69146e47cd481481',
                 '84478366594b424af694a6c784cb991a16b87c21',
                 '17f8e105dddb9f339600389c6dc7175d395a535c',
                 '20a662e756499bde3095ffc9bc0643d1def2d0eb',
                 '2e319b85e70a707bba0beff866d9f9de032aa4f9',
                 '786facd2c61deb9cf91e9534735124fb8fc11842',
                 '94593d2128d38210a2fcd1aabff6dda0d6d9edf8',
                 'aa6a0de05b7612707db567078e130a6cd114a9a7',
                 'eada5a770da98ab0dd7325e29d00e0714f228d09',
                 '2c1885c735575ca478bf9e17b0029dca68824458',
                 'd9bcd465040bf869799b09ad732c04e0eea99fe9',
                 '469e9c847fe1f6f7a697b8b25b4bc5b48780c1a7',
                 '4fb8326d78e5120da2c7468dcf7098997be385da',
                 '62b4a097164940bd66030c4db51687f3ec035eed',
                 '536c1a19428381cfea92ac44985304f6a8049569',
                 '965e8ab3c44b070cdaa5bf727ddef0ada980ecc4',
                 '9bb326a04ae5d98d437dece54be04f830cf1edd9',
                 'f8940bcb890a98c4702319fbe36db75ea309b475',
                 'ff5ab059786ebc7411e559a2cc309dfae3625a3b',
                 '6b6ad5f82ad5bb6190037671bd254bd4e1f4bf08',
                 'ee87846a61c12153b51543bf860e1026c6d3dcba', ]
         assert org == self.repo.revisions[:31]
     def test_iter_slice(self):
         sliced = list(self.repo[:10])
         itered = list(self.repo)[:10]
         assert sliced == itered
     def test_slicing(self):
         # 4 1 5 10 95
         for sfrom, sto, size in [(0, 4, 4), (1, 2, 1), (10, 15, 5),
                                  (10, 20, 10), (5, 100, 95)]:
             revs = list(self.repo[sfrom:sto])
             assert len(revs) == size
             assert revs[0] == self.repo.get_changeset(sfrom)
             assert revs[-1] == self.repo.get_changeset(sto - 1)
     def test_branches(self):
         # TODO: Need more tests here
         # active branches
         assert 'default' in self.repo.branches
         assert 'stable' in self.repo.branches
         # closed
         assert 'git' in self.repo._get_branches(closed=True)
         assert 'web' in self.repo._get_branches(closed=True)
         for name, id in self.repo.branches.items():
             assert isinstance(self.repo.get_changeset(id), MercurialChangeset)
     def test_tip_in_tags(self):
         # tip is always a tag
         assert 'tip' in self.repo.tags
     def test_tip_changeset_in_tags(self):
         tip = self.repo.get_changeset()
         assert self.repo.tags['tip'] == tip.raw_id
     def test_initial_changeset(self):
         init_chset = self.repo.get_changeset(0)
         assert init_chset.message == 'initial import'
         assert init_chset.author == 'Marcin Kuzminski <marcin@python-blog.com>'
         assert sorted(init_chset._file_paths) == sorted([
             'vcs/__init__.py',
             'vcs/backends/BaseRepository.py',
             'vcs/backends/__init__.py',
         ])
         assert sorted(init_chset._dir_paths) == sorted(['', 'vcs', 'vcs/backends'])
         with pytest.raises(NodeDoesNotExistError):
             init_chset.get_node(path='foobar')
         node = init_chset.get_node('vcs/')
         assert hasattr(node, 'kind')
         assert node.kind == NodeKind.DIR
         node = init_chset.get_node('vcs')
         assert hasattr(node, 'kind')
         assert node.kind == NodeKind.DIR
         node = init_chset.get_node('vcs/__init__.py')
         assert hasattr(node, 'kind')
         assert node.kind == NodeKind.FILE
     def test_not_existing_changeset(self):
         # rawid
         with pytest.raises(RepositoryError):
             self.repo.get_changeset('abcd' * 10)
         # shortid
         with pytest.raises(RepositoryError):
             self.repo.get_changeset('erro' * 4)
         # numeric
         with pytest.raises(RepositoryError):
             self.repo.get_changeset(self.repo.count() + 1)
         # Small chance we ever get to this one
         revision = pow(2, 30)
         with pytest.raises(RepositoryError):
             self.repo.get_changeset(revision)
     def test_changeset10(self):
         chset10 = self.repo.get_changeset(10)
         readme = b"""===
 VCS
 ===
 Various Version Control System management abstraction layer for Python.
 Introduction
 ------------
 TODO: To be written...
 """
         node = chset10.get_node('README.rst')
         assert node.kind == NodeKind.FILE
         assert node.content == readme
     @mock.patch('mercurial.mdiff.diffopts')
     def test_get_diff_does_not_sanitize_zero_context(self, mock_diffopts):
         zero_context = 0
         self.repo.get_diff(0, 1, 'foo', context=zero_context)
         mock_diffopts.assert_called_once_with(git=True, showfunc=True, ignorews=False, context=zero_context)
     @mock.patch('mercurial.mdiff.diffopts')
     def test_get_diff_sanitizes_negative_context(self, mock_diffopts):
         negative_context = -10
         zero_context = 0
         self.repo.get_diff(0, 1, 'foo', context=negative_context)
         mock_diffopts.assert_called_once_with(git=True, showfunc=True, ignorews=False, context=zero_context)
 class TestMercurialChangeset(object):
     def setup_method(self):
-        self.repo = MercurialRepository(safe_str(TEST_HG_REPO))
         self.repo = MercurialRepository(TEST_HG_REPO)
     def _test_equality(self, changeset):
         revision = changeset.revision
         assert changeset == self.repo.get_changeset(revision)
     def test_equality(self):
         revs = [0, 10, 20]
         changesets = [self.repo.get_changeset(rev) for rev in revs]
         for changeset in changesets:
             self._test_equality(changeset)
     def test_default_changeset(self):
         tip = self.repo.get_changeset('tip')
         assert tip == self.repo.get_changeset()
         assert tip == self.repo.get_changeset(revision=None)
         assert tip == list(self.repo[-1:])[0]
     def test_root_node(self):
         tip = self.repo.get_changeset('tip')
         assert tip.root is tip.get_node('')
     def test_lazy_fetch(self):
         """
         Test if changeset's nodes expands and are cached as we walk through
         the revision. This test is somewhat hard to write as order of tests
         is a key here. Written by running command after command in a shell.
         """
         chset = self.repo.get_changeset(45)
         assert len(chset.nodes) == 0
         root = chset.root
         assert len(chset.nodes) == 1
         assert len(root.nodes) == 8
         # accessing root.nodes updates chset.nodes
         assert len(chset.nodes) == 9
         docs = root.get_node('docs')
         # we haven't yet accessed anything new as docs dir was already cached
         assert len(chset.nodes) == 9
         assert len(docs.nodes) == 8
         # accessing docs.nodes updates chset.nodes
         assert len(chset.nodes) == 17
         assert docs is chset.get_node('docs')
         assert docs is root.nodes[0]
         assert docs is root.dirs[0]
         assert docs is chset.get_node('docs')
     def test_nodes_with_changeset(self):
         chset = self.repo.get_changeset(45)
         root = chset.root
         docs = root.get_node('docs')
         assert docs is chset.get_node('docs')
         api = docs.get_node('api')
         assert api is chset.get_node('docs/api')
         index = api.get_node('index.rst')
         assert index is chset.get_node('docs/api/index.rst')
         assert index is chset.get_node('docs').get_node('api').get_node('index.rst')
     def test_branch_and_tags(self):
         chset0 = self.repo.get_changeset(0)
         assert chset0.branch == 'default'
         assert chset0.branches == ['default']
         assert chset0.tags == []
         chset10 = self.repo.get_changeset(10)
         assert chset10.branch == 'default'
         assert chset10.branches == ['default']
         assert chset10.tags == []
         chset44 = self.repo.get_changeset(44)
         assert chset44.branch == 'web'
         assert chset44.branches == ['web']
         tip = self.repo.get_changeset('tip')
         assert 'tip' in tip.tags
     def _test_file_size(self, revision, path, size):
         node = self.repo.get_changeset(revision).get_node(path)
         assert node.is_file()
         assert node.size == size
     def test_file_size(self):
         to_check = (
             (10, 'setup.py', 1068),
             (20, 'setup.py', 1106),
             (60, 'setup.py', 1074),
             (10, 'vcs/backends/base.py', 2921),
             (20, 'vcs/backends/base.py', 3936),
             (60, 'vcs/backends/base.py', 6189),
+        )
         for revision, path, size in to_check:
             self._test_file_size(revision, path, size)
     def _test_dir_size(self, revision, path, size):
         node = self.repo.get_changeset(revision).get_node(path)
         assert not node.is_file()
         assert node.size == size
     def test_dir_size(self):
         to_check = (
             ('96507bd11ecc', '/', 682421),
             ('a53d9201d4bc', '/', 682410),
             ('90243de06161', '/', 682006),
+        )
         for revision, path, size in to_check:
             self._test_dir_size(revision, path, size)
     def test_repo_size(self):
         assert self.repo.size == 682421
     def test_file_history(self):
         # we can only check if those revisions are present in the history
         # as we cannot update this test every time file is changed
         files = {
             'setup.py': [7, 18, 45, 46, 47, 69, 77],
             'vcs/nodes.py': [7, 8, 24, 26, 30, 45, 47, 49, 56, 57, 58, 59, 60,
 , 73, 76],
             'vcs/backends/hg.py': [4, 5, 6, 11, 12, 13, 14, 15, 16, 21, 22, 23,
 , 27, 28, 30, 31, 33, 35, 36, 37, 38, 39, 40, 41, 44, 45, 47,
 , 49, 53, 54, 55, 58, 60, 61, 67, 68, 69, 70, 73, 77, 78, 79,
 ],
+        }
         for path, revs in files.items():
             tip = self.repo.get_changeset(revs[-1])
             node = tip.get_node(path)
             node_revs = [chset.revision for chset in node.history]
             assert set(revs).issubset(set(node_revs)), \
                 "We assumed that %s is subset of revisions for which file %s " \
                 "has been changed, and history of that node returned: %s" \
                 % (revs, path, node_revs)
     def test_file_annotate(self):
         files = {
                  'vcs/backends/__init__.py':
                   {89: {'lines_no': 31,
                         'changesets': [32, 32, 61, 32, 32, 37, 32, 32, 32, 44,
 , 37, 37, 37, 45, 37, 44, 37, 37, 37,
 , 32, 32, 32, 37, 32, 37, 37, 32,
 , 32]},
 : {'lines_no': 1,
                         'changesets': [4]},
 : {'lines_no': 31,
                         'changesets': [32, 32, 45, 32, 32, 37, 32, 32, 32, 44,
 , 37, 37, 37, 45, 37, 44, 37, 37, 37,
 , 32, 32, 32, 37, 32, 37, 37, 32,
 , 32]}},
                  'vcs/exceptions.py':
                  {89: {'lines_no': 18,
                        'changesets': [16, 16, 16, 16, 16, 16, 16, 16, 16, 16,
 , 16, 17, 16, 16, 18, 18, 18]},
 : {'lines_no': 18,
                        'changesets': [16, 16, 16, 16, 16, 16, 16, 16, 16, 16,
 , 16, 17, 16, 16, 18, 18, 18]},
 : {'lines_no': 18, 'changesets': [16, 16, 16, 16, 16, 16,
 , 16, 16, 16, 16, 16,
 , 16, 16, 18, 18, 18]}},
                  'MANIFEST.in': {89: {'lines_no': 5,
                                       'changesets': [7, 7, 7, 71, 71]},
 : {'lines_no': 3,
                                       'changesets': [7, 7, 7]},
 : {'lines_no': 3,
                                      'changesets': [7, 7, 7]}}}
         for fname, revision_dict in files.items():
             for rev, data in revision_dict.items():
                 cs = self.repo.get_changeset(rev)
                 l1_1 = [x[1] for x in cs.get_file_annotate(fname)]
                 l1_2 = [x[2]().raw_id for x in cs.get_file_annotate(fname)]
                 assert l1_1 == l1_2
                 l1 = l1_2 = [x[2]().revision for x in cs.get_file_annotate(fname)]
                 l2 = files[fname][rev]['changesets']
                 assert l1 == l2, "The lists of revision for %s@rev%s" \
                     "from annotation list should match each other," \
                     "got \n%s \nvs \n%s " % (fname, rev, l1, l2)
     def test_changeset_state(self):
         """
         Tests which files have been added/changed/removed at particular revision
         """
         # rev 46ad32a4f974:
         # hg st --rev 46ad32a4f974
         #    changed: 13
         #    added:   20
         #    removed: 1
         changed = set(['.hgignore'
             , 'README.rst', 'docs/conf.py', 'docs/index.rst', 'setup.py'
             , 'tests/test_hg.py', 'tests/test_nodes.py', 'vcs/__init__.py'
             , 'vcs/backends/__init__.py', 'vcs/backends/base.py'
             , 'vcs/backends/hg.py', 'vcs/nodes.py', 'vcs/utils/__init__.py'])

kallithea/tests/vcs/test_vcs.py

➞

Show inline comments

 import os
 import shutil
 import pytest
 from kallithea.lib.vcs import VCSError, get_backend, get_repo
 from kallithea.lib.vcs.backends.hg import MercurialRepository
 from kallithea.lib.vcs.utils import safe_str
 from kallithea.tests.vcs.conf import TEST_GIT_REPO, TEST_HG_REPO, TESTS_TMP_PATH
 class TestVCS(object):
     """
     Tests for main module's methods.
     """
     def test_get_backend(self):
         hg = get_backend('hg')
         assert hg == MercurialRepository
     def test_alias_detect_hg(self):
         alias = 'hg'
         path = TEST_HG_REPO
         backend = get_backend(alias)
-        repo = backend(safe_str(path))
         repo = backend(path)
         assert 'hg' == repo.alias
     def test_alias_detect_git(self):
         alias = 'git'
         path = TEST_GIT_REPO
         backend = get_backend(alias)
-        repo = backend(safe_str(path))
         repo = backend(path)
         assert 'git' == repo.alias
     def test_wrong_alias(self):
         alias = 'wrong_alias'
         with pytest.raises(VCSError):
             get_backend(alias)
     def test_get_repo(self):
         alias = 'hg'
         path = TEST_HG_REPO
         backend = get_backend(alias)
-        repo = backend(safe_str(path))
         repo = backend(path)
         assert repo.__class__ == get_repo(safe_str(path), alias).__class__
         assert repo.path == get_repo(safe_str(path), alias).path
         assert repo.__class__ == get_repo(path, alias).__class__
         assert repo.path == get_repo(path, alias).path
     def test_get_repo_autoalias_hg(self):
         alias = 'hg'
         path = TEST_HG_REPO
         backend = get_backend(alias)
-        repo = backend(safe_str(path))
         repo = backend(path)
         assert repo.__class__ == get_repo(safe_str(path)).__class__
         assert repo.path == get_repo(safe_str(path)).path
         assert repo.__class__ == get_repo(path).__class__
         assert repo.path == get_repo(path).path
     def test_get_repo_autoalias_git(self):
         alias = 'git'
         path = TEST_GIT_REPO
         backend = get_backend(alias)
-        repo = backend(safe_str(path))
         repo = backend(path)
         assert repo.__class__ == get_repo(safe_str(path)).__class__
         assert repo.path == get_repo(safe_str(path)).path
         assert repo.__class__ == get_repo(path).__class__
         assert repo.path == get_repo(path).path
     def test_get_repo_err(self):
         blank_repo_path = os.path.join(TESTS_TMP_PATH, 'blank-error-repo')
         if os.path.isdir(blank_repo_path):
             shutil.rmtree(blank_repo_path)
         os.mkdir(blank_repo_path)
         with pytest.raises(VCSError):
             get_repo(blank_repo_path)
         with pytest.raises(VCSError):
             get_repo(blank_repo_path + 'non_existing')
     def test_get_repo_multialias(self):
         multialias_repo_path = os.path.join(TESTS_TMP_PATH, 'hg-git-repo')
         if os.path.isdir(multialias_repo_path):
             shutil.rmtree(multialias_repo_path)
         os.mkdir(multialias_repo_path)
         os.mkdir(os.path.join(multialias_repo_path, '.git'))
         os.mkdir(os.path.join(multialias_repo_path, '.hg'))
         with pytest.raises(VCSError):
             get_repo(multialias_repo_path)

0 comments (0 inline, 0 general)