here:
# since the content in the other cell also is wrapped in a div,
@@ -176,7 +166,7 @@ class AnnotateHtmlFormatter(HtmlFormatte
'')
- yield 0, dummyoutfile.getvalue()
+ yield 0, ''.join(inner_lines)
yield 0, ' " % (self.user_id, self.username)
+ return "<%s %s: %r>" % (self.__class__.__name__, self.user_id, self.username)
def to_cookie(self):
""" Serializes this login session to a cookie `dict`. """
@@ -566,14 +539,10 @@ class AuthUser(object):
)
@classmethod
- def get_allowed_ips(cls, user_id, cache=False):
+ def get_allowed_ips(cls, user_id):
_set = set()
- default_ips = UserIpMap.query().filter(UserIpMap.user_id ==
- User.get_default_user(cache=True).user_id)
- if cache:
- default_ips = default_ips.options(FromCache("sql_cache_short",
- "get_user_ips_default"))
+ default_ips = UserIpMap.query().filter(UserIpMap.user_id == kallithea.DEFAULT_USER_ID)
for ip in default_ips:
try:
_set.add(ip.ip_addr)
@@ -583,9 +552,6 @@ class AuthUser(object):
pass
user_ips = UserIpMap.query().filter(UserIpMap.user_id == user_id)
- if cache:
- user_ips = user_ips.options(FromCache("sql_cache_short",
- "get_user_ips_%s" % user_id))
for ip in user_ips:
try:
_set.add(ip.ip_addr)
@@ -596,24 +562,6 @@ class AuthUser(object):
return _set or set(['0.0.0.0/0', '::/0'])
-def set_available_permissions(config):
- """
- This function will propagate globals with all available defined
- permission given in db. We don't want to check each time from db for new
- permissions since adding a new permission also requires application restart
- ie. to decorate new views with the newly created permission
-
- :param config: current config instance
-
- """
- log.info('getting information about all available permissions')
- try:
- all_perms = Session().query(Permission).all()
- config['available_permissions'] = [x.permission_name for x in all_perms]
- finally:
- Session.remove()
-
-
#==============================================================================
# CHECK DECORATORS
#==============================================================================
@@ -778,7 +726,7 @@ class _PermsFunction(object):
def __init__(self, *required_perms):
self.required_perms = required_perms # usually very short - a list is thus fine
- def __nonzero__(self):
+ def __bool__(self):
""" Defend against accidentally forgetting to call the object
and instead evaluating it directly in a boolean context,
which could have security implications.
@@ -835,10 +783,6 @@ class HasPermissionAnyMiddleware(object)
self.required_perms = set(perms)
def __call__(self, authuser, repo_name, purpose=None):
- # repo_name MUST be unicode, since we handle keys in ok
- # dict by unicode
- repo_name = safe_unicode(repo_name)
-
try:
ok = authuser.permissions['repositories'][repo_name] in self.required_perms
except KeyError:
diff --git a/kallithea/lib/auth_modules/__init__.py b/kallithea/lib/auth_modules/__init__.py
--- a/kallithea/lib/auth_modules/__init__.py
+++ b/kallithea/lib/auth_modules/__init__.py
@@ -20,7 +20,7 @@ import logging
import traceback
from kallithea.lib.auth import AuthUser, PasswordGenerator
-from kallithea.lib.compat import formatted_json, hybrid_property
+from kallithea.lib.compat import hybrid_property
from kallithea.lib.utils2 import str2bool
from kallithea.model.db import Setting, User
from kallithea.model.meta import Session
@@ -136,9 +136,6 @@ class KallitheaAuthPluginBase(object):
username)
if username:
user = User.get_by_username_or_email(username)
- if user is None:
- log.debug('Fallback to fetch user in case insensitive mode')
- user = User.get_by_username(username, case_insensitive=True)
else:
log.debug('provided username:`%s` is empty skipping...', username)
return user
@@ -286,11 +283,11 @@ def loadplugin(plugin):
ImportError -- if we couldn't import the plugin at all
"""
log.debug("Importing %s", plugin)
- if not plugin.startswith(u'kallithea.lib.auth_modules.auth_'):
- parts = plugin.split(u'.lib.auth_modules.auth_', 1)
+ if not plugin.startswith('kallithea.lib.auth_modules.auth_'):
+ parts = plugin.split('.lib.auth_modules.auth_', 1)
if len(parts) == 2:
_module, pn = parts
- plugin = u'kallithea.lib.auth_modules.auth_' + pn
+ plugin = 'kallithea.lib.auth_modules.auth_' + pn
PLUGIN_CLASS_NAME = "KallitheaAuthPlugin"
try:
module = importlib.import_module(plugin)
@@ -309,7 +306,7 @@ def loadplugin(plugin):
"a subclass of %s" % (plugin, KallitheaAuthPluginBase))
plugin = pluginclass()
- if plugin.plugin_settings.im_func != KallitheaAuthPluginBase.plugin_settings.im_func:
+ if plugin.plugin_settings.__func__ != KallitheaAuthPluginBase.plugin_settings:
raise TypeError("Authentication class %s.KallitheaAuthPluginBase "
"has overridden the plugin_settings method, which is "
"forbidden." % plugin)
@@ -351,7 +348,7 @@ def authenticate(username, password, env
conf_key = "auth_%s_%s" % (plugin_name, v["name"])
setting = Setting.get_by_name(conf_key)
plugin_settings[v["name"]] = setting.app_settings_value if setting else None
- log.debug('Settings for auth plugin %s:\n%s', plugin_name, formatted_json(plugin_settings))
+ log.debug('Settings for auth plugin %s: %s', plugin_name, plugin_settings)
if not str2bool(plugin_settings["enabled"]):
log.info("Authentication plugin %s is disabled, skipping for %s",
diff --git a/kallithea/lib/auth_modules/auth_container.py b/kallithea/lib/auth_modules/auth_container.py
--- a/kallithea/lib/auth_modules/auth_container.py
+++ b/kallithea/lib/auth_modules/auth_container.py
@@ -29,7 +29,7 @@ import logging
from kallithea.lib import auth_modules
from kallithea.lib.compat import hybrid_property
-from kallithea.lib.utils2 import safe_str, safe_unicode, str2bool
+from kallithea.lib.utils2 import str2bool
from kallithea.model.db import Setting
@@ -180,7 +180,7 @@ class KallitheaAuthPlugin(auth_modules.K
# only way to log in is using environ
username = None
if userobj:
- username = safe_str(getattr(userobj, 'username'))
+ username = getattr(userobj, 'username')
if not username:
# we don't have any objects in DB, user doesn't exist, extract
@@ -199,8 +199,8 @@ class KallitheaAuthPlugin(auth_modules.K
user_data = {
'username': username,
- 'firstname': safe_unicode(firstname or username),
- 'lastname': safe_unicode(lastname or ''),
+ 'firstname': firstname or username,
+ 'lastname': lastname or '',
'groups': [],
'email': email or '',
'admin': admin or False,
diff --git a/kallithea/lib/auth_modules/auth_crowd.py b/kallithea/lib/auth_modules/auth_crowd.py
--- a/kallithea/lib/auth_modules/auth_crowd.py
+++ b/kallithea/lib/auth_modules/auth_crowd.py
@@ -28,10 +28,12 @@ Original author and date, and relevant c
import base64
import logging
-import urllib2
+import urllib.parse
+import urllib.request
-from kallithea.lib import auth_modules
-from kallithea.lib.compat import formatted_json, hybrid_property, json
+from kallithea.lib import auth_modules, ext_json
+from kallithea.lib.compat import hybrid_property
+from kallithea.lib.utils2 import ascii_bytes, ascii_str, safe_bytes
log = logging.getLogger(__name__)
@@ -71,10 +73,10 @@ class CrowdServer(object):
self._make_opener()
def _make_opener(self):
- mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
+ mgr = urllib.request.HTTPPasswordMgrWithDefaultRealm()
mgr.add_password(None, self._uri, self.user, self.passwd)
- handler = urllib2.HTTPBasicAuthHandler(mgr)
- self.opener = urllib2.build_opener(handler)
+ handler = urllib.request.HTTPBasicAuthHandler(mgr)
+ self.opener = urllib.request.build_opener(handler)
def _request(self, url, body=None, headers=None,
method=None, noformat=False,
@@ -82,14 +84,12 @@ class CrowdServer(object):
_headers = {"Content-type": "application/json",
"Accept": "application/json"}
if self.user and self.passwd:
- authstring = base64.b64encode("%s:%s" % (self.user, self.passwd))
+ authstring = ascii_str(base64.b64encode(safe_bytes("%s:%s" % (self.user, self.passwd))))
_headers["Authorization"] = "Basic %s" % authstring
if headers:
_headers.update(headers)
- log.debug("Sent crowd: \n%s",
- formatted_json({"url": url, "body": body,
- "headers": _headers}))
- req = urllib2.Request(url, body, _headers)
+ log.debug("Sent to crowd at %s:\nHeaders: %s\nBody:\n%s", url, _headers, body)
+ req = urllib.request.Request(url, body, _headers)
if method:
req.get_method = lambda: method
@@ -103,7 +103,7 @@ class CrowdServer(object):
rval["status"] = True
rval["error"] = "Response body was empty"
elif not noformat:
- rval = json.loads(msg)
+ rval = ext_json.loads(msg)
rval["status"] = True
else:
rval = "".join(rdoc.readlines())
@@ -120,14 +120,14 @@ class CrowdServer(object):
"""Authenticate a user against crowd. Returns brief information about
the user."""
url = ("%s/rest/usermanagement/%s/authentication?username=%s"
- % (self._uri, self._version, urllib2.quote(username)))
- body = json.dumps({"value": password})
+ % (self._uri, self._version, urllib.parse.quote(username)))
+ body = ascii_bytes(ext_json.dumps({"value": password}))
return self._request(url, body)
def user_groups(self, username):
"""Retrieve a list of groups to which this user belongs."""
url = ("%s/rest/usermanagement/%s/user/group/nested?username=%s"
- % (self._uri, self._version, urllib2.quote(username)))
+ % (self._uri, self._version, urllib.parse.quote(username)))
return self._request(url)
@@ -209,11 +209,11 @@ class KallitheaAuthPlugin(auth_modules.K
log.debug('Empty username or password skipping...')
return None
- log.debug("Crowd settings: \n%s", formatted_json(settings))
+ log.debug("Crowd settings: %s", settings)
server = CrowdServer(**settings)
server.set_credentials(settings["app_name"], settings["app_password"])
crowd_user = server.user_auth(username, password)
- log.debug("Crowd returned: \n%s", formatted_json(crowd_user))
+ log.debug("Crowd returned: %s", crowd_user)
if not crowd_user["status"]:
log.error('Crowd authentication as %s returned no status', username)
return None
@@ -223,7 +223,7 @@ class KallitheaAuthPlugin(auth_modules.K
return None
res = server.user_groups(crowd_user["name"])
- log.debug("Crowd groups: \n%s", formatted_json(res))
+ log.debug("Crowd groups: %s", res)
crowd_user["groups"] = [x["name"] for x in res["groups"]]
# old attrs fetched from Kallithea database
@@ -246,7 +246,7 @@ class KallitheaAuthPlugin(auth_modules.K
for group in settings["admin_groups"].split(","):
if group in user_data["groups"]:
user_data["admin"] = True
- log.debug("Final crowd user object: \n%s", formatted_json(user_data))
+ log.debug("Final crowd user object: %s", user_data)
log.info('user %s authenticated correctly', user_data['username'])
return user_data
diff --git a/kallithea/lib/auth_modules/auth_internal.py b/kallithea/lib/auth_modules/auth_internal.py
--- a/kallithea/lib/auth_modules/auth_internal.py
+++ b/kallithea/lib/auth_modules/auth_internal.py
@@ -29,8 +29,7 @@ Original author and date, and relevant c
import logging
from kallithea.lib import auth_modules
-from kallithea.lib.compat import formatted_json, hybrid_property
-from kallithea.model.db import User
+from kallithea.lib.compat import hybrid_property
log = logging.getLogger(__name__)
@@ -77,7 +76,7 @@ class KallitheaAuthPlugin(auth_modules.K
"admin": userobj.admin,
"extern_name": userobj.user_id,
}
- log.debug(formatted_json(user_data))
+ log.debug('user data: %s', user_data)
from kallithea.lib import auth
password_match = auth.check_password(password, userobj.password)
diff --git a/kallithea/lib/auth_modules/auth_ldap.py b/kallithea/lib/auth_modules/auth_ldap.py
--- a/kallithea/lib/auth_modules/auth_ldap.py
+++ b/kallithea/lib/auth_modules/auth_ldap.py
@@ -31,7 +31,7 @@ import logging
from kallithea.lib import auth_modules
from kallithea.lib.compat import hybrid_property
from kallithea.lib.exceptions import LdapConnectionError, LdapImportError, LdapPasswordError, LdapUsernameError
-from kallithea.lib.utils2 import safe_str, safe_unicode
+from kallithea.lib.utils2 import safe_str
log = logging.getLogger(__name__)
@@ -70,11 +70,11 @@ class AuthLdap(object):
port)
for host in server.split(',')))
- self.LDAP_BIND_DN = safe_str(bind_dn)
- self.LDAP_BIND_PASS = safe_str(bind_pass)
+ self.LDAP_BIND_DN = bind_dn
+ self.LDAP_BIND_PASS = bind_pass
- self.BASE_DN = safe_str(base_dn)
- self.LDAP_FILTER = safe_str(ldap_filter)
+ self.BASE_DN = base_dn
+ self.LDAP_FILTER = ldap_filter
self.SEARCH_SCOPE = getattr(ldap, 'SCOPE_%s' % search_scope)
self.attr_login = attr_login
@@ -139,7 +139,7 @@ class AuthLdap(object):
try:
log.debug('Trying simple bind with %s', dn)
- server.simple_bind_s(dn, safe_str(password))
+ server.simple_bind_s(dn, password)
results = server.search_ext_s(dn, ldap.SCOPE_BASE,
'(objectClass=*)')
if len(results) == 1:
@@ -328,7 +328,8 @@ class KallitheaAuthPlugin(auth_modules.K
(user_dn, ldap_attrs) = aldap.authenticate_ldap(username, password)
log.debug('Got ldap DN response %s', user_dn)
- get_ldap_attr = lambda k: ldap_attrs.get(settings.get(k), [''])[0]
+ def get_ldap_attr(k):
+ return safe_str(ldap_attrs.get(settings.get(k), [b''])[0])
# old attrs fetched from Kallithea database
admin = getattr(userobj, 'admin', False)
@@ -338,8 +339,8 @@ class KallitheaAuthPlugin(auth_modules.K
user_data = {
'username': username,
- 'firstname': safe_unicode(get_ldap_attr('attr_firstname') or firstname),
- 'lastname': safe_unicode(get_ldap_attr('attr_lastname') or lastname),
+ 'firstname': get_ldap_attr('attr_firstname') or firstname,
+ 'lastname': get_ldap_attr('attr_lastname') or lastname,
'groups': [],
'email': get_ldap_attr('attr_email') or email,
'admin': admin,
diff --git a/kallithea/lib/auth_modules/auth_pam.py b/kallithea/lib/auth_modules/auth_pam.py
--- a/kallithea/lib/auth_modules/auth_pam.py
+++ b/kallithea/lib/auth_modules/auth_pam.py
@@ -32,7 +32,7 @@ import threading
import time
from kallithea.lib import auth_modules
-from kallithea.lib.compat import formatted_json, hybrid_property
+from kallithea.lib.compat import hybrid_property
try:
@@ -142,7 +142,7 @@ class KallitheaAuthPlugin(auth_modules.K
log.warning("Cannot extract additional info for PAM user %s", username)
pass
- log.debug("pamuser: \n%s", formatted_json(user_data))
+ log.debug("pamuser: %s", user_data)
log.info('user %s authenticated correctly', user_data['username'])
return user_data
diff --git a/kallithea/lib/base.py b/kallithea/lib/base.py
--- a/kallithea/lib/base.py
+++ b/kallithea/lib/base.py
@@ -28,9 +28,9 @@ Original author and date, and relevant c
:license: GPLv3, see LICENSE.md for more details.
"""
+import base64
import datetime
import logging
-import time
import traceback
import warnings
@@ -45,12 +45,11 @@ from tg.i18n import ugettext as _
from kallithea import BACKENDS, __version__
from kallithea.config.routing import url
-from kallithea.lib import auth_modules
+from kallithea.lib import auth_modules, ext_json
from kallithea.lib.auth import AuthUser, HasPermissionAnyMiddleware
-from kallithea.lib.compat import json
from kallithea.lib.exceptions import UserCreationError
from kallithea.lib.utils import get_repo_slug, is_valid_repo
-from kallithea.lib.utils2 import AttributeDict, safe_int, safe_str, safe_unicode, set_hook_environment, str2bool
+from kallithea.lib.utils2 import AttributeDict, ascii_bytes, safe_int, safe_str, set_hook_environment, str2bool
from kallithea.lib.vcs.exceptions import ChangesetDoesNotExistError, EmptyRepositoryError, RepositoryError
from kallithea.model import meta
from kallithea.model.db import PullRequest, Repository, Setting, User
@@ -97,12 +96,18 @@ def _get_ip_addr(environ):
return _filter_proxy(ip)
-def _get_access_path(environ):
- """Return PATH_INFO from environ ... using tg.original_request if available."""
+def get_path_info(environ):
+ """Return PATH_INFO from environ ... using tg.original_request if available.
+
+ In Python 3 WSGI, PATH_INFO is a unicode str, but kind of contains encoded
+ bytes. The code points are guaranteed to only use the lower 8 bit bits, and
+ encoding the string with the 1:1 encoding latin1 will give the
+ corresponding byte string ... which then can be decoded to proper unicode.
+ """
org_req = environ.get('tg.original_request')
if org_req is not None:
environ = org_req.environ
- return environ.get('PATH_INFO')
+ return safe_str(environ['PATH_INFO'].encode('latin1'))
def log_in_user(user, remember, is_external_auth, ip_addr):
@@ -172,7 +177,7 @@ class BasicAuth(paste.auth.basic.AuthBas
(authmeth, auth) = authorization.split(' ', 1)
if 'basic' != authmeth.lower():
return self.build_authentication(environ)
- auth = auth.strip().decode('base64')
+ auth = safe_str(base64.b64decode(auth.strip()))
_parts = auth.split(':', 1)
if len(_parts) == 2:
username, password = _parts
@@ -218,7 +223,7 @@ class BaseVCSController(object):
Returns (None, wsgi_app) to send the wsgi_app response to the client.
"""
# Use anonymous access if allowed for action on repo.
- default_user = User.get_default_user(cache=True)
+ default_user = User.get_default_user()
default_authuser = AuthUser.make(dbuser=default_user, ip_addr=ip_addr)
if default_authuser is None:
log.debug('No anonymous access at all') # move on to proper user auth
@@ -242,7 +247,7 @@ class BaseVCSController(object):
# If not authenticated by the container, running basic auth
if not username:
- self.authenticate.realm = safe_str(self.config['realm'])
+ self.authenticate.realm = self.config['realm']
result = self.authenticate(environ)
if isinstance(result, str):
paste.httpheaders.AUTH_TYPE.update(environ, 'basic')
@@ -273,11 +278,8 @@ class BaseVCSController(object):
def _check_permission(self, action, authuser, repo_name):
"""
- Checks permissions using action (push/pull) user and repository
- name
-
- :param action: 'push' or 'pull' action
- :param user: `User` instance
+ :param action: 'push' or 'pull'
+ :param user: `AuthUser` instance
:param repo_name: repository name
"""
if action == 'push':
@@ -286,7 +288,7 @@ class BaseVCSController(object):
repo_name):
return False
- else:
+ elif action == 'pull':
#any other action need at least read permission
if not HasPermissionAnyMiddleware('repository.read',
'repository.write',
@@ -294,13 +296,15 @@ class BaseVCSController(object):
repo_name):
return False
+ else:
+ assert False, action
+
return True
def _get_ip_addr(self, environ):
return _get_ip_addr(environ)
def __call__(self, environ, start_response):
- start = time.time()
try:
# try parsing a request for this VCS - if it fails, call the wrapped app
parsed_request = self.parse_request(environ)
@@ -334,7 +338,7 @@ class BaseVCSController(object):
try:
log.info('%s action on %s repo "%s" by "%s" from %s',
- parsed_request.action, self.scm_alias, parsed_request.repo_name, safe_str(user.username), ip_addr)
+ parsed_request.action, self.scm_alias, parsed_request.repo_name, user.username, ip_addr)
app = self._make_app(parsed_request)
return app(environ, start_response)
except Exception:
@@ -343,10 +347,6 @@ class BaseVCSController(object):
except webob.exc.HTTPException as e:
return e(environ, start_response)
- finally:
- log_ = logging.getLogger('kallithea.' + self.__class__.__name__)
- log_.debug('Request time: %.3fs', time.time() - start)
- meta.Session.remove()
class BaseController(TGController):
@@ -413,7 +413,7 @@ class BaseController(TGController):
# END CONFIG VARS
c.repo_name = get_repo_slug(request) # can be empty
- c.backends = BACKENDS.keys()
+ c.backends = list(BACKENDS)
self.cut_off_limit = safe_int(config.get('cut_off_limit'))
@@ -454,7 +454,7 @@ class BaseController(TGController):
return log_in_user(user, remember=False, is_external_auth=True, ip_addr=ip_addr)
# User is default user (if active) or anonymous
- default_user = User.get_default_user(cache=True)
+ default_user = User.get_default_user()
authuser = AuthUser.make(dbuser=default_user, ip_addr=ip_addr)
if authuser is None: # fall back to anonymous
authuser = AuthUser(dbuser=default_user) # TODO: somehow use .make?
@@ -529,9 +529,9 @@ class BaseController(TGController):
request.ip_addr = ip_addr
request.needs_csrf_check = needs_csrf_check
- log.info('IP: %s User: %s accessed %s',
+ log.info('IP: %s User: %s Request: %s',
request.ip_addr, request.authuser,
- safe_unicode(_get_access_path(environ)),
+ get_path_info(environ),
)
return super(BaseController, self).__call__(environ, context)
except webob.exc.HTTPException as e:
@@ -552,13 +552,13 @@ class BaseRepoController(BaseController)
def _before(self, *args, **kwargs):
super(BaseRepoController, self)._before(*args, **kwargs)
- if c.repo_name: # extracted from routes
+ if c.repo_name: # extracted from request by base-base BaseController._before
_dbr = Repository.get_by_repo_name(c.repo_name)
if not _dbr:
return
log.debug('Found repository in database %s with state `%s`',
- safe_unicode(_dbr), safe_unicode(_dbr.repo_state))
+ _dbr, _dbr.repo_state)
route = getattr(request.environ.get('routes.route'), 'name', '')
# allow to delete repos that are somehow damages in filesystem
@@ -608,7 +608,7 @@ class BaseRepoController(BaseController)
raise webob.exc.HTTPNotFound()
except RepositoryError as e:
log.error(traceback.format_exc())
- h.flash(safe_str(e), category='error')
+ h.flash(e, category='error')
raise webob.exc.HTTPBadRequest()
@@ -634,7 +634,7 @@ def jsonify(func, *args, **kwargs):
warnings.warn(msg, Warning, 2)
log.warning(msg)
log.debug("Returning JSON wrapped action output")
- return json.dumps(data, encoding='utf-8')
+ return ascii_bytes(ext_json.dumps(data))
@decorator.decorator
def IfSshEnabled(func, *args, **kwargs):
diff --git a/kallithea/lib/caching_query.py b/kallithea/lib/caching_query.py
deleted file mode 100644
--- a/kallithea/lib/caching_query.py
+++ /dev/null
@@ -1,240 +0,0 @@
-"""caching_query.py
-
-Represent persistence structures which allow the usage of
-Beaker caching with SQLAlchemy.
-
-The three new concepts introduced here are:
-
- * CachingQuery - a Query subclass that caches and
- retrieves results in/from Beaker.
- * FromCache - a query option that establishes caching
- parameters on a Query
- * _params_from_query - extracts value parameters from
- a Query.
-
-The rest of what's here are standard SQLAlchemy and
-Beaker constructs.
-
-"""
-import beaker
-from beaker.exceptions import BeakerException
-from sqlalchemy.orm.interfaces import MapperOption
-from sqlalchemy.orm.query import Query
-from sqlalchemy.sql import visitors
-
-from kallithea.lib.utils2 import safe_str
-
-
-class CachingQuery(Query):
- """A Query subclass which optionally loads full results from a Beaker
- cache region.
-
- The CachingQuery stores additional state that allows it to consult
- a Beaker cache before accessing the database:
-
- * A "region", which is a cache region argument passed to a
- Beaker CacheManager, specifies a particular cache configuration
- (including backend implementation, expiration times, etc.)
- * A "namespace", which is a qualifying name that identifies a
- group of keys within the cache. A query that filters on a name
- might use the name "by_name", a query that filters on a date range
- to a joined table might use the name "related_date_range".
-
- When the above state is present, a Beaker cache is retrieved.
-
- The "namespace" name is first concatenated with
- a string composed of the individual entities and columns the Query
- requests, i.e. such as ``Query(User.id, User.name)``.
-
- The Beaker cache is then loaded from the cache manager based
- on the region and composed namespace. The key within the cache
- itself is then constructed against the bind parameters specified
- by this query, which are usually literals defined in the
- WHERE clause.
-
- The FromCache mapper option below represent
- the "public" method of configuring this state upon the CachingQuery.
-
- """
-
- def __init__(self, manager, *args, **kw):
- self.cache_manager = manager
- Query.__init__(self, *args, **kw)
-
- def __iter__(self):
- """override __iter__ to pull results from Beaker
- if particular attributes have been configured.
-
- Note that this approach does *not* detach the loaded objects from
- the current session. If the cache backend is an in-process cache
- (like "memory") and lives beyond the scope of the current session's
- transaction, those objects may be expired. The method here can be
- modified to first expunge() each loaded item from the current
- session before returning the list of items, so that the items
- in the cache are not the same ones in the current Session.
-
- """
- if hasattr(self, '_cache_parameters'):
- return self.get_value(createfunc=lambda:
- list(Query.__iter__(self)))
- else:
- return Query.__iter__(self)
-
- def invalidate(self):
- """Invalidate the value represented by this Query."""
-
- cache, cache_key = _get_cache_parameters(self)
- cache.remove(cache_key)
-
- def get_value(self, merge=True, createfunc=None):
- """Return the value from the cache for this query.
-
- Raise KeyError if no value present and no
- createfunc specified.
-
- """
- cache, cache_key = _get_cache_parameters(self)
- ret = cache.get_value(cache_key, createfunc=createfunc)
- if merge:
- ret = self.merge_result(ret, load=False)
- return ret
-
- def set_value(self, value):
- """Set the value in the cache for this query."""
-
- cache, cache_key = _get_cache_parameters(self)
- cache.put(cache_key, value)
-
-
-def query_callable(manager, query_cls=CachingQuery):
- def query(*arg, **kw):
- return query_cls(manager, *arg, **kw)
- return query
-
-
-def get_cache_region(name, region):
- if region not in beaker.cache.cache_regions:
- raise BeakerException('Cache region `%s` not configured '
- 'Check if proper cache settings are in the .ini files' % region)
- kw = beaker.cache.cache_regions[region]
- return beaker.cache.Cache._get_cache(name, kw)
-
-
-def _get_cache_parameters(query):
- """For a query with cache_region and cache_namespace configured,
- return the corresponding Cache instance and cache key, based
- on this query's current criterion and parameter values.
-
- """
- if not hasattr(query, '_cache_parameters'):
- raise ValueError("This Query does not have caching "
- "parameters configured.")
-
- region, namespace, cache_key = query._cache_parameters
-
- namespace = _namespace_from_query(namespace, query)
-
- if cache_key is None:
- # cache key - the value arguments from this query's parameters.
- args = [safe_str(x) for x in _params_from_query(query)]
- args.extend(filter(lambda k: k not in ['None', None, u'None'],
- [str(query._limit), str(query._offset)]))
-
- cache_key = " ".join(args)
-
- if cache_key is None:
- raise Exception('Cache key cannot be None')
-
- # get cache
- #cache = query.cache_manager.get_cache_region(namespace, region)
- cache = get_cache_region(namespace, region)
- # optional - hash the cache_key too for consistent length
- # import uuid
- # cache_key= str(uuid.uuid5(uuid.NAMESPACE_DNS, cache_key))
-
- return cache, cache_key
-
-
-def _namespace_from_query(namespace, query):
- # cache namespace - the token handed in by the
- # option + class we're querying against
- namespace = " ".join([namespace] + [str(x) for x in query._entities])
-
- # memcached wants this
- namespace = namespace.replace(' ', '_')
-
- return namespace
-
-
-def _set_cache_parameters(query, region, namespace, cache_key):
-
- if hasattr(query, '_cache_parameters'):
- region, namespace, cache_key = query._cache_parameters
- raise ValueError("This query is already configured "
- "for region %r namespace %r" %
- (region, namespace)
- )
- query._cache_parameters = region, safe_str(namespace), cache_key
-
-
-class FromCache(MapperOption):
- """Specifies that a Query should load results from a cache."""
-
- propagate_to_loaders = False
-
- def __init__(self, region, namespace, cache_key=None):
- """Construct a new FromCache.
-
- :param region: the cache region. Should be a
- region configured in the Beaker CacheManager.
-
- :param namespace: the cache namespace. Should
- be a name uniquely describing the target Query's
- lexical structure.
-
- :param cache_key: optional. A string cache key
- that will serve as the key to the query. Use this
- if your query has a huge amount of parameters (such
- as when using in_()) which correspond more simply to
- some other identifier.
-
- """
- self.region = region
- self.namespace = namespace
- self.cache_key = cache_key
-
- def process_query(self, query):
- """Process a Query during normal loading operation."""
-
- _set_cache_parameters(query, self.region, self.namespace,
- self.cache_key)
-
-
-def _params_from_query(query):
- """Pull the bind parameter values from a query.
-
- This takes into account any scalar attribute bindparam set up.
-
- E.g. params_from_query(query.filter(Cls.foo==5).filter(Cls.bar==7)))
- would return [5, 7].
-
- """
- v = []
-
- def visit_bindparam(bind):
- if bind.key in query._params:
- value = query._params[bind.key]
- elif bind.callable:
- # lazyloader may dig a callable in here, intended
- # to late-evaluate params after autoflush is called.
- # convert to a scalar value.
- value = bind.callable()
- else:
- value = bind.value
-
- v.append(value)
- if query._criterion is not None:
- visitors.traverse(query._criterion, {}, {'bindparam': visit_bindparam})
- for f in query._from_obj:
- visitors.traverse(f, {}, {'bindparam': visit_bindparam})
- return v
diff --git a/kallithea/lib/celerylib/__init__.py b/kallithea/lib/celerylib/__init__.py
--- a/kallithea/lib/celerylib/__init__.py
+++ b/kallithea/lib/celerylib/__init__.py
@@ -33,9 +33,9 @@ from hashlib import md5
from decorator import decorator
from tg import config
-from kallithea import CELERY_EAGER, CELERY_ON
+import kallithea
from kallithea.lib.pidlock import DaemonLock, LockHeld
-from kallithea.lib.utils2 import safe_str
+from kallithea.lib.utils2 import safe_bytes
from kallithea.model import meta
@@ -57,10 +57,10 @@ class FakeTask(object):
def task(f_org):
- """Wrapper of celery.task.task, running async if CELERY_ON
+ """Wrapper of celery.task.task, running async if CELERY_APP
"""
- if CELERY_ON:
+ if kallithea.CELERY_APP:
def f_async(*args, **kwargs):
log.info('executing %s task', f_org.__name__)
try:
@@ -68,8 +68,7 @@ def task(f_org):
finally:
log.info('executed %s task', f_org.__name__)
f_async.__name__ = f_org.__name__
- from kallithea.lib import celerypylons
- runner = celerypylons.task(ignore_result=True)(f_async)
+ runner = kallithea.CELERY_APP.task(ignore_result=True)(f_async)
def f_wrapped(*args, **kwargs):
t = runner.apply_async(args=args, kwargs=kwargs)
@@ -95,7 +94,7 @@ def __get_lockkey(func, *fargs, **fkwarg
func_name = str(func.__name__) if hasattr(func, '__name__') else str(func)
lockkey = 'task_%s.lock' % \
- md5(func_name + '-' + '-'.join(map(safe_str, params))).hexdigest()
+ md5(safe_bytes(func_name + '-' + '-'.join(str(x) for x in params))).hexdigest()
return lockkey
@@ -128,7 +127,7 @@ def dbsession(func):
ret = func(*fargs, **fkwargs)
return ret
finally:
- if CELERY_ON and not CELERY_EAGER:
+ if kallithea.CELERY_APP and not kallithea.CELERY_EAGER:
meta.Session.remove()
return decorator(__wrapper, func)
diff --git a/kallithea/lib/celerylib/tasks.py b/kallithea/lib/celerylib/tasks.py
--- a/kallithea/lib/celerylib/tasks.py
+++ b/kallithea/lib/celerylib/tasks.py
@@ -26,24 +26,23 @@ Original author and date, and relevant c
:license: GPLv3, see LICENSE.md for more details.
"""
-import logging
+import email.utils
import os
-import rfc822
import traceback
from collections import OrderedDict
from operator import itemgetter
from time import mktime
+import celery.utils.log
from tg import config
-from kallithea import CELERY_ON
-from kallithea.lib import celerylib
-from kallithea.lib.compat import json
+import kallithea
+from kallithea.lib import celerylib, ext_json
from kallithea.lib.helpers import person
from kallithea.lib.hooks import log_create_repository
from kallithea.lib.rcmail.smtp_mailer import SmtpMailer
from kallithea.lib.utils import action_logger
-from kallithea.lib.utils2 import str2bool
+from kallithea.lib.utils2 import ascii_bytes, str2bool
from kallithea.lib.vcs.utils import author_email
from kallithea.model.db import RepoGroup, Repository, Statistics, User
@@ -51,7 +50,7 @@ from kallithea.model.db import RepoGroup
__all__ = ['whoosh_index', 'get_commits_stats', 'send_email']
-log = logging.getLogger(__name__)
+log = celery.utils.log.get_task_logger(__name__)
@celerylib.task
@@ -67,6 +66,11 @@ def whoosh_index(repo_location, full_ind
.run(full_index=full_index)
+# for js data compatibility cleans the key for person from '
+def akc(k):
+ return person(k).replace('"', '')
+
+
@celerylib.task
@celerylib.dbsession
def get_commits_stats(repo_name, ts_min_y, ts_max_y, recurse_limit=100):
@@ -80,9 +84,6 @@ def get_commits_stats(repo_name, ts_min_
try:
lock = celerylib.DaemonLock(os.path.join(lockkey_path, lockkey))
- # for js data compatibility cleans the key for person from '
- akc = lambda k: person(k).replace('"', "")
-
co_day_auth_aggr = {}
commits_by_day_aggregate = {}
repo = Repository.get_by_repo_name(repo_name)
@@ -118,22 +119,21 @@ def get_commits_stats(repo_name, ts_min_
return True
if cur_stats:
- commits_by_day_aggregate = OrderedDict(json.loads(
+ commits_by_day_aggregate = OrderedDict(ext_json.loads(
cur_stats.commit_activity_combined))
- co_day_auth_aggr = json.loads(cur_stats.commit_activity)
+ co_day_auth_aggr = ext_json.loads(cur_stats.commit_activity)
log.debug('starting parsing %s', parse_limit)
- lmktime = mktime
- last_rev = last_rev + 1 if last_rev >= 0 else 0
+ last_rev = last_rev + 1 if last_rev and last_rev >= 0 else 0
log.debug('Getting revisions from %s to %s',
last_rev, last_rev + parse_limit
)
for cs in repo[last_rev:last_rev + parse_limit]:
log.debug('parsing %s', cs)
last_cs = cs # remember last parsed changeset
- k = lmktime([cs.date.timetuple()[0], cs.date.timetuple()[1],
- cs.date.timetuple()[2], 0, 0, 0, 0, 0, 0])
+ tt = cs.date.timetuple()
+ k = mktime(tt[:3] + (0, 0, 0, 0, 0, 0))
if akc(cs.author) in co_day_auth_aggr:
try:
@@ -143,8 +143,7 @@ def get_commits_stats(repo_name, ts_min_
except ValueError:
time_pos = None
- if time_pos >= 0 and time_pos is not None:
-
+ if time_pos is not None and time_pos >= 0:
datadict = \
co_day_auth_aggr[akc(cs.author)]['data'][time_pos]
@@ -195,8 +194,8 @@ def get_commits_stats(repo_name, ts_min_
}
stats = cur_stats if cur_stats else Statistics()
- stats.commit_activity = json.dumps(co_day_auth_aggr)
- stats.commit_activity_combined = json.dumps(overview_data)
+ stats.commit_activity = ascii_bytes(ext_json.dumps(co_day_auth_aggr))
+ stats.commit_activity_combined = ascii_bytes(ext_json.dumps(overview_data))
log.debug('last revision %s', last_rev)
leftovers = len(repo.revisions[last_rev:])
@@ -204,7 +203,7 @@ def get_commits_stats(repo_name, ts_min_
if last_rev == 0 or leftovers < parse_limit:
log.debug('getting code trending stats')
- stats.languages = json.dumps(__get_codes_stats(repo_name))
+ stats.languages = ascii_bytes(ext_json.dumps(__get_codes_stats(repo_name)))
try:
stats.repository = dbrepo
@@ -221,7 +220,7 @@ def get_commits_stats(repo_name, ts_min_
lock.release()
# execute another task if celery is enabled
- if len(repo.revisions) > 1 and CELERY_ON and recurse_limit > 0:
+ if len(repo.revisions) > 1 and kallithea.CELERY_APP and recurse_limit > 0:
get_commits_stats(repo_name, ts_min_y, ts_max_y, recurse_limit - 1)
elif recurse_limit <= 0:
log.debug('Not recursing - limit has been reached')
@@ -234,7 +233,7 @@ def get_commits_stats(repo_name, ts_min_
@celerylib.task
@celerylib.dbsession
-def send_email(recipients, subject, body='', html_body='', headers=None, author=None):
+def send_email(recipients, subject, body='', html_body='', headers=None, from_name=None):
"""
Sends an email with defined parameters from the .ini files.
@@ -244,7 +243,8 @@ def send_email(recipients, subject, body
:param body: body of the mail
:param html_body: html version of body
:param headers: dictionary of prepopulated e-mail headers
- :param author: User object of the author of this mail, if known and relevant
+ :param from_name: full name to be used as sender of this mail - often a
+ .full_name_or_username value
"""
assert isinstance(recipients, list), recipients
if headers is None:
@@ -276,13 +276,13 @@ def send_email(recipients, subject, body
# SMTP sender
envelope_from = email_config.get('app_email_from', 'Kallithea')
# 'From' header
- if author is not None:
- # set From header based on author but with a generic e-mail address
+ if from_name is not None:
+ # set From header based on from_name but with a generic e-mail address
# In case app_email_from is in "Some Name " format, we first
# extract the e-mail address.
envelope_addr = author_email(envelope_from)
headers['From'] = '"%s" <%s>' % (
- rfc822.quote('%s (no-reply)' % author.full_name_or_username),
+ email.utils.quote('%s (no-reply)' % from_name),
envelope_addr)
user = email_config.get('smtp_username')
@@ -414,7 +414,7 @@ def create_repo_fork(form_data, cur_user
DBS = celerylib.get_session()
- base_path = Repository.base_path()
+ base_path = kallithea.CONFIG['base_path']
cur_user = User.guess_instance(cur_user)
repo_name = form_data['repo_name'] # fork in this case
@@ -489,7 +489,7 @@ def __get_codes_stats(repo_name):
for _topnode, _dirnodes, filenodes in tip.walk('/'):
for filenode in filenodes:
ext = filenode.extension.lower()
- if ext in LANGUAGES_EXTENSIONS_MAP.keys() and not filenode.is_binary:
+ if ext in LANGUAGES_EXTENSIONS_MAP and not filenode.is_binary:
if ext in code_stats:
code_stats[ext] += 1
else:
diff --git a/kallithea/lib/celerypylons/__init__.py b/kallithea/lib/celerypylons/__init__.py
--- a/kallithea/lib/celerypylons/__init__.py
+++ b/kallithea/lib/celerypylons/__init__.py
@@ -14,34 +14,64 @@ To make sure that the config really has
mandatory settings.
"""
+import logging
+
import celery
-import celery.result as result
import tg
-from celery.bin import worker
-from celery.task import task
+
+import kallithea
-def celery_config(config):
- """Return Celery config object populated from relevant settings in a config dict, such as tg.config"""
+class CeleryConfig(object):
+ imports = ['kallithea.lib.celerylib.tasks']
+ task_always_eager = False
- # Verify .ini file configuration has been loaded
- assert config['celery.imports'] == 'kallithea.lib.celerylib.tasks', 'Kallithea Celery configuration has not been loaded'
+# map from Kallithea .ini Celery 3 config names to Celery 4 config names
+celery3_compat = {
+ 'broker.url': 'broker_url',
+ 'celery.accept.content': 'accept_content',
+ 'celery.always.eager': 'task_always_eager',
+ 'celery.amqp.task.result.expires': 'result_expires',
+ 'celeryd.concurrency': 'worker_concurrency',
+ 'celeryd.max.tasks.per.child': 'worker_max_tasks_per_child',
+ #'celery.imports' ends up unchanged
+ 'celery.result.backend': 'result_backend',
+ 'celery.result.serializer': 'result_serializer',
+ 'celery.task.serializer': 'task_serializer',
+}
- class CeleryConfig(object):
- pass
+list_config_names = """imports accept_content""".split()
+
+
+desupported = set([
+ 'celery.result.dburi',
+ 'celery.result.serialier',
+ 'celery.send.task.error.emails',
+])
+
+
+log = logging.getLogger(__name__)
+
+
+def make_celery_config(config):
+ """Return Celery config object populated from relevant settings in a config dict, such as tg.config"""
celery_config = CeleryConfig()
- PREFIXES = """ADMINS BROKER CASSANDRA CELERYBEAT CELERYD CELERYMON CELERY EMAIL SERVER""".split()
- LIST_PARAMS = """CELERY_IMPORTS ADMINS ROUTES CELERY_ACCEPT_CONTENT""".split()
-
for config_key, config_value in sorted(config.items()):
- celery_key = config_key.replace('.', '_').upper()
- if celery_key.split('_', 1)[0] not in PREFIXES:
+ if config_key in desupported and config_value:
+ log.error('Celery configuration setting %r is no longer supported', config_key)
+ celery_key = celery3_compat.get(config_key)
+ parts = config_key.split('.', 1)
+ if celery_key: # explicit Celery 3 backwards compatibility
+ pass
+ elif parts[0] == 'celery' and len(parts) == 2: # Celery 4 config key
+ celery_key = parts[1]
+ else:
continue
- if not isinstance(config_value, basestring):
+ if not isinstance(config_value, str):
continue
- if celery_key in LIST_PARAMS:
+ if celery_key in list_config_names:
celery_value = config_value.split()
elif config_value.isdigit():
celery_value = int(config_value)
@@ -53,6 +83,10 @@ def celery_config(config):
return celery_config
-# Create celery app from the TurboGears configuration file
-app = celery.Celery()
-app.config_from_object(celery_config(tg.config))
+def make_app():
+ """Create celery app from the TurboGears configuration file"""
+ app = celery.Celery()
+ celery_config = make_celery_config(tg.config)
+ kallithea.CELERY_EAGER = celery_config.task_always_eager
+ app.config_from_object(celery_config)
+ return app
diff --git a/kallithea/lib/colored_formatter.py b/kallithea/lib/colored_formatter.py
--- a/kallithea/lib/colored_formatter.py
+++ b/kallithea/lib/colored_formatter.py
@@ -15,7 +15,7 @@
import logging
-BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = xrange(30, 38)
+BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = range(30, 38)
# Sequences
RESET_SEQ = "\033[0m"
diff --git a/kallithea/lib/compat.py b/kallithea/lib/compat.py
--- a/kallithea/lib/compat.py
+++ b/kallithea/lib/compat.py
@@ -29,7 +29,6 @@ Original author and date, and relevant c
import functools
import os
-import sys
#==============================================================================
# Hybrid property/method
@@ -43,15 +42,10 @@ from sqlalchemy.util import OrderedSet
#==============================================================================
# json
#==============================================================================
-from kallithea.lib.ext_json import json
+from kallithea.lib import ext_json
-# alias for formatted json
-formatted_json = functools.partial(json.dumps, indent=4, sort_keys=True)
-
-
-
-
+formatted_json = functools.partial(ext_json.dumps, indent=4, sort_keys=True)
#==============================================================================
@@ -68,3 +62,8 @@ if os.name == 'nt': # Windows
else:
kill = os.kill
+
+
+# mute pyflakes "imported but unused"
+assert hybrid_property
+assert OrderedSet
diff --git a/kallithea/lib/db_manage.py b/kallithea/lib/db_manage.py
--- a/kallithea/lib/db_manage.py
+++ b/kallithea/lib/db_manage.py
@@ -26,8 +26,6 @@ Original author and date, and relevant c
:license: GPLv3, see LICENSE.md for more details.
"""
-from __future__ import print_function
-
import logging
import os
import sys
@@ -56,7 +54,6 @@ class DbManage(object):
self.tests = tests
self.root = root
self.dburi = dbconf
- self.db_exists = False
self.cli_args = cli_args or {}
self.init_db(SESSION=SESSION)
@@ -189,7 +186,7 @@ class DbManage(object):
return password
if username is None:
- username = raw_input('Specify admin username:')
+ username = input('Specify admin username:')
if password is None:
password = get_password()
if not password:
@@ -198,7 +195,7 @@ class DbManage(object):
if not password:
sys.exit()
if email is None:
- email = raw_input('Specify admin email:')
+ email = input('Specify admin email:')
self.create_user(username, password, email, True)
else:
log.info('creating admin and regular test users')
@@ -294,7 +291,7 @@ class DbManage(object):
if _path is not None:
path = _path
elif not self.tests and not test_repo_path:
- path = raw_input(
+ path = input(
'Enter a valid absolute path to store repositories. '
'All repositories in that path will be added automatically:'
)
@@ -385,18 +382,18 @@ class DbManage(object):
def create_user(self, username, password, email='', admin=False):
log.info('creating user %s', username)
UserModel().create_or_update(username, password, email,
- firstname=u'Kallithea', lastname=u'Admin',
+ firstname='Kallithea', lastname='Admin',
active=True, admin=admin,
extern_type=User.DEFAULT_AUTH_TYPE)
def create_default_user(self):
log.info('creating default user')
# create default user for handling default permissions.
- user = UserModel().create_or_update(username=User.DEFAULT_USER,
+ user = UserModel().create_or_update(username=User.DEFAULT_USER_NAME,
password=str(uuid.uuid1())[:20],
email='anonymous@kallithea-scm.org',
- firstname=u'Anonymous',
- lastname=u'User')
+ firstname='Anonymous',
+ lastname='User')
# based on configuration options activate/deactivate this user which
# controls anonymous access
if self.cli_args.get('public_access') is False:
@@ -419,4 +416,4 @@ class DbManage(object):
permissions that are missing, and not alter already defined ones
"""
log.info('creating default user permissions')
- PermissionModel().create_default_permissions(user=User.DEFAULT_USER)
+ PermissionModel().create_default_permissions(user=User.DEFAULT_USER_NAME)
diff --git a/kallithea/lib/diffs.py b/kallithea/lib/diffs.py
--- a/kallithea/lib/diffs.py
+++ b/kallithea/lib/diffs.py
@@ -32,7 +32,7 @@ import re
from tg.i18n import ugettext as _
from kallithea.lib import helpers as h
-from kallithea.lib.utils2 import safe_unicode
+from kallithea.lib.utils2 import safe_str
from kallithea.lib.vcs.backends.base import EmptyChangeset
from kallithea.lib.vcs.exceptions import VCSError
from kallithea.lib.vcs.nodes import FileNode, SubModuleNode
@@ -216,8 +216,7 @@ def wrapped_diff(filenode_old, filenode_
stats = (0, 0)
if not html_diff:
- submodules = filter(lambda o: isinstance(o, SubModuleNode),
- [filenode_new, filenode_old])
+ submodules = [o for o in [filenode_new, filenode_old] if isinstance(o, SubModuleNode)]
if submodules:
html_diff = wrap_to_table(h.escape('Submodule %r' % submodules[0]))
else:
@@ -235,10 +234,9 @@ def get_gitdiff(filenode_old, filenode_n
"""
# make sure we pass in default context
context = context or 3
- submodules = filter(lambda o: isinstance(o, SubModuleNode),
- [filenode_new, filenode_old])
+ submodules = [o for o in [filenode_new, filenode_old] if isinstance(o, SubModuleNode)]
if submodules:
- return ''
+ return b''
for filenode in (filenode_old, filenode_new):
if not isinstance(filenode, FileNode):
@@ -263,7 +261,7 @@ def get_diff(scm_instance, rev1, rev2, p
ignore_whitespace=ignore_whitespace, context=context)
except MemoryError:
h.flash('MemoryError: Diff is too big', category='error')
- return ''
+ return b''
NEW_FILENODE = 1
@@ -281,7 +279,7 @@ class DiffProcessor(object):
mentioned in the diff together with a dict of meta information that
can be used to render it in a HTML template.
"""
- _diff_git_re = re.compile('^diff --git', re.MULTILINE)
+ _diff_git_re = re.compile(b'^diff --git', re.MULTILINE)
def __init__(self, diff, vcs='hg', diff_limit=None, inline_diff=True):
"""
@@ -291,10 +289,10 @@ class DiffProcessor(object):
based on that parameter cut off will be triggered, set to None
to show full diff
"""
- if not isinstance(diff, basestring):
- raise Exception('Diff must be a basestring got %s instead' % type(diff))
+ if not isinstance(diff, bytes):
+ raise Exception('Diff must be bytes - got %s' % type(diff))
- self._diff = diff
+ self._diff = memoryview(diff)
self.adds = 0
self.removes = 0
self.diff_limit = diff_limit
@@ -317,7 +315,7 @@ class DiffProcessor(object):
self.limited_diff = True
continue
- head, diff_lines = _get_header(self.vcs, buffer(self._diff, start, end - start))
+ head, diff_lines = _get_header(self.vcs, self._diff[start:end])
op = None
stats = {
@@ -399,7 +397,7 @@ class DiffProcessor(object):
'new_lineno': '',
'action': 'context',
'line': msg,
- } for _op, msg in stats['ops'].iteritems()
+ } for _op, msg in stats['ops'].items()
if _op not in [MOD_FILENODE]])
_files.append({
@@ -420,22 +418,22 @@ class DiffProcessor(object):
for chunk in diff_data['chunks']:
lineiter = iter(chunk)
try:
- peekline = lineiter.next()
+ peekline = next(lineiter)
while True:
# find a first del line
while peekline['action'] != 'del':
- peekline = lineiter.next()
+ peekline = next(lineiter)
delline = peekline
- peekline = lineiter.next()
+ peekline = next(lineiter)
# if not followed by add, eat all following del lines
if peekline['action'] != 'add':
while peekline['action'] == 'del':
- peekline = lineiter.next()
+ peekline = next(lineiter)
continue
# found an add - make sure it is the only one
addline = peekline
try:
- peekline = lineiter.next()
+ peekline = next(lineiter)
except StopIteration:
# add was last line - ok
_highlight_inline_diff(delline, addline)
@@ -479,10 +477,10 @@ def _escaper(string):
return ' '
assert False
- return _escape_re.sub(substitute, safe_unicode(string))
+ return _escape_re.sub(substitute, safe_str(string))
-_git_header_re = re.compile(r"""
+_git_header_re = re.compile(br"""
^diff[ ]--git[ ]a/(?P.+?)[ ]b/(?P.+?)\n
(?:^old[ ]mode[ ](?P\d+)\n
^new[ ]mode[ ](?P\d+)(?:\n|$))?
@@ -499,7 +497,7 @@ _git_header_re = re.compile(r"""
""", re.VERBOSE | re.MULTILINE)
-_hg_header_re = re.compile(r"""
+_hg_header_re = re.compile(br"""
^diff[ ]--git[ ]a/(?P.+?)[ ]b/(?P.+?)\n
(?:^old[ ]mode[ ](?P\d+)\n
^new[ ]mode[ ](?P\d+)(?:\n|$))?
@@ -518,6 +516,9 @@ _hg_header_re = re.compile(r"""
""", re.VERBOSE | re.MULTILINE)
+_header_next_check = re.compile(br'''(?!@)(?!literal )(?!delta )''')
+
+
def _get_header(vcs, diff_chunk):
"""
Parses a Git diff for a single file (header and chunks) and returns a tuple with:
@@ -537,11 +538,11 @@ def _get_header(vcs, diff_chunk):
match = _hg_header_re.match(diff_chunk)
if match is None:
raise Exception('diff not recognized as valid %s diff' % vcs)
- meta_info = match.groupdict()
+ meta_info = {k: None if v is None else safe_str(v) for k, v in match.groupdict().items()}
rest = diff_chunk[match.end():]
- if rest and not rest.startswith('@') and not rest.startswith('literal ') and not rest.startswith('delta '):
- raise Exception('cannot parse %s diff header: %r followed by %r' % (vcs, diff_chunk[:match.end()], rest[:1000]))
- diff_lines = (_escaper(m.group(0)) for m in re.finditer(r'.*\n|.+$', rest)) # don't split on \r as str.splitlines do
+ if rest and _header_next_check.match(rest):
+ raise Exception('cannot parse %s diff header: %r followed by %r' % (vcs, safe_str(bytes(diff_chunk[:match.end()])), safe_str(bytes(rest[:1000]))))
+ diff_lines = (_escaper(m.group(0)) for m in re.finditer(br'.*\n|.+$', rest)) # don't split on \r as str.splitlines do
return meta_info, diff_lines
@@ -559,7 +560,7 @@ def _parse_lines(diff_lines):
chunks = []
try:
- line = diff_lines.next()
+ line = next(diff_lines)
while True:
lines = []
@@ -590,7 +591,7 @@ def _parse_lines(diff_lines):
'line': line,
})
- line = diff_lines.next()
+ line = next(diff_lines)
while old_line < old_end or new_line < new_end:
if not line:
@@ -623,7 +624,7 @@ def _parse_lines(diff_lines):
'line': line[1:],
})
- line = diff_lines.next()
+ line = next(diff_lines)
if _newline_marker.match(line):
# we need to append to lines, since this is not
@@ -634,7 +635,7 @@ def _parse_lines(diff_lines):
'action': 'context',
'line': line,
})
- line = diff_lines.next()
+ line = next(diff_lines)
if old_line > old_end:
raise Exception('error parsing diff - more than %s "-" lines at -%s+%s' % (old_end, old_line, new_line))
if new_line > new_end:
diff --git a/kallithea/lib/exceptions.py b/kallithea/lib/exceptions.py
--- a/kallithea/lib/exceptions.py
+++ b/kallithea/lib/exceptions.py
@@ -74,9 +74,8 @@ class UserCreationError(Exception):
pass
-class RepositoryCreationError(Exception):
+class HgsubversionImportError(Exception):
pass
-
-class HgsubversionImportError(Exception):
+class InvalidCloneUriException(Exception):
pass
diff --git a/kallithea/lib/ext_json.py b/kallithea/lib/ext_json.py
--- a/kallithea/lib/ext_json.py
+++ b/kallithea/lib/ext_json.py
@@ -1,16 +1,16 @@
"""
-Extended JSON encoder for json
+Extended JSON encoder with support for more data types
-json.org does not specify how date time can be represented - monkeypatch it to do something.
+json.org does not specify how date time can be represented - just encode it somehow and ignore decoding ...
"""
import datetime
import decimal
import functools
-import json # is re-exported after monkey patching
+import json
-__all__ = ['json']
+__all__ = ['dumps', 'dump', 'load', 'loads']
def _is_tz_aware(value):
@@ -70,10 +70,12 @@ class ExtendedEncoder(json.JSONEncoder):
try:
return _obj_dump(obj)
except NotImplementedError:
- pass
+ pass # quiet skipping of unsupported types!
raise TypeError("%r is not JSON serializable" % (obj,))
-# monkey-patch and export JSON encoder to use custom encoding method
-json.dumps = functools.partial(json.dumps, cls=ExtendedEncoder)
-json.dump = functools.partial(json.dump, cls=ExtendedEncoder)
+dumps = functools.partial(json.dumps, cls=ExtendedEncoder)
+dump = functools.partial(json.dump, cls=ExtendedEncoder)
+# No special support for loading these types back!!!
+load = json.load
+loads = json.loads
diff --git a/kallithea/lib/feeds.py b/kallithea/lib/feeds.py
new file mode 100644
--- /dev/null
+++ b/kallithea/lib/feeds.py
@@ -0,0 +1,152 @@
+# -*- coding: utf-8 -*-
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see
+ ${title}
+
+ ${link}
+ ${rfc3339_date(latest_pubdate)}
+ % for entry in entries:
+
+ ${entry.title}
+
+ ${rfc3339_date(entry.pubdate)}
+ ${rfc3339_date(entry.pubdate)}
+
+ ${entry.author_name}
+ ${entry.author_email}
+
+ ${get_tag_uri(entry.link, entry.pubdate)}
+ ${entry.description}
+
+ % endfor
+
+''', default_filters=['x'], output_encoding='utf-8', encoding_errors='replace')
+
+
+class RssFeed(_Feeder):
+
+ content_type = 'application/rss+xml'
+
+ template = mako.template.Template('''\
+
+
+
+ ${title}
+ ${link}
+ ${description}
+ ${language}
+ ${rfc2822_date(latest_pubdate)}
+ ${ttl}
+ % for entry in entries:
+ -
+
${entry.title}
+ ${entry.link}
+ ${entry.description}
+ ${entry.author_email} (${entry.author_name})
+ ${rfc2822_date(entry.pubdate)}
+
+ % endfor
+
+
+''', default_filters=['x'], output_encoding='utf-8', encoding_errors='replace')
diff --git a/kallithea/lib/helpers.py b/kallithea/lib/helpers.py
--- a/kallithea/lib/helpers.py
+++ b/kallithea/lib/helpers.py
@@ -22,9 +22,8 @@ import json
import logging
import random
import re
-import StringIO
import textwrap
-import urlparse
+import urllib.parse
from beaker.cache import cache_region
from pygments import highlight as code_highlight
@@ -49,7 +48,7 @@ from kallithea.lib.markup_renderer impor
from kallithea.lib.pygmentsutils import get_custom_lexer
from kallithea.lib.utils2 import MENTIONS_REGEX, AttributeDict
from kallithea.lib.utils2 import age as _age
-from kallithea.lib.utils2 import credentials_filter, safe_int, safe_str, safe_unicode, str2bool, time_to_datetime
+from kallithea.lib.utils2 import credentials_filter, safe_bytes, safe_int, safe_str, str2bool, time_to_datetime
from kallithea.lib.vcs.backends.base import BaseChangeset, EmptyChangeset
from kallithea.lib.vcs.exceptions import ChangesetDoesNotExistError
#==============================================================================
@@ -58,6 +57,25 @@ from kallithea.lib.vcs.exceptions import
from kallithea.lib.vcs.utils import author_email, author_name
+# mute pyflakes "imported but unused"
+assert Option
+assert checkbox
+assert end_form
+assert password
+assert radio
+assert submit
+assert text
+assert textarea
+assert format_byte_size
+assert chop_at
+assert wrap_paragraphs
+assert HasPermissionAny
+assert HasRepoGroupPermissionLevel
+assert HasRepoPermissionLevel
+assert time_to_datetime
+assert EmptyChangeset
+
+
log = logging.getLogger(__name__)
@@ -167,7 +185,7 @@ def select(name, selected_values, option
for x in option_list:
if isinstance(x, tuple) and len(x) == 2:
value, label = x
- elif isinstance(x, basestring):
+ elif isinstance(x, str):
value = label = x
else:
log.error('invalid select option %r', x)
@@ -177,7 +195,7 @@ def select(name, selected_values, option
for x in value:
if isinstance(x, tuple) and len(x) == 2:
group_value, group_label = x
- elif isinstance(x, basestring):
+ elif isinstance(x, str):
group_value = group_label = x
else:
log.error('invalid select option %r', x)
@@ -200,14 +218,12 @@ def FID(raw_id, path):
:param path:
"""
- return 'C-%s-%s' % (short_id(raw_id), hashlib.md5(safe_str(path)).hexdigest()[:12])
+ return 'C-%s-%s' % (short_id(raw_id), hashlib.md5(safe_bytes(path)).hexdigest()[:12])
class _FilesBreadCrumbs(object):
def __call__(self, repo_name, rev, paths):
- if isinstance(paths, str):
- paths = safe_unicode(paths)
url_l = [link_to(repo_name, url('files_home',
repo_name=repo_name,
revision=rev, f_path=''),
@@ -246,12 +262,12 @@ class CodeHtmlFormatter(HtmlFormatter):
yield i, t
def _wrap_tablelinenos(self, inner):
- dummyoutfile = StringIO.StringIO()
+ inner_lines = []
lncount = 0
for t, line in inner:
if t:
lncount += 1
- dummyoutfile.write(line)
+ inner_lines.append(line)
fl = self.linenostart
mw = len(str(lncount + fl - 1))
@@ -304,7 +320,7 @@ class CodeHtmlFormatter(HtmlFormatter):
'')
- yield 0, dummyoutfile.getvalue()
+ yield 0, ''.join(inner_lines)
yield 0, '
" - "Date: %s
" - "Message: %s") % (author, date, message) + def url_func(changeset): + author = escape(changeset.author) + date = changeset.date + message = escape(changeset.message) + tooltip_html = ("Author: %s
" + "Date: %s
" + "Message: %s") % (author, date, message) - lnk_format = show_id(changeset) - uri = link_to( - lnk_format, - url('changeset_home', repo_name=repo_name, - revision=changeset.raw_id), - style=get_color_string(changeset.raw_id), - **{'data-toggle': 'popover', - 'data-content': tooltip_html} - ) + lnk_format = show_id(changeset) + uri = link_to( + lnk_format, + url('changeset_home', repo_name=repo_name, + revision=changeset.raw_id), + style=get_color_string(changeset.raw_id), + **{'data-toggle': 'popover', + 'data-content': tooltip_html} + ) - uri += '\n' - return uri - return _url_func + uri += '\n' + return uri - return literal(markup_whitespace(annotate_highlight(filenode, url_func(repo_name), **kwargs))) + return literal(markup_whitespace(annotate_highlight(filenode, url_func, **kwargs))) class _Message(object): @@ -424,22 +437,14 @@ class _Message(object): Converting the message to a string returns the message text. Instances also have the following attributes: - * ``message``: the message text. * ``category``: the category specified when the message was created. + * ``message``: the html-safe message text. """ def __init__(self, category, message): self.category = category self.message = message - def __str__(self): - return self.message - - __unicode__ = __str__ - - def __html__(self): - return escape(safe_unicode(self.message)) - def _session_flash_messages(append=None, clear=False): """Manage a message queue in tg.session: return the current message queue @@ -461,7 +466,7 @@ def _session_flash_messages(append=None, return flash_messages -def flash(message, category=None, logf=None): +def flash(message, category, logf=None): """ Show a message to the user _and_ log it through the specified function @@ -471,14 +476,22 @@ def flash(message, category=None, logf=N logf defaults to log.info, unless category equals 'success', in which case logf defaults to log.debug. """ + assert category in ('error', 'success', 'warning'), category + if hasattr(message, '__html__'): + # render to HTML for storing in cookie + safe_message = str(message) + else: + # Apply str - the message might be an exception with __str__ + # Escape, so we can trust the result without further escaping, without any risk of injection + safe_message = html_escape(str(message)) if logf is None: logf = log.info if category == 'success': logf = log.debug - logf('Flash %s: %s', category, message) + logf('Flash %s: %s', category, safe_message) - _session_flash_messages(append=(category, message)) + _session_flash_messages(append=(category, safe_message)) def pop_flash_messages(): @@ -486,14 +499,22 @@ def pop_flash_messages(): The return value is a list of ``Message`` objects. """ - return [_Message(*m) for m in _session_flash_messages(clear=True)] + return [_Message(category, message) for category, message in _session_flash_messages(clear=True)] -age = lambda x, y=False: _age(x, y) -capitalize = lambda x: x.capitalize() +def age(x, y=False): + return _age(x, y) + +def capitalize(x): + return x.capitalize() + email = author_email -short_id = lambda x: x[:12] -hide_credentials = lambda x: ''.join(credentials_filter(x)) + +def short_id(x): + return x[:12] + +def hide_credentials(x): + return ''.join(credentials_filter(x)) def show_id(cs): @@ -516,8 +537,7 @@ def show_id(cs): def fmt_date(date): if date: - return date.strftime("%Y-%m-%d %H:%M:%S").decode('utf-8') - + return date.strftime("%Y-%m-%d %H:%M:%S") return "" @@ -548,7 +568,7 @@ def user_attr_or_none(author, show_attr) email = author_email(author) if email: from kallithea.model.db import User - user = User.get_by_email(email, cache=True) # cache will only use sql_cache_short + user = User.get_by_email(email) if user is not None: return getattr(user, show_attr) return None @@ -590,15 +610,12 @@ def person(author, show_attr="username") def person_by_id(id_, show_attr="username"): from kallithea.model.db import User - # attr to return from fetched user - person_getter = lambda usr: getattr(usr, show_attr) - # maybe it's an ID ? if str(id_).isdigit() or isinstance(id_, int): id_ = int(id_) user = User.get(id_) if user is not None: - return person_getter(user) + return getattr(user, show_attr) return id_ @@ -677,7 +694,7 @@ def action_parser(user_log, feed=False, return _op, _name revs = [] - if len(filter(lambda v: v != '', revs_ids)) > 0: + if len([v for v in revs_ids if v != '']) > 0: repo = None for rev in revs_ids[:revs_top_limit]: _op, _name = _get_op(rev) @@ -850,10 +867,7 @@ def action_parser(user_log, feed=False, .replace('[', '') \ .replace(']', '') - action_params_func = lambda: "" - - if callable(action_str[1]): - action_params_func = action_str[1] + action_params_func = action_str[1] if callable(action_str[1]) else (lambda: "") def action_parser_icon(): action = user_log.action @@ -937,13 +951,13 @@ def gravatar_url(email_address, size=30, if email_address == _def: return default - parsed_url = urlparse.urlparse(url.current(qualified=True)) + parsed_url = urllib.parse.urlparse(url.current(qualified=True)) url = (c.visual.gravatar_url or User.DEFAULT_GRAVATAR_URL) \ .replace('{email}', email_address) \ - .replace('{md5email}', hashlib.md5(safe_str(email_address).lower()).hexdigest()) \ + .replace('{md5email}', hashlib.md5(safe_bytes(email_address).lower()).hexdigest()) \ .replace('{netloc}', parsed_url.netloc) \ .replace('{scheme}', parsed_url.scheme) \ - .replace('{size}', safe_str(size)) + .replace('{size}', str(size)) return url @@ -959,7 +973,7 @@ def changed_tooltip(nodes): suf = '' if len(nodes) > 30: suf = '
' + _(' and %s more') % (len(nodes) - 30) - return literal(pref + '
'.join([safe_unicode(x.path) + return literal(pref + '
'.join([x.path for x in nodes[:30]]) + suf) else: return ': ' + _('No files') @@ -1069,6 +1083,8 @@ def urlify_text(s, repo_name=None, link_ URLs links to what they say. Issues are linked to given issue-server. If link_ is provided, all text not already linking somewhere will link there. + >>> urlify_text("Urlify http://example.com/ and 'https://example.com' *and* markup/b>") + literal('Urlify http://example.com/ and 'https://example.com' *and* <b>markup/b>') """ def _replace(match_obj): @@ -1162,10 +1178,11 @@ def urlify_issues(newtext, repo_name): assert CONFIG['sqlalchemy.url'] # make sure config has been loaded # Build chain of urlify functions, starting with not doing any transformation - tmp_urlify_issues_f = lambda s: s + def tmp_urlify_issues_f(s): + return s issue_pat_re = re.compile(r'issue_pat(.*)') - for k in CONFIG.keys(): + for k in CONFIG: # Find all issue_pat* settings that also have corresponding server_link and prefix configuration m = issue_pat_re.match(k) if m is None: @@ -1214,9 +1231,9 @@ def urlify_issues(newtext, repo_name): 'url': issue_url, 'text': issue_text, } - tmp_urlify_issues_f = (lambda s, - issue_re=issue_re, issues_replace=issues_replace, chain_f=tmp_urlify_issues_f: - issue_re.sub(issues_replace, chain_f(s))) + + def tmp_urlify_issues_f(s, issue_re=issue_re, issues_replace=issues_replace, chain_f=tmp_urlify_issues_f): + return issue_re.sub(issues_replace, chain_f(s)) # Set tmp function globally - atomically _urlify_issues_f = tmp_urlify_issues_f @@ -1229,7 +1246,7 @@ def render_w_mentions(source, repo_name= Render plain text with revision hashes and issue references urlified and with @mention highlighting. """ - s = safe_unicode(source) + s = safe_str(source) s = urlify_text(s, repo_name=repo_name) return literal('
[first-section]
@@ -87,6 +102,19 @@ def expand(template, mako_variable_value
[second-section]
# option a was chosen
+ [comment-section]
+ variable3 = 3.0
+ #variable4 = 4.0
+ variable4 = 4.1
+ #variable5 = 5.0
+ #variable5 = 5.1
+ variable5 = 5.2
+ #variable6 = 6.0
+ #variable6 = 6.1
+ variable6 = 6.2
+ variable7 = 7.0
+ #variable7 = 7.1
+
[fourth-section]
fourth = "four"
fourth_extra = 4
@@ -99,6 +127,12 @@ def expand(template, mako_variable_value
mako_variables.update(mako_variable_values or {})
settings = dict((k, dict(v)) for k, v in settings.items()) # deep copy before mutating
+ for key, value in mako_variables.items():
+ if key in variable_options:
+ if value not in variable_options[key]:
+ print('ERROR: %s is %r - it should be one of %s' %
+ (key, value, ', '.join(repr(x) for x in variable_options[key])))
+
ini_lines = mako.template.Template(template).render(**mako_variables)
def process_section(m):
@@ -106,20 +140,44 @@ def expand(template, mako_variable_value
sectionname, lines = m.groups()
if sectionname in settings:
section_settings = settings.pop(sectionname)
+ add_after_key_value = {} # map key to value it should be added after
- def process_line(m):
- """process a section line and update value if necessary"""
- key, value = m.groups()
+ # 1st pass:
+ # comment out lines with keys that have new values
+ # find best line for keeping or un-commenting (because it has the right value) or adding after (because it is the last with other value)
+ def comment_out(m):
+ """process a section line if in section_settings and comment out and track in add_after_key_value"""
line = m.group(0)
- if key in section_settings:
- new_line = '%s = %s' % (key, section_settings.pop(key))
- if new_line != line:
- # keep old entry as example - comments might refer to it
- line = '#%s\n%s' % (line, new_line)
- return line.rstrip()
+ comment, key, line_value = m.groups()
+ if key not in section_settings:
+ return line
+ new_value = section_settings[key]
+ if line_value == new_value or add_after_key_value.get(key) != new_value:
+ add_after_key_value[key] = line_value
+ if comment:
+ return line
+ return '#' + line
+
+ lines = re.sub(r'^(#)?([^#\n\s]*)[ \t]*=[ \t]*(.*)$', comment_out, lines, flags=re.MULTILINE)
- # process lines that not are comments or empty and look like name=value
- lines = re.sub(r'^([^#\n\s]*)[ \t]*=[ \t]*(.*)$', process_line, lines, flags=re.MULTILINE)
+ def add_after_comment(m):
+ """process a section comment line and add new value"""
+ line = m.group(0)
+ key, line_value = m.groups()
+ if key not in section_settings:
+ return line
+ if line_value != add_after_key_value.get(key):
+ return line
+ new_value = section_settings[key]
+ if new_value == line_value:
+ line = line.lstrip('#')
+ else:
+ line += '\n%s = %s' % (key, new_value)
+ section_settings.pop(key)
+ return line
+
+ lines = re.sub(r'^#([^#\n\s]*)[ \t]*=[ \t]*(.*)$', add_after_comment, lines, flags=re.MULTILINE)
+
# add unused section settings
if section_settings:
lines += '\n' + ''.join('%s = %s\n' % (key, value) for key, value in sorted(section_settings.items()))
diff --git a/kallithea/lib/locale.py b/kallithea/lib/locale.py
--- a/kallithea/lib/locale.py
+++ b/kallithea/lib/locale.py
@@ -24,7 +24,7 @@ def current_locale_is_valid():
Note: UTF-8 is preferred, but for example ISO-8859-1 or mbcs should also
work under the right circumstances."""
try:
- u'\xe9'.encode(sys.getfilesystemencoding()) # Test using é (é)
+ '\xe9'.encode(sys.getfilesystemencoding()) # Test using é (é)
except UnicodeEncodeError:
log.error("Cannot encode Unicode paths to file system encoding %r", sys.getfilesystemencoding())
for var in ['LC_ALL', 'LC_CTYPE', 'LANG']:
diff --git a/kallithea/lib/markup_renderer.py b/kallithea/lib/markup_renderer.py
--- a/kallithea/lib/markup_renderer.py
+++ b/kallithea/lib/markup_renderer.py
@@ -33,7 +33,7 @@ import traceback
import bleach
import markdown as markdown_mod
-from kallithea.lib.utils2 import MENTIONS_REGEX, safe_unicode
+from kallithea.lib.utils2 import MENTIONS_REGEX, safe_str
log = logging.getLogger(__name__)
@@ -119,17 +119,17 @@ class MarkupRenderer(object):
At last it will just do a simple html replacing new lines with
>>> MarkupRenderer.render('''
''', '.md')
- u'
''', 'b.mkd')
- u'
https://example.com/' + """ + source = safe_str(source) if universal_newline: newline = '\n' source = newline.join(source.splitlines()) @@ -168,30 +172,30 @@ class MarkupRenderer(object): with "safe" fall-back to plaintext. Output from this method should be sanitized before use. >>> MarkupRenderer.markdown('''''')
- u'''')
- u'
"""
- source = safe_unicode(source)
+ source = safe_str(source)
try:
if flavored:
source = cls._flavored_markdown(source)
@@ -209,7 +213,7 @@ class MarkupRenderer(object):
@classmethod
def rst(cls, source, safe=True):
- source = safe_unicode(source)
+ source = safe_str(source)
try:
from docutils.core import publish_parts
from docutils.parsers.rst import directives
@@ -219,7 +223,7 @@ class MarkupRenderer(object):
docutils_settings.update({'input_encoding': 'unicode',
'report_level': 4})
- for k, v in docutils_settings.iteritems():
+ for k, v in docutils_settings.items():
directives.register_directive(k, v)
parts = publish_parts(source=source,
diff --git a/kallithea/lib/middleware/permanent_repo_url.py b/kallithea/lib/middleware/permanent_repo_url.py
--- a/kallithea/lib/middleware/permanent_repo_url.py
+++ b/kallithea/lib/middleware/permanent_repo_url.py
@@ -20,7 +20,8 @@ middleware to handle permanent repo URLs
"""
-from kallithea.lib.utils import fix_repo_id_name, safe_str
+from kallithea.lib.utils import fix_repo_id_name
+from kallithea.lib.utils2 import safe_bytes, safe_str
class PermanentRepoUrl(object):
@@ -30,9 +31,11 @@ class PermanentRepoUrl(object):
self.config = config
def __call__(self, environ, start_response):
- path_info = environ['PATH_INFO']
+ # Extract path_info as get_path_info does, but do it explicitly because
+ # we also have to do the reverse operation when patching it back in
+ path_info = safe_str(environ['PATH_INFO'].encode('latin1'))
if path_info.startswith('/'): # it must
- path_info = '/' + safe_str(fix_repo_id_name(path_info[1:]))
- environ['PATH_INFO'] = path_info
+ path_info = '/' + fix_repo_id_name(path_info[1:])
+ environ['PATH_INFO'] = safe_bytes(path_info).decode('latin1')
return self.application(environ, start_response)
diff --git a/kallithea/lib/middleware/pygrack.py b/kallithea/lib/middleware/pygrack.py
--- a/kallithea/lib/middleware/pygrack.py
+++ b/kallithea/lib/middleware/pygrack.py
@@ -33,7 +33,7 @@ import traceback
from webob import Request, Response, exc
import kallithea
-from kallithea.lib.utils2 import safe_unicode
+from kallithea.lib.utils2 import ascii_bytes
from kallithea.lib.vcs import subprocessio
@@ -87,7 +87,6 @@ class GitRepository(object):
:param path:
"""
- path = safe_unicode(path)
assert path.startswith('/' + self.repo_name + '/')
return path[len(self.repo_name) + 2:].strip('/')
@@ -113,14 +112,14 @@ class GitRepository(object):
# ref_list
# "0000"
server_advert = '# service=%s\n' % git_command
- packet_len = str(hex(len(server_advert) + 4)[2:].rjust(4, '0')).lower()
+ packet_len = hex(len(server_advert) + 4)[2:].rjust(4, '0').lower()
_git_path = kallithea.CONFIG.get('git_path', 'git')
cmd = [_git_path, git_command[4:],
'--stateless-rpc', '--advertise-refs', self.content_path]
log.debug('handling cmd %s', cmd)
try:
out = subprocessio.SubprocessIOChunker(cmd,
- starting_values=[packet_len + server_advert + '0000']
+ starting_values=[ascii_bytes(packet_len + server_advert + '0000')]
)
except EnvironmentError as e:
log.error(traceback.format_exc())
@@ -166,7 +165,7 @@ class GitRepository(object):
log.error(traceback.format_exc())
raise exc.HTTPExpectationFailed()
- if git_command in [u'git-receive-pack']:
+ if git_command in ['git-receive-pack']:
# updating refs manually after each push.
# Needed for pre-1.7.0.4 git clients using regular HTTP mode.
from kallithea.lib.vcs import get_repo
@@ -186,7 +185,7 @@ class GitRepository(object):
_path = self._get_fixedpath(req.path_info)
if _path.startswith('info/refs'):
app = self.inforefs
- elif [a for a in self.valid_accepts if a in req.accept]:
+ elif req.accept.acceptable_offers(self.valid_accepts):
app = self.backend
try:
resp = app(req, environ)
diff --git a/kallithea/lib/middleware/sessionmiddleware.py b/kallithea/lib/middleware/sessionmiddleware.py
deleted file mode 100644
--- a/kallithea/lib/middleware/sessionmiddleware.py
+++ /dev/null
@@ -1,63 +0,0 @@
-# -*- coding: utf-8 -*-
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see , and with different handling of current_page
+ text = item['value']
+ if item['type'] == 'current_page': # we need active on the li and can thus not use curpage_attr
+ return ''' %s ''' % text
- # Note: the number of items on this page can be less than
- # items_per_page if the last page is not full
- self.first_item = max(0, (self.item_count) - (self.page *
- items_per_page))
- self.last_item = ((self.item_count - 1) - items_per_page *
- (self.page - 1))
-
- self.items = list(self.collection[self.first_item:self.last_item + 1])
-
- # Links to previous and next page
- if self.page > self.first_page:
- self.previous_page = self.page - 1
- else:
- self.previous_page = None
-
- if self.page < self.last_page:
- self.next_page = self.page + 1
- else:
- self.next_page = None
-
- # No items available
+ if not item['href'] or item['type'] == 'span':
+ if item['attrs']:
+ text = paginate.make_html_tag('span', **item['attrs']) + text + ''
else:
- self.first_page = None
- self.page_count = 0
- self.last_page = None
- self.first_item = None
- self.last_item = None
- self.previous_page = None
- self.next_page = None
- self.items = []
-
- # This is a subclass of the 'list' type. Initialise the list now.
- list.__init__(self, reversed(self.items))
+ target_url = item['href']
+ text = paginate.make_html_tag('a', text=text, href=target_url, **item['attrs'])
+ return '''%s ''' % text
diff --git a/kallithea/lib/paster_commands/template.ini.mako b/kallithea/lib/paster_commands/template.ini.mako
--- a/kallithea/lib/paster_commands/template.ini.mako
+++ b/kallithea/lib/paster_commands/template.ini.mako
@@ -1,11 +1,11 @@
## -*- coding: utf-8 -*-
-<%text>################################################################################
-<%text>################################################################################
-# Kallithea - config file generated with kallithea-config #
-# #
-# The %(here)s variable will be replaced with the parent directory of this file#
-<%text>################################################################################
-<%text>################################################################################
+<%text>###################################################################################
+<%text>###################################################################################
+<%text>## Kallithea config file generated with kallithea-config ##
+<%text>## ##
+<%text>## The %(here)s variable will be replaced with the parent directory of this file ##
+<%text>###################################################################################
+<%text>###################################################################################
[DEFAULT]
@@ -111,64 +111,33 @@ timeout = 3600
%endif
%else:
<%text>## UWSGI ##
-<%text>## run with uwsgi --ini-paste-logged
[uwsgi]
-socket = /tmp/uwsgi.sock
-master = true
-http = ${host}:${port}
-
-<%text>## set as daemon and redirect all output to file
-#daemonize = ./uwsgi_kallithea.log
-
-<%text>## master process PID
-pidfile = ./uwsgi_kallithea.pid
+<%text>## Note: this section is parsed by the uWSGI .ini parser when run as:
+<%text>## uwsgi --venv /srv/kallithea/venv --ini-paste-logged my.ini
+<%text>## Note: in uWSGI 2.0.18 or older, pastescript needs to be installed to
+<%text>## get correct application logging. In later versions this is not necessary.
+<%text>## pip install pastescript
-<%text>## stats server with workers statistics, use uwsgitop
-<%text>## for monitoring, `uwsgitop 127.0.0.1:1717`
-stats = 127.0.0.1:1717
-memory-report = true
-
-<%text>## log 5XX errors
-log-5xx = true
-
-<%text>## Set the socket listen queue size.
-listen = 128
-
-<%text>## Gracefully Reload workers after the specified amount of managed requests
-<%text>## (avoid memory leaks).
-max-requests = 1000
+<%text>## HTTP Basics:
+http-socket = ${host}:${port}
+buffer-size = 65535 ; Mercurial will use huge GET headers for discovery
-<%text>## enable large buffers
-buffer-size = 65535
-
-<%text>## socket and http timeouts ##
-http-timeout = 3600
-socket-timeout = 3600
-
-<%text>## Log requests slower than the specified number of milliseconds.
-log-slow = 10
-
-<%text>## Exit if no app can be loaded.
-need-app = true
-
-<%text>## Set lazy mode (load apps in workers instead of master).
-lazy = true
+<%text>## Scaling:
+master = true ; Use separate master and worker processes
+auto-procname = true ; Name worker processes accordingly
+lazy = true ; App *must* be loaded in workers - db connections can't be shared
+workers = 4 ; On demand scaling up to this many worker processes
+cheaper = 1 ; Initial and on demand scaling down to this many worker processes
+max-requests = 1000 ; Graceful reload of worker processes to avoid leaks
-<%text>## scaling ##
-<%text>## set cheaper algorithm to use, if not set default will be used
-cheaper-algo = spare
-
-<%text>## minimum number of workers to keep at all times
-cheaper = 1
-
-<%text>## number of workers to spawn at startup
-cheaper-initial = 1
-
-<%text>## maximum number of workers that can be spawned
-workers = 4
-
-<%text>## how many workers should be spawned at a time
-cheaper-step = 1
+<%text>## Tweak defaults:
+strict = true ; Fail on unknown config directives
+enable-threads = true ; Enable Python threads (not threaded workers)
+vacuum = true ; Delete sockets during shutdown
+single-interpreter = true
+die-on-term = true ; Shutdown when receiving SIGTERM (default is respawn)
+need-app = true ; Exit early if no app can be loaded.
+reload-on-exception = true ; Don't assume that the application worker can process more requests after a severe error
%endif
<%text>## middleware for hosting the WSGI application under a URL prefix
@@ -220,7 +189,7 @@ commit_parse_limit = 25
<%text>## used, which is correct in many cases but for example not when using uwsgi.
<%text>## If you change this setting, you should reinstall the Git hooks via
<%text>## Admin > Settings > Remap and Rescan.
-# git_hook_interpreter = /srv/kallithea/venv/bin/python2
+#git_hook_interpreter = /srv/kallithea/venv/bin/python3
%if git_hook_interpreter:
git_hook_interpreter = ${git_hook_interpreter}
%endif
@@ -295,7 +264,7 @@ issue_sub =
<%text>## issue_pat, issue_server_link and issue_sub can have suffixes to specify
<%text>## multiple patterns, to other issues server, wiki or others
<%text>## below an example how to create a wiki pattern
-# wiki-some-id -> https://wiki.example.com/some-id
+<%text>## wiki-some-id -> https://wiki.example.com/some-id
#issue_pat_wiki = wiki-(\S+)
#issue_server_link_wiki = https://wiki.example.com/\1
@@ -313,12 +282,12 @@ allow_repo_location_change = True
allow_custom_hooks_settings = True
<%text>## extra extensions for indexing, space separated and without the leading '.'.
-# index.extensions =
+#index.extensions =
# gemfile
# lock
<%text>## extra filenames for indexing, space separated
-# index.filenames =
+#index.filenames =
# .dockerignore
# .editorconfig
# INSTALL
@@ -356,25 +325,23 @@ ssh_locale = ${ssh_locale}
<%text>### CELERY CONFIG ####
<%text>####################################
+<%text>## Note: Celery doesn't support Windows.
use_celery = false
-<%text>## Example: connect to the virtual host 'rabbitmqhost' on localhost as rabbitmq:
-broker.url = amqp://rabbitmq:qewqew@localhost:5672/rabbitmqhost
+<%text>## Celery config settings from https://docs.celeryproject.org/en/4.4.0/userguide/configuration.html prefixed with 'celery.'.
-celery.imports = kallithea.lib.celerylib.tasks
-celery.accept.content = pickle
-celery.result.backend = amqp
-celery.result.dburi = amqp://
-celery.result.serializer = json
+<%text>## Example: use the message queue on the local virtual host 'kallitheavhost' as the RabbitMQ user 'kallithea':
+celery.broker_url = amqp://kallithea:thepassword@localhost:5672/kallitheavhost
-#celery.send.task.error.emails = true
+celery.result.backend = db+sqlite:///celery-results.db
+
#celery.amqp.task.result.expires = 18000
-celeryd.concurrency = 2
-celeryd.max.tasks.per.child = 1
+celery.worker_concurrency = 2
+celery.worker_max_tasks_per_child = 1
<%text>## If true, tasks will never be sent to the queue, but executed locally instead.
-celery.always.eager = false
+celery.task_always_eager = false
<%text>####################################
<%text>### BEAKER CACHE ####
@@ -383,19 +350,15 @@ celery.always.eager = false
beaker.cache.data_dir = %(here)s/data/cache/data
beaker.cache.lock_dir = %(here)s/data/cache/lock
-beaker.cache.regions = short_term,long_term,sql_cache_short
-
-beaker.cache.short_term.type = memory
-beaker.cache.short_term.expire = 60
-beaker.cache.short_term.key_length = 256
+beaker.cache.regions = long_term,long_term_file
beaker.cache.long_term.type = memory
beaker.cache.long_term.expire = 36000
beaker.cache.long_term.key_length = 256
-beaker.cache.sql_cache_short.type = memory
-beaker.cache.sql_cache_short.expire = 10
-beaker.cache.sql_cache_short.key_length = 256
+beaker.cache.long_term_file.type = file
+beaker.cache.long_term_file.expire = 604800
+beaker.cache.long_term_file.key_length = 256
<%text>####################################
<%text>### BEAKER SESSION ####
@@ -429,12 +392,24 @@ session.secret = ${uuid()}
#session.sa.url = postgresql://postgres:qwe@localhost/kallithea
#session.table_name = db_session
-<%text>############################
-<%text>## ERROR HANDLING SYSTEMS ##
-<%text>############################
+<%text>####################################
+<%text>### ERROR HANDLING ####
+<%text>####################################
+
+<%text>## Show a nice error page for application HTTP errors and exceptions (default true)
+#errorpage.enabled = true
-# Propagate email settings to ErrorReporter of TurboGears2
-# You do not normally need to change these lines
+<%text>## Enable Backlash client-side interactive debugger (default false)
+<%text>## WARNING: *THIS MUST BE false IN PRODUCTION ENVIRONMENTS!!!*
+<%text>## This debug mode will allow all visitors to execute malicious code.
+#debug = false
+
+<%text>## Enable Backlash server-side error reporting (unless debug mode handles it client-side) (default true)
+#trace_errors.enable = true
+<%text>## Errors will be reported by mail if trace_errors.error_email is set.
+
+<%text>## Propagate email settings to ErrorReporter of TurboGears2
+<%text>## You do not normally need to change these lines
get trace_errors.smtp_server = smtp_server
get trace_errors.smtp_port = smtp_port
get trace_errors.from_address = error_email_from
@@ -443,63 +418,7 @@ get trace_errors.smtp_username = smtp_us
get trace_errors.smtp_password = smtp_password
get trace_errors.smtp_use_tls = smtp_use_tls
-%if error_aggregation_service == 'appenlight':
-<%text>####################
-<%text>### [appenlight] ###
-<%text>####################
-
-<%text>## AppEnlight is tailored to work with Kallithea, see
-<%text>## http://appenlight.com for details how to obtain an account
-<%text>## you must install python package `appenlight_client` to make it work
-
-<%text>## appenlight enabled
-appenlight = false
-
-appenlight.server_url = https://api.appenlight.com
-appenlight.api_key = YOUR_API_KEY
-
-<%text>## TWEAK AMOUNT OF INFO SENT HERE
-
-<%text>## enables 404 error logging (default False)
-appenlight.report_404 = false
-
-<%text>## time in seconds after request is considered being slow (default 1)
-appenlight.slow_request_time = 1
-
-<%text>## record slow requests in application
-<%text>## (needs to be enabled for slow datastore recording and time tracking)
-appenlight.slow_requests = true
-
-<%text>## enable hooking to application loggers
-#appenlight.logging = true
-
-<%text>## minimum log level for log capture
-#appenlight.logging.level = WARNING
-
-<%text>## send logs only from erroneous/slow requests
-<%text>## (saves API quota for intensive logging)
-appenlight.logging_on_error = false
-
-<%text>## list of additional keywords that should be grabbed from environ object
-<%text>## can be string with comma separated list of words in lowercase
-<%text>## (by default client will always send following info:
-<%text>## 'REMOTE_USER', 'REMOTE_ADDR', 'SERVER_NAME', 'CONTENT_TYPE' + all keys that
-<%text>## start with HTTP* this list be extended with additional keywords here
-appenlight.environ_keys_whitelist =
-
-<%text>## list of keywords that should be blanked from request object
-<%text>## can be string with comma separated list of words in lowercase
-<%text>## (by default client will always blank keys that contain following words
-<%text>## 'password', 'passwd', 'pwd', 'auth_tkt', 'secret', 'csrf'
-<%text>## this list be extended with additional keywords set here
-appenlight.request_keys_blacklist =
-
-<%text>## list of namespaces that should be ignores when gathering log entries
-<%text>## can be string with comma separated list of namespaces
-<%text>## (by default the client ignores own entries: appenlight_client.client)
-appenlight.log_namespace_blacklist =
-
-%elif error_aggregation_service == 'sentry':
+%if error_aggregation_service == 'sentry':
<%text>################
<%text>### [sentry] ###
<%text>################
@@ -519,12 +438,6 @@ sentry.include_paths =
sentry.exclude_paths =
%endif
-<%text>################################################################################
-<%text>## WARNING: *DEBUG MODE MUST BE OFF IN A PRODUCTION ENVIRONMENT* ##
-<%text>## Debug mode will enable the interactive debugging tool, allowing ANYONE to ##
-<%text>## execute malicious code after an exception is raised. ##
-<%text>################################################################################
-debug = false
<%text>##################################
<%text>### LOGVIEW CONFIG ###
@@ -539,19 +452,19 @@ logview.pylons.util = #eee
<%text>#########################################################
%if database_engine == 'sqlite':
-# SQLITE [default]
+<%text>## SQLITE [default]
sqlalchemy.url = sqlite:///%(here)s/kallithea.db?timeout=60
%elif database_engine == 'postgres':
-# POSTGRESQL
+<%text>## POSTGRESQL
sqlalchemy.url = postgresql://user:pass@localhost/kallithea
%elif database_engine == 'mysql':
-# MySQL
+<%text>## MySQL
sqlalchemy.url = mysql://user:pass@localhost/kallithea?charset=utf8
%endif
-# see sqlalchemy docs for others
+<%text>## see sqlalchemy docs for other backends
sqlalchemy.pool_recycle = 3600
@@ -582,8 +495,8 @@ keys = generic, color_formatter, color_f
[logger_root]
level = NOTSET
handlers = console
-# For coloring based on log level:
-# handlers = console_color
+<%text>## For coloring based on log level:
+#handlers = console_color
[logger_routes]
level = WARN
@@ -620,10 +533,10 @@ qualname = gearbox
level = WARN
handlers =
qualname = sqlalchemy.engine
-# For coloring based on log level and pretty printing of SQL:
-# level = INFO
-# handlers = console_color_sql
-# propagate = 0
+<%text>## For coloring based on log level and pretty printing of SQL:
+#level = INFO
+#handlers = console_color_sql
+#propagate = 0
[logger_whoosh_indexer]
level = WARN
@@ -650,13 +563,13 @@ args = (sys.stderr,)
formatter = generic
[handler_console_color]
-# ANSI color coding based on log level
+<%text>## ANSI color coding based on log level
class = StreamHandler
args = (sys.stderr,)
formatter = color_formatter
[handler_console_color_sql]
-# ANSI color coding and pretty printing of SQL statements
+<%text>## ANSI color coding and pretty printing of SQL statements
class = StreamHandler
args = (sys.stderr,)
formatter = color_formatter_sql
@@ -687,16 +600,16 @@ datefmt = %Y-%m-%d %H:%M:%S
<%text>## SSH LOGGING ##
<%text>#################
-# The default loggers use 'handler_console' that uses StreamHandler with
-# destination 'sys.stderr'. In the context of the SSH server process, these log
-# messages would be sent to the client, which is normally not what you want.
-# By default, when running ssh-serve, just use NullHandler and disable logging
-# completely. For other logging options, see:
-# https://docs.python.org/2/library/logging.handlers.html
+<%text>## The default loggers use 'handler_console' that uses StreamHandler with
+<%text>## destination 'sys.stderr'. In the context of the SSH server process, these log
+<%text>## messages would be sent to the client, which is normally not what you want.
+<%text>## By default, when running ssh-serve, just use NullHandler and disable logging
+<%text>## completely. For other logging options, see:
+<%text>## https://docs.python.org/2/library/logging.handlers.html
[ssh_serve:logger_root]
level = CRITICAL
handlers = null
-# Note: If logging is configured with other handlers, they might need similar
-# muting for ssh-serve too.
+<%text>## Note: If logging is configured with other handlers, they might need similar
+<%text>## muting for ssh-serve too.
diff --git a/kallithea/lib/pidlock.py b/kallithea/lib/pidlock.py
--- a/kallithea/lib/pidlock.py
+++ b/kallithea/lib/pidlock.py
@@ -12,8 +12,6 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see .+)\n$', content)
if match:
return match.groupdict()['branch']
@@ -20,7 +21,7 @@ class GitWorkdir(BaseWorkdir):
raise RepositoryError("Couldn't compute workdir's branch")
def get_changeset(self):
- wk_dir_id = self.repository._repo.refs.as_dict().get('HEAD')
+ wk_dir_id = ascii_str(self.repository._repo.refs.as_dict().get(b'HEAD'))
return self.repository.get_changeset(wk_dir_id)
def checkout_branch(self, branch=None):
diff --git a/kallithea/lib/vcs/backends/hg/__init__.py b/kallithea/lib/vcs/backends/hg/__init__.py
--- a/kallithea/lib/vcs/backends/hg/__init__.py
+++ b/kallithea/lib/vcs/backends/hg/__init__.py
@@ -9,6 +9,8 @@
:copyright: (c) 2010-2011 by Marcin Kuzminski, Lukasz Balcerzak.
"""
+from kallithea.lib.vcs.utils import hgcompat
+
from .changeset import MercurialChangeset
from .inmemory import MercurialInMemoryChangeset
from .repository import MercurialRepository
@@ -19,3 +21,5 @@ __all__ = [
'MercurialRepository', 'MercurialChangeset',
'MercurialInMemoryChangeset', 'MercurialWorkdir',
]
+
+hgcompat.monkey_do()
diff --git a/kallithea/lib/vcs/backends/hg/changeset.py b/kallithea/lib/vcs/backends/hg/changeset.py
--- a/kallithea/lib/vcs/backends/hg/changeset.py
+++ b/kallithea/lib/vcs/backends/hg/changeset.py
@@ -1,41 +1,44 @@
import os
import posixpath
+import mercurial.archival
+import mercurial.node
+import mercurial.obsutil
+
from kallithea.lib.vcs.backends.base import BaseChangeset
from kallithea.lib.vcs.conf import settings
from kallithea.lib.vcs.exceptions import ChangesetDoesNotExistError, ChangesetError, ImproperArchiveTypeError, NodeDoesNotExistError, VCSError
-from kallithea.lib.vcs.nodes import (
- AddedFileNodesGenerator, ChangedFileNodesGenerator, DirNode, FileNode, NodeKind, RemovedFileNodesGenerator, RootNode, SubModuleNode)
-from kallithea.lib.vcs.utils import date_fromtimestamp, safe_str, safe_unicode
-from kallithea.lib.vcs.utils.hgcompat import archival, hex, obsutil
+from kallithea.lib.vcs.nodes import (AddedFileNodesGenerator, ChangedFileNodesGenerator, DirNode, FileNode, NodeKind, RemovedFileNodesGenerator, RootNode,
+ SubModuleNode)
+from kallithea.lib.vcs.utils import ascii_bytes, ascii_str, date_fromtimestamp, safe_bytes, safe_str
from kallithea.lib.vcs.utils.lazy import LazyProperty
from kallithea.lib.vcs.utils.paths import get_dirs_for_path
class MercurialChangeset(BaseChangeset):
"""
- Represents state of the repository at the single revision.
+ Represents state of the repository at a revision.
"""
def __init__(self, repository, revision):
self.repository = repository
- assert isinstance(revision, basestring), repr(revision)
- self.raw_id = revision
- self._ctx = repository._repo[revision]
+ assert isinstance(revision, str), repr(revision)
+ self._ctx = repository._repo[ascii_bytes(revision)]
+ self.raw_id = ascii_str(self._ctx.hex())
self.revision = self._ctx._rev
self.nodes = {}
@LazyProperty
def tags(self):
- return map(safe_unicode, self._ctx.tags())
+ return [safe_str(tag) for tag in self._ctx.tags()]
@LazyProperty
def branch(self):
- return safe_unicode(self._ctx.branch())
+ return safe_str(self._ctx.branch())
@LazyProperty
def branches(self):
- return [safe_unicode(self._ctx.branch())]
+ return [safe_str(self._ctx.branch())]
@LazyProperty
def closesbranch(self):
@@ -47,17 +50,11 @@ class MercurialChangeset(BaseChangeset):
@LazyProperty
def bumped(self):
- try:
- return self._ctx.phasedivergent()
- except AttributeError: # renamed in Mercurial 4.6 (9fa874fb34e1)
- return self._ctx.bumped()
+ return self._ctx.phasedivergent()
@LazyProperty
def divergent(self):
- try:
- return self._ctx.contentdivergent()
- except AttributeError: # renamed in Mercurial 4.6 (8b2d7684407b)
- return self._ctx.divergent()
+ return self._ctx.contentdivergent()
@LazyProperty
def extinct(self):
@@ -65,10 +62,7 @@ class MercurialChangeset(BaseChangeset):
@LazyProperty
def unstable(self):
- try:
- return self._ctx.orphan()
- except AttributeError: # renamed in Mercurial 4.6 (03039ff3082b)
- return self._ctx.unstable()
+ return self._ctx.orphan()
@LazyProperty
def phase(self):
@@ -81,33 +75,30 @@ class MercurialChangeset(BaseChangeset):
@LazyProperty
def successors(self):
- successors = obsutil.successorssets(self._ctx._repo, self._ctx.node(), closest=True)
- if successors:
- # flatten the list here handles both divergent (len > 1)
- # and the usual case (len = 1)
- successors = [hex(n)[:12] for sub in successors for n in sub if n != self._ctx.node()]
-
- return successors
+ successors = mercurial.obsutil.successorssets(self._ctx._repo, self._ctx.node(), closest=True)
+ # flatten the list here handles both divergent (len > 1)
+ # and the usual case (len = 1)
+ return [safe_str(mercurial.node.hex(n)[:12]) for sub in successors for n in sub if n != self._ctx.node()]
@LazyProperty
def predecessors(self):
- return [hex(n)[:12] for n in obsutil.closestpredecessors(self._ctx._repo, self._ctx.node())]
+ return [safe_str(mercurial.node.hex(n)[:12]) for n in mercurial.obsutil.closestpredecessors(self._ctx._repo, self._ctx.node())]
@LazyProperty
def bookmarks(self):
- return map(safe_unicode, self._ctx.bookmarks())
+ return [safe_str(bookmark) for bookmark in self._ctx.bookmarks()]
@LazyProperty
def message(self):
- return safe_unicode(self._ctx.description())
+ return safe_str(self._ctx.description())
@LazyProperty
def committer(self):
- return safe_unicode(self.author)
+ return safe_str(self.author)
@LazyProperty
def author(self):
- return safe_unicode(self._ctx.user())
+ return safe_str(self._ctx.user())
@LazyProperty
def date(self):
@@ -127,7 +118,7 @@ class MercurialChangeset(BaseChangeset):
@LazyProperty
def _file_paths(self):
- return list(self._ctx)
+ return list(safe_str(f) for f in self._ctx)
@LazyProperty
def _dir_paths(self):
@@ -140,12 +131,6 @@ class MercurialChangeset(BaseChangeset):
return self._dir_paths + self._file_paths
@LazyProperty
- def id(self):
- if self.last:
- return u'tip'
- return self.short_id
-
- @LazyProperty
def short_id(self):
return self.raw_id[:12]
@@ -202,22 +187,11 @@ class MercurialChangeset(BaseChangeset):
return cs
def diff(self):
- # Only used for feed diffstat
- return ''.join(self._ctx.diff())
-
- def _fix_path(self, path):
- """
- Paths are stored without trailing slash so we need to get rid off it if
- needed. Also mercurial keeps filenodes as str so we need to decode
- from unicode to str
- """
- if path.endswith('/'):
- path = path.rstrip('/')
-
- return safe_str(path)
+ # Only used to feed diffstat
+ return b''.join(self._ctx.diff())
def _get_kind(self, path):
- path = self._fix_path(path)
+ path = path.rstrip('/')
if path in self._file_paths:
return NodeKind.FILE
elif path in self._dir_paths:
@@ -227,11 +201,11 @@ class MercurialChangeset(BaseChangeset):
% (path))
def _get_filectx(self, path):
- path = self._fix_path(path)
+ path = path.rstrip('/')
if self._get_kind(path) != NodeKind.FILE:
raise ChangesetError("File does not exist for revision %s at "
" '%s'" % (self.raw_id, path))
- return self._ctx.filectx(path)
+ return self._ctx.filectx(safe_bytes(path))
def _extract_submodules(self):
"""
@@ -245,10 +219,10 @@ class MercurialChangeset(BaseChangeset):
Returns stat mode of the file at the given ``path``.
"""
fctx = self._get_filectx(path)
- if 'x' in fctx.flags():
- return 0100755
+ if b'x' in fctx.flags():
+ return 0o100755
else:
- return 0100644
+ return 0o100644
def get_file_content(self, path):
"""
@@ -280,7 +254,7 @@ class MercurialChangeset(BaseChangeset):
cnt = 0
for cs in reversed([x for x in fctx.filelog()]):
cnt += 1
- hist.append(hex(fctx.filectx(cs).node()))
+ hist.append(mercurial.node.hex(fctx.filectx(cs).node()))
if limit is not None and cnt == limit:
break
@@ -292,13 +266,10 @@ class MercurialChangeset(BaseChangeset):
lineno, sha, changeset lazy loader and line
"""
annotations = self._get_filectx(path).annotate()
- try:
- annotation_lines = [(annotateline.fctx, annotateline.text) for annotateline in annotations]
- except AttributeError: # annotateline was introduced in Mercurial 4.6 (b33b91ca2ec2)
- annotation_lines = [(aline.fctx, l) for aline, l in annotations]
- for i, (fctx, l) in enumerate(annotation_lines):
- sha = fctx.hex()
- yield (i + 1, sha, lambda sha=sha, l=l: self.repository.get_changeset(sha), l)
+ annotation_lines = [(annotateline.fctx, annotateline.text) for annotateline in annotations]
+ for i, (fctx, line) in enumerate(annotation_lines):
+ sha = ascii_str(fctx.hex())
+ yield (i + 1, sha, lambda sha=sha: self.repository.get_changeset(sha), line)
def fill_archive(self, stream=None, kind='tgz', prefix=None,
subrepos=False):
@@ -316,11 +287,10 @@ class MercurialChangeset(BaseChangeset):
:raise ImproperArchiveTypeError: If given kind is wrong.
:raise VcsError: If given stream is None
"""
-
- allowed_kinds = settings.ARCHIVE_SPECS.keys()
+ allowed_kinds = settings.ARCHIVE_SPECS
if kind not in allowed_kinds:
raise ImproperArchiveTypeError('Archive kind not supported use one'
- 'of %s' % allowed_kinds)
+ 'of %s' % ' '.join(allowed_kinds))
if stream is None:
raise VCSError('You need to pass in a valid stream for filling'
@@ -333,8 +303,8 @@ class MercurialChangeset(BaseChangeset):
elif prefix.strip() == '':
raise VCSError("Prefix cannot be empty")
- archival.archive(self.repository._repo, stream, self.raw_id,
- kind, prefix=prefix, subrepos=subrepos)
+ mercurial.archival.archive(self.repository._repo, stream, ascii_bytes(self.raw_id),
+ safe_bytes(kind), prefix=safe_bytes(prefix), subrepos=subrepos)
def get_nodes(self, path):
"""
@@ -346,8 +316,7 @@ class MercurialChangeset(BaseChangeset):
if self._get_kind(path) != NodeKind.DIR:
raise ChangesetError("Directory does not exist for revision %s at "
" '%s'" % (self.revision, path))
- path = self._fix_path(path)
-
+ path = path.rstrip('/')
filenodes = [FileNode(f, changeset=self) for f in self._file_paths
if os.path.dirname(f) == path]
dirs = path == '' and '' or [d for d in self._dir_paths
@@ -356,18 +325,16 @@ class MercurialChangeset(BaseChangeset):
if os.path.dirname(d) == path]
als = self.repository.alias
- for k, vals in self._extract_submodules().iteritems():
+ for k, vals in self._extract_submodules().items():
#vals = url,rev,type
loc = vals[0]
cs = vals[1]
dirnodes.append(SubModuleNode(k, url=loc, changeset=cs,
alias=als))
nodes = dirnodes + filenodes
- # cache nodes
for node in nodes:
self.nodes[node.path] = node
nodes.sort()
-
return nodes
def get_node(self, path):
@@ -375,9 +342,7 @@ class MercurialChangeset(BaseChangeset):
Returns ``Node`` object from the given ``path``. If there is no node at
the given ``path``, ``ChangesetError`` would be raised.
"""
-
- path = self._fix_path(path)
-
+ path = path.rstrip('/')
if path not in self.nodes:
if path in self._file_paths:
node = FileNode(path, changeset=self)
@@ -406,21 +371,21 @@ class MercurialChangeset(BaseChangeset):
"""
Returns list of added ``FileNode`` objects.
"""
- return AddedFileNodesGenerator([n for n in self.status[1]], self)
+ return AddedFileNodesGenerator([safe_str(n) for n in self.status.added], self)
@property
def changed(self):
"""
Returns list of modified ``FileNode`` objects.
"""
- return ChangedFileNodesGenerator([n for n in self.status[0]], self)
+ return ChangedFileNodesGenerator([safe_str(n) for n in self.status.modified], self)
@property
def removed(self):
"""
Returns list of removed ``FileNode`` objects.
"""
- return RemovedFileNodesGenerator([n for n in self.status[2]], self)
+ return RemovedFileNodesGenerator([safe_str(n) for n in self.status.removed], self)
@LazyProperty
def extra(self):
diff --git a/kallithea/lib/vcs/backends/hg/inmemory.py b/kallithea/lib/vcs/backends/hg/inmemory.py
--- a/kallithea/lib/vcs/backends/hg/inmemory.py
+++ b/kallithea/lib/vcs/backends/hg/inmemory.py
@@ -1,14 +1,17 @@
import datetime
+import mercurial.context
+import mercurial.node
+
from kallithea.lib.vcs.backends.base import BaseInMemoryChangeset
from kallithea.lib.vcs.exceptions import RepositoryError
-from kallithea.lib.vcs.utils.hgcompat import hex, memctx, memfilectx, tolocal
+from kallithea.lib.vcs.utils import ascii_str, safe_bytes, safe_str
class MercurialInMemoryChangeset(BaseInMemoryChangeset):
def commit(self, message, author, parents=None, branch=None, date=None,
- **kwargs):
+ **kwargs):
"""
Performs in-memory commit (doesn't check workdir in any way) and
returns newly created ``Changeset``. Updates repository's
@@ -27,21 +30,22 @@ class MercurialInMemoryChangeset(BaseInM
"""
self.check_integrity(parents)
+ if not isinstance(message, str):
+ raise RepositoryError('message must be a str - got %r' % type(message))
+ if not isinstance(author, str):
+ raise RepositoryError('author must be a str - got %r' % type(author))
+
from .repository import MercurialRepository
- if not isinstance(message, unicode) or not isinstance(author, unicode):
- raise RepositoryError('Given message and author needs to be '
- 'an instance got %r & %r instead'
- % (type(message), type(author)))
-
if branch is None:
branch = MercurialRepository.DEFAULT_BRANCH_NAME
- kwargs['branch'] = branch
+ kwargs[b'branch'] = safe_bytes(branch)
- def filectxfn(_repo, memctx, path):
+ def filectxfn(_repo, memctx, bytes_path):
"""
- Marks given path as added/changed/removed in a given _repo. This is
- for internal mercurial commit function.
+ Callback from Mercurial, returning ctx to commit for the given
+ path.
"""
+ path = safe_str(bytes_path)
# check if this path is removed
if path in (node.path for node in self.removed):
@@ -50,9 +54,8 @@ class MercurialInMemoryChangeset(BaseInM
# check if this path is added
for node in self.added:
if node.path == path:
- return memfilectx(_repo, memctx, path=node.path,
- data=(node.content.encode('utf-8')
- if not node.is_binary else node.content),
+ return mercurial.context.memfilectx(_repo, memctx, path=bytes_path,
+ data=node.content,
islink=False,
isexec=node.is_executable,
copysource=False)
@@ -60,14 +63,13 @@ class MercurialInMemoryChangeset(BaseInM
# or changed
for node in self.changed:
if node.path == path:
- return memfilectx(_repo, memctx, path=node.path,
- data=(node.content.encode('utf-8')
- if not node.is_binary else node.content),
+ return mercurial.context.memfilectx(_repo, memctx, path=bytes_path,
+ data=node.content,
islink=False,
isexec=node.is_executable,
copysource=False)
- raise RepositoryError("Given path haven't been marked as added,"
+ raise RepositoryError("Given path haven't been marked as added, "
"changed or removed (%s)" % path)
parents = [None, None]
@@ -76,22 +78,21 @@ class MercurialInMemoryChangeset(BaseInM
parents[i] = parent._ctx.node()
if date and isinstance(date, datetime.datetime):
- date = date.strftime('%a, %d %b %Y %H:%M:%S')
+ date = safe_bytes(date.strftime('%a, %d %b %Y %H:%M:%S'))
- commit_ctx = memctx(repo=self.repository._repo,
+ commit_ctx = mercurial.context.memctx(
+ repo=self.repository._repo,
parents=parents,
- text='',
- files=self.get_paths(),
+ text=b'',
+ files=[safe_bytes(x) for x in self.get_paths()],
filectxfn=filectxfn,
- user=author,
+ user=safe_bytes(author),
date=date,
extra=kwargs)
- loc = lambda u: tolocal(u.encode('utf-8'))
-
# injecting given _repo params
- commit_ctx._text = loc(message)
- commit_ctx._user = loc(author)
+ commit_ctx._text = safe_bytes(message)
+ commit_ctx._user = safe_bytes(author)
commit_ctx._date = date
# TODO: Catch exceptions!
@@ -100,9 +101,8 @@ class MercurialInMemoryChangeset(BaseInM
self._commit_ctx = commit_ctx # For reference
# Update vcs repository object & recreate mercurial _repo
# new_ctx = self.repository._repo[node]
- # new_tip = self.repository.get_changeset(new_ctx.hex())
- new_id = hex(n)
- self.repository.revisions.append(new_id)
+ # new_tip = ascii_str(self.repository.get_changeset(new_ctx.hex()))
+ self.repository.revisions.append(ascii_str(mercurial.node.hex(n)))
self._repo = self.repository._get_repo(create=False)
self.repository.branches = self.repository._get_branches()
tip = self.repository.get_changeset()
diff --git a/kallithea/lib/vcs/backends/hg/repository.py b/kallithea/lib/vcs/backends/hg/repository.py
--- a/kallithea/lib/vcs/backends/hg/repository.py
+++ b/kallithea/lib/vcs/backends/hg/repository.py
@@ -13,16 +13,33 @@ import datetime
import logging
import os
import time
-import urllib
-import urllib2
+import urllib.error
+import urllib.parse
+import urllib.request
from collections import OrderedDict
+import mercurial.commands
+import mercurial.error
+import mercurial.exchange
+import mercurial.hg
+import mercurial.hgweb
+import mercurial.httppeer
+import mercurial.localrepo
+import mercurial.match
+import mercurial.mdiff
+import mercurial.node
+import mercurial.patch
+import mercurial.scmutil
+import mercurial.sshpeer
+import mercurial.tags
+import mercurial.ui
+import mercurial.url
+import mercurial.util
+
from kallithea.lib.vcs.backends.base import BaseRepository, CollectionGenerator
-from kallithea.lib.vcs.exceptions import (
- BranchDoesNotExistError, ChangesetDoesNotExistError, EmptyRepositoryError, RepositoryError, TagAlreadyExistError, TagDoesNotExistError, VCSError)
-from kallithea.lib.vcs.utils import author_email, author_name, date_fromtimestamp, makedate, safe_str, safe_unicode
-from kallithea.lib.vcs.utils.hgcompat import (
- Abort, RepoError, RepoLookupError, clone, diffopts, get_contact, hex, hg_url, httpbasicauthhandler, httpdigestauthhandler, httppeer, localrepo, match_exact, nullid, patch, peer, scmutil, sshpeer, tag, ui)
+from kallithea.lib.vcs.exceptions import (BranchDoesNotExistError, ChangesetDoesNotExistError, EmptyRepositoryError, RepositoryError, TagAlreadyExistError,
+ TagDoesNotExistError, VCSError)
+from kallithea.lib.vcs.utils import ascii_str, author_email, author_name, date_fromtimestamp, makedate, safe_bytes, safe_str
from kallithea.lib.vcs.utils.lazy import LazyProperty
from kallithea.lib.vcs.utils.paths import abspath
@@ -60,9 +77,8 @@ class MercurialRepository(BaseRepository
raise VCSError('Mercurial backend requires repository path to '
'be instance of got %s instead' %
type(repo_path))
-
self.path = abspath(repo_path)
- self.baseui = baseui or ui.ui()
+ self.baseui = baseui or mercurial.ui.ui()
# We've set path and ui, now we can set _repo itself
self._repo = self._get_repo(create, src_url, update_after_clone)
@@ -115,14 +131,13 @@ class MercurialRepository(BaseRepository
return {}
bt = OrderedDict()
- for bn, _heads, tip, isclosed in sorted(self._repo.branchmap().iterbranches()):
+ for bn, _heads, node, isclosed in sorted(self._repo.branchmap().iterbranches()):
if isclosed:
if closed:
- bt[safe_unicode(bn)] = hex(tip)
+ bt[safe_str(bn)] = ascii_str(mercurial.node.hex(node))
else:
if normal:
- bt[safe_unicode(bn)] = hex(tip)
-
+ bt[safe_str(bn)] = ascii_str(mercurial.node.hex(node))
return bt
@LazyProperty
@@ -136,11 +151,11 @@ class MercurialRepository(BaseRepository
if self._empty:
return {}
- sortkey = lambda ctx: ctx[0] # sort by name
- _tags = [(safe_unicode(n), hex(h),) for n, h in
- self._repo.tags().items()]
-
- return OrderedDict(sorted(_tags, key=sortkey, reverse=True))
+ return OrderedDict(sorted(
+ ((safe_str(n), ascii_str(mercurial.node.hex(h))) for n, h in self._repo.tags().items()),
+ reverse=True,
+ key=lambda x: x[0], # sort by name
+ ))
def tag(self, name, user, revision=None, message=None, date=None,
**kwargs):
@@ -165,12 +180,12 @@ class MercurialRepository(BaseRepository
changeset.short_id)
if date is None:
- date = datetime.datetime.now().strftime('%a, %d %b %Y %H:%M:%S')
+ date = safe_bytes(datetime.datetime.now().strftime('%a, %d %b %Y %H:%M:%S'))
try:
- tag(self._repo, name, changeset._ctx.node(), message, local, user, date)
- except Abort as e:
- raise RepositoryError(e.message)
+ mercurial.tags.tag(self._repo, safe_bytes(name), changeset._ctx.node(), safe_bytes(message), local, safe_bytes(user), date)
+ except mercurial.error.Abort as e:
+ raise RepositoryError(e.args[0])
# Reinitialize tags
self.tags = self._get_tags()
@@ -194,14 +209,14 @@ class MercurialRepository(BaseRepository
if message is None:
message = "Removed tag %s" % name
if date is None:
- date = datetime.datetime.now().strftime('%a, %d %b %Y %H:%M:%S')
+ date = safe_bytes(datetime.datetime.now().strftime('%a, %d %b %Y %H:%M:%S'))
local = False
try:
- tag(self._repo, name, nullid, message, local, user, date)
+ mercurial.tags.tag(self._repo, safe_bytes(name), mercurial.commands.nullid, safe_bytes(message), local, safe_bytes(user), date)
self.tags = self._get_tags()
- except Abort as e:
- raise RepositoryError(e.message)
+ except mercurial.error.Abort as e:
+ raise RepositoryError(e.args[0])
@LazyProperty
def bookmarks(self):
@@ -214,14 +229,14 @@ class MercurialRepository(BaseRepository
if self._empty:
return {}
- sortkey = lambda ctx: ctx[0] # sort by name
- _bookmarks = [(safe_unicode(n), hex(h),) for n, h in
- self._repo._bookmarks.items()]
- return OrderedDict(sorted(_bookmarks, key=sortkey, reverse=True))
+ return OrderedDict(sorted(
+ ((safe_str(n), ascii_str(h)) for n, h in self._repo._bookmarks.items()),
+ reverse=True,
+ key=lambda x: x[0], # sort by name
+ ))
def _get_all_revisions(self):
-
- return [self._repo[x].hex() for x in self._repo.filtered('visible').changelog.revs()]
+ return [ascii_str(self._repo[x].hex()) for x in self._repo.filtered(b'visible').changelog.revs()]
def get_diff(self, rev1, rev2, path='', ignore_whitespace=False,
context=3):
@@ -257,12 +272,12 @@ class MercurialRepository(BaseRepository
self.get_changeset(rev1)
self.get_changeset(rev2)
if path:
- file_filter = match_exact(path)
+ file_filter = mercurial.match.exact(path)
else:
file_filter = None
- return ''.join(patch.diff(self._repo, rev1, rev2, match=file_filter,
- opts=diffopts(git=True,
+ return b''.join(mercurial.patch.diff(self._repo, rev1, rev2, match=file_filter,
+ opts=mercurial.mdiff.diffopts(git=True,
showfunc=True,
ignorews=ignore_whitespace,
context=context)))
@@ -279,42 +294,46 @@ class MercurialRepository(BaseRepository
when the return code is non 200
"""
# check first if it's not an local url
- if os.path.isdir(url) or url.startswith('file:'):
+ url = safe_bytes(url)
+ if os.path.isdir(url) or url.startswith(b'file:'):
return True
- if url.startswith('ssh:'):
+ if url.startswith(b'ssh:'):
# in case of invalid uri or authentication issues, sshpeer will
# throw an exception.
- sshpeer.instance(repoui or ui.ui(), url, False).lookup('tip')
+ mercurial.sshpeer.instance(repoui or mercurial.ui.ui(), url, False).lookup(b'tip')
return True
url_prefix = None
- if '+' in url[:url.find('://')]:
- url_prefix, url = url.split('+', 1)
+ if b'+' in url[:url.find(b'://')]:
+ url_prefix, url = url.split(b'+', 1)
handlers = []
- url_obj = hg_url(url)
+ url_obj = mercurial.util.url(url)
test_uri, authinfo = url_obj.authinfo()
- url_obj.passwd = '*****'
+ url_obj.passwd = b'*****'
cleaned_uri = str(url_obj)
if authinfo:
# create a password manager
- passmgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
+ passmgr = urllib.request.HTTPPasswordMgrWithDefaultRealm()
passmgr.add_password(*authinfo)
- handlers.extend((httpbasicauthhandler(passmgr),
- httpdigestauthhandler(passmgr)))
+ handlers.extend((mercurial.url.httpbasicauthhandler(passmgr),
+ mercurial.url.httpdigestauthhandler(passmgr)))
- o = urllib2.build_opener(*handlers)
+ o = urllib.request.build_opener(*handlers)
o.addheaders = [('Content-Type', 'application/mercurial-0.1'),
('Accept', 'application/mercurial-0.1')]
- q = {"cmd": 'between'}
- q.update({'pairs': "%s-%s" % ('0' * 40, '0' * 40)})
- qs = '?%s' % urllib.urlencode(q)
- cu = "%s%s" % (test_uri, qs)
- req = urllib2.Request(cu, None, {})
+ req = urllib.request.Request(
+ "%s?%s" % (
+ test_uri,
+ urllib.parse.urlencode({
+ 'cmd': 'between',
+ 'pairs': "%s-%s" % ('0' * 40, '0' * 40),
+ })
+ ))
try:
resp = o.open(req)
@@ -322,14 +341,14 @@ class MercurialRepository(BaseRepository
raise Exception('Return Code is not 200')
except Exception as e:
# means it cannot be cloned
- raise urllib2.URLError("[%s] org_exc: %s" % (cleaned_uri, e))
+ raise urllib.error.URLError("[%s] org_exc: %s" % (cleaned_uri, e))
if not url_prefix: # skip svn+http://... (and git+... too)
# now check if it's a proper hg repo
try:
- httppeer.instance(repoui or ui.ui(), url, False).lookup('tip')
+ mercurial.httppeer.instance(repoui or mercurial.ui.ui(), url, False).lookup(b'tip')
except Exception as e:
- raise urllib2.URLError(
+ raise urllib.error.URLError(
"url [%s] does not look like an hg repo org_exc: %s"
% (cleaned_uri, e))
@@ -345,26 +364,25 @@ class MercurialRepository(BaseRepository
location at given clone_point. Additionally it'll make update to
working copy accordingly to ``update_after_clone`` flag
"""
-
try:
if src_url:
- url = safe_str(self._get_url(src_url))
+ url = safe_bytes(self._get_url(src_url))
opts = {}
if not update_after_clone:
opts.update({'noupdate': True})
MercurialRepository._check_url(url, self.baseui)
- clone(self.baseui, url, self.path, **opts)
+ mercurial.commands.clone(self.baseui, url, safe_bytes(self.path), **opts)
# Don't try to create if we've already cloned repo
create = False
- return localrepo.instance(self.baseui, self.path, create=create)
- except (Abort, RepoError) as err:
+ return mercurial.localrepo.instance(self.baseui, safe_bytes(self.path), create=create)
+ except (mercurial.error.Abort, mercurial.error.RepoError) as err:
if create:
msg = "Cannot create repository at %s. Original error was %s" \
- % (self.path, err)
+ % (self.name, err)
else:
msg = "Not valid repository at %s. Original error was %s" \
- % (self.path, err)
+ % (self.name, err)
raise RepositoryError(msg)
@LazyProperty
@@ -373,15 +391,13 @@ class MercurialRepository(BaseRepository
@LazyProperty
def description(self):
- undefined_description = u'unknown'
- _desc = self._repo.ui.config('web', 'description', None, untrusted=True)
- return safe_unicode(_desc or undefined_description)
+ _desc = self._repo.ui.config(b'web', b'description', None, untrusted=True)
+ return safe_str(_desc or b'unknown')
@LazyProperty
def contact(self):
- undefined_contact = u'Unknown'
- return safe_unicode(get_contact(self._repo.ui.config)
- or undefined_contact)
+ return safe_str(mercurial.hgweb.common.get_contact(self._repo.ui.config)
+ or b'Unknown')
@LazyProperty
def last_change(self):
@@ -404,39 +420,33 @@ class MercurialRepository(BaseRepository
def _get_revision(self, revision):
"""
- Gets an ID revision given as str. This will always return a full
- 40 char revision number
+ Given any revision identifier, returns a 40 char string with revision hash.
:param revision: str or int or None
"""
- if isinstance(revision, unicode):
- revision = safe_str(revision)
-
if self._empty:
raise EmptyRepositoryError("There are no changesets yet")
if revision in [-1, None]:
- revision = 'tip'
+ revision = b'tip'
+ elif isinstance(revision, str):
+ revision = safe_bytes(revision)
try:
if isinstance(revision, int):
- return self._repo[revision].hex()
- try:
- return scmutil.revsymbol(self._repo, revision).hex()
- except AttributeError: # revsymbol was introduced in Mercurial 4.6
- return self._repo[revision].hex()
- except (IndexError, ValueError, RepoLookupError, TypeError):
- msg = ("Revision %s does not exist for %s" % (revision, self))
+ return ascii_str(self._repo[revision].hex())
+ return ascii_str(mercurial.scmutil.revsymbol(self._repo, revision).hex())
+ except (IndexError, ValueError, mercurial.error.RepoLookupError, TypeError):
+ msg = "Revision %r does not exist for %s" % (safe_str(revision), self.name)
raise ChangesetDoesNotExistError(msg)
except (LookupError, ):
- msg = ("Ambiguous identifier `%s` for %s" % (revision, self))
+ msg = "Ambiguous identifier `%s` for %s" % (safe_str(revision), self.name)
raise ChangesetDoesNotExistError(msg)
def get_ref_revision(self, ref_type, ref_name):
"""
Returns revision number for the given reference.
"""
- ref_name = safe_str(ref_name)
if ref_type == 'rev' and not ref_name.strip('0'):
return self.EMPTY_CHANGESET
# lookup up the exact node id
@@ -451,17 +461,13 @@ class MercurialRepository(BaseRepository
try:
revs = self._repo.revs(rev_spec, ref_name, ref_name)
except LookupError:
- msg = ("Ambiguous identifier %s:%s for %s" % (ref_type, ref_name, self.name))
+ msg = "Ambiguous identifier %s:%s for %s" % (ref_type, ref_name, self.name)
raise ChangesetDoesNotExistError(msg)
- except RepoLookupError:
- msg = ("Revision %s:%s does not exist for %s" % (ref_type, ref_name, self.name))
+ except mercurial.error.RepoLookupError:
+ msg = "Revision %s:%s does not exist for %s" % (ref_type, ref_name, self.name)
raise ChangesetDoesNotExistError(msg)
if revs:
- try:
- revision = revs.last()
- except AttributeError:
- # removed in hg 3.2
- revision = revs[-1]
+ revision = revs.last()
else:
# TODO: just report 'not found'?
revision = ref_name
@@ -469,39 +475,29 @@ class MercurialRepository(BaseRepository
return self._get_revision(revision)
def _get_archives(self, archive_name='tip'):
- allowed = self.baseui.configlist("web", "allow_archive",
+ allowed = self.baseui.configlist(b"web", b"allow_archive",
untrusted=True)
- for i in [('zip', '.zip'), ('gz', '.tar.gz'), ('bz2', '.tar.bz2')]:
- if i[0] in allowed or self._repo.ui.configbool("web",
- "allow" + i[0],
+ for name, ext in [(b'zip', '.zip'), (b'gz', '.tar.gz'), (b'bz2', '.tar.bz2')]:
+ if name in allowed or self._repo.ui.configbool(b"web",
+ b"allow" + name,
untrusted=True):
- yield {"type": i[0], "extension": i[1], "node": archive_name}
+ yield {"type": safe_str(name), "extension": ext, "node": archive_name}
def _get_url(self, url):
"""
- Returns normalized url. If schema is not given, would fall
- to filesystem
- (``file:///``) schema.
+ Returns normalized url. If schema is not given, fall back to
+ filesystem (``file:///``) schema.
"""
- url = safe_str(url)
if url != 'default' and '://' not in url:
- url = "file:" + urllib.pathname2url(url)
+ url = "file:" + urllib.request.pathname2url(url)
return url
- def get_hook_location(self):
- """
- returns absolute path to location where hooks are stored
- """
- return os.path.join(self.path, '.hg', '.hgrc')
-
def get_changeset(self, revision=None):
"""
Returns ``MercurialChangeset`` object representing repository's
changeset at the given ``revision``.
"""
- revision = self._get_revision(revision)
- changeset = MercurialChangeset(repository=self, revision=revision)
- return changeset
+ return MercurialChangeset(repository=self, revision=self._get_revision(revision))
def get_changesets(self, start=None, end=None, start_date=None,
end_date=None, branch_name=None, reverse=False, max_revisions=None):
@@ -517,35 +513,35 @@ class MercurialRepository(BaseRepository
:param reversed: return changesets in reversed order
"""
start_raw_id = self._get_revision(start)
- start_pos = self.revisions.index(start_raw_id) if start else None
+ start_pos = None if start is None else self.revisions.index(start_raw_id)
end_raw_id = self._get_revision(end)
- end_pos = self.revisions.index(end_raw_id) if end else None
+ end_pos = None if end is None else self.revisions.index(end_raw_id)
- if None not in [start, end] and start_pos > end_pos:
+ if start_pos is not None and end_pos is not None and start_pos > end_pos:
raise RepositoryError("Start revision '%s' cannot be "
"after end revision '%s'" % (start, end))
- if branch_name and branch_name not in self.allbranches.keys():
- msg = ("Branch %s not found in %s" % (branch_name, self))
+ if branch_name and branch_name not in self.allbranches:
+ msg = "Branch %r not found in %s" % (branch_name, self.name)
raise BranchDoesNotExistError(msg)
if end_pos is not None:
end_pos += 1
# filter branches
filter_ = []
if branch_name:
- filter_.append('branch("%s")' % safe_str(branch_name))
+ filter_.append(b'branch("%s")' % safe_bytes(branch_name))
if start_date:
- filter_.append('date(">%s")' % start_date)
+ filter_.append(b'date(">%s")' % safe_bytes(str(start_date)))
if end_date:
- filter_.append('date("<%s")' % end_date)
+ filter_.append(b'date("<%s")' % safe_bytes(str(end_date)))
if filter_ or max_revisions:
if filter_:
- revspec = ' and '.join(filter_)
+ revspec = b' and '.join(filter_)
else:
- revspec = 'all()'
+ revspec = b'all()'
if max_revisions:
- revspec = 'limit(%s, %s)' % (revspec, max_revisions)
- revisions = scmutil.revrange(self._repo, [revspec])
+ revspec = b'limit(%s, %d)' % (revspec, max_revisions)
+ revisions = mercurial.scmutil.revrange(self._repo, [revspec])
else:
revisions = self.revisions
@@ -553,7 +549,7 @@ class MercurialRepository(BaseRepository
# would be to get rid of this function entirely and use revsets
revs = list(revisions)[start_pos:end_pos]
if reverse:
- revs = reversed(revs)
+ revs.reverse()
return CollectionGenerator(self, revs)
@@ -561,15 +557,10 @@ class MercurialRepository(BaseRepository
"""
Tries to pull changes from external location.
"""
- url = self._get_url(url)
- other = peer(self._repo, {}, url)
+ other = mercurial.hg.peer(self._repo, {}, safe_bytes(self._get_url(url)))
try:
- # hg 3.2 moved push / pull to exchange module
- from mercurial import exchange
- exchange.pull(self._repo, other, heads=None, force=None)
- except ImportError:
- self._repo.pull(other, heads=None, force=None)
- except Abort as err:
+ mercurial.exchange.pull(self._repo, other, heads=None, force=None)
+ except mercurial.error.Abort as err:
# Propagate error but with vcs's type
raise RepositoryError(str(err))
@@ -591,15 +582,16 @@ class MercurialRepository(BaseRepository
"""
if config_file is None:
config_file = []
- elif isinstance(config_file, basestring):
+ elif isinstance(config_file, str):
config_file = [config_file]
config = self._repo.ui
if config_file:
- config = ui.ui()
+ config = mercurial.ui.ui()
for path in config_file:
- config.readconfig(path)
- return config.config(section, name)
+ config.readconfig(safe_bytes(path))
+ value = config.config(safe_bytes(section), safe_bytes(name))
+ return value if value is None else safe_str(value)
def get_user_name(self, config_file=None):
"""
diff --git a/kallithea/lib/vcs/backends/hg/ssh.py b/kallithea/lib/vcs/backends/hg/ssh.py
--- a/kallithea/lib/vcs/backends/hg/ssh.py
+++ b/kallithea/lib/vcs/backends/hg/ssh.py
@@ -14,18 +14,12 @@
import logging
-from mercurial import hg
+import mercurial.hg
+import mercurial.wireprotoserver
from kallithea.lib.utils import make_ui
-from kallithea.lib.utils2 import safe_str, safe_unicode
from kallithea.lib.vcs.backends.ssh import BaseSshHandler
-
-
-try:
- from mercurial.wireprotoserver import sshserver
-except ImportError:
- from mercurial.sshserver import sshserver # moved in Mercurial 4.6 (1bf5263fe5cc)
-
+from kallithea.lib.vcs.utils import safe_bytes
log = logging.getLogger(__name__)
@@ -40,11 +34,11 @@ class MercurialSshHandler(BaseSshHandler
>>> import shlex
>>> MercurialSshHandler.make(shlex.split('hg -R "foo bar" serve --stdio')).repo_name
- u'foo bar'
+ 'foo bar'
>>> MercurialSshHandler.make(shlex.split(' hg -R blåbærgrød serve --stdio ')).repo_name
- u'bl\xe5b\xe6rgr\xf8d'
+ 'bl\xe5b\xe6rgr\xf8d'
>>> MercurialSshHandler.make(shlex.split('''hg -R 'foo"bar' serve --stdio''')).repo_name
- u'foo"bar'
+ 'foo"bar'
>>> MercurialSshHandler.make(shlex.split('/bin/hg -R "foo" serve --stdio'))
>>> MercurialSshHandler.make(shlex.split('''hg -R "foo"bar" serve --stdio''')) # ssh-serve will report: Error parsing SSH command "...": invalid syntax
@@ -53,7 +47,7 @@ class MercurialSshHandler(BaseSshHandler
>>> MercurialSshHandler.make(shlex.split('git-upload-pack "/foo"')) # not handled here
"""
if ssh_command_parts[:2] == ['hg', '-R'] and ssh_command_parts[3:] == ['serve', '--stdio']:
- return cls(safe_unicode(ssh_command_parts[2]))
+ return cls(ssh_command_parts[2])
return None
@@ -61,9 +55,9 @@ class MercurialSshHandler(BaseSshHandler
# Note: we want a repo with config based on .hg/hgrc and can thus not use self.db_repo.scm_instance._repo.ui
baseui = make_ui(repo_path=self.db_repo.repo_full_path)
if not self.allow_push:
- baseui.setconfig('hooks', 'pretxnopen._ssh_reject', 'python:kallithea.lib.hooks.rejectpush')
- baseui.setconfig('hooks', 'prepushkey._ssh_reject', 'python:kallithea.lib.hooks.rejectpush')
+ baseui.setconfig(b'hooks', b'pretxnopen._ssh_reject', b'python:kallithea.lib.hooks.rejectpush')
+ baseui.setconfig(b'hooks', b'prepushkey._ssh_reject', b'python:kallithea.lib.hooks.rejectpush')
- repo = hg.repository(baseui, safe_str(self.db_repo.repo_full_path))
+ repo = mercurial.hg.repository(baseui, safe_bytes(self.db_repo.repo_full_path))
log.debug("Starting Mercurial sshserver for %s", self.db_repo.repo_full_path)
- sshserver(baseui, repo).serve_forever()
+ mercurial.wireprotoserver.sshserver(baseui, repo).serve_forever()
diff --git a/kallithea/lib/vcs/backends/hg/workdir.py b/kallithea/lib/vcs/backends/hg/workdir.py
--- a/kallithea/lib/vcs/backends/hg/workdir.py
+++ b/kallithea/lib/vcs/backends/hg/workdir.py
@@ -1,15 +1,17 @@
+import mercurial.merge
+
from kallithea.lib.vcs.backends.base import BaseWorkdir
from kallithea.lib.vcs.exceptions import BranchDoesNotExistError
-from kallithea.lib.vcs.utils.hgcompat import hg_merge
+from kallithea.lib.vcs.utils import ascii_bytes, ascii_str, safe_str
class MercurialWorkdir(BaseWorkdir):
def get_branch(self):
- return self.repository._repo.dirstate.branch()
+ return safe_str(self.repository._repo.dirstate.branch())
def get_changeset(self):
- wk_dir_id = self.repository._repo[None].parents()[0].hex()
+ wk_dir_id = ascii_str(self.repository._repo[None].parents()[0].hex())
return self.repository.get_changeset(wk_dir_id)
def checkout_branch(self, branch=None):
@@ -19,4 +21,4 @@ class MercurialWorkdir(BaseWorkdir):
raise BranchDoesNotExistError
raw_id = self.repository.branches[branch]
- hg_merge.update(self.repository._repo, raw_id, False, False, None)
+ mercurial.merge.update(self.repository._repo, ascii_bytes(raw_id), False, False, None)
diff --git a/kallithea/lib/vcs/backends/ssh.py b/kallithea/lib/vcs/backends/ssh.py
--- a/kallithea/lib/vcs/backends/ssh.py
+++ b/kallithea/lib/vcs/backends/ssh.py
@@ -24,7 +24,7 @@ import logging
import sys
from kallithea.lib.auth import AuthUser, HasPermissionAnyMiddleware
-from kallithea.lib.utils2 import safe_str, set_hook_environment
+from kallithea.lib.utils2 import set_hook_environment
from kallithea.model.db import Repository, User, UserSshKeys
from kallithea.model.meta import Session
@@ -82,7 +82,7 @@ class BaseSshHandler(object):
elif HasPermissionAnyMiddleware('repository.read')(self.authuser, self.repo_name):
self.allow_push = False
else:
- self.exit('Access to %r denied' % safe_str(self.repo_name))
+ self.exit('Access to %r denied' % self.repo_name)
self.db_repo = Repository.get_by_repo_name(self.repo_name)
if self.db_repo is None:
diff --git a/kallithea/lib/vcs/conf/settings.py b/kallithea/lib/vcs/conf/settings.py
--- a/kallithea/lib/vcs/conf/settings.py
+++ b/kallithea/lib/vcs/conf/settings.py
@@ -1,24 +1,7 @@
-import os
-import tempfile
-
from kallithea.lib.vcs.utils import aslist
-from kallithea.lib.vcs.utils.paths import get_user_home
-abspath = lambda * p: os.path.abspath(os.path.join(*p))
-
-VCSRC_PATH = os.environ.get('VCSRC_PATH')
-
-if not VCSRC_PATH:
- HOME_ = get_user_home()
- if not HOME_:
- HOME_ = tempfile.gettempdir()
-
-VCSRC_PATH = VCSRC_PATH or abspath(HOME_, '.vcsrc')
-if os.path.isdir(VCSRC_PATH):
- VCSRC_PATH = os.path.join(VCSRC_PATH, '__init__.py')
-
-# list of default encoding used in safe_unicode/safe_str methods
+# list of default encoding used in safe_str/safe_bytes methods
DEFAULT_ENCODINGS = aslist('utf-8')
# path to git executable run by run_git_command function
diff --git a/kallithea/lib/vcs/exceptions.py b/kallithea/lib/vcs/exceptions.py
--- a/kallithea/lib/vcs/exceptions.py
+++ b/kallithea/lib/vcs/exceptions.py
@@ -30,10 +30,6 @@ class TagDoesNotExistError(RepositoryErr
pass
-class BranchAlreadyExistError(RepositoryError):
- pass
-
-
class BranchDoesNotExistError(RepositoryError):
pass
@@ -50,10 +46,6 @@ class CommitError(RepositoryError):
pass
-class NothingChangedError(CommitError):
- pass
-
-
class NodeError(VCSError):
pass
@@ -88,7 +80,3 @@ class NodeAlreadyRemovedError(CommitErro
class ImproperArchiveTypeError(VCSError):
pass
-
-
-class CommandError(VCSError):
- pass
diff --git a/kallithea/lib/vcs/nodes.py b/kallithea/lib/vcs/nodes.py
--- a/kallithea/lib/vcs/nodes.py
+++ b/kallithea/lib/vcs/nodes.py
@@ -9,13 +9,14 @@
:copyright: (c) 2010-2011 by Marcin Kuzminski, Lukasz Balcerzak.
"""
+import functools
import mimetypes
import posixpath
import stat
from kallithea.lib.vcs.backends.base import EmptyChangeset
from kallithea.lib.vcs.exceptions import NodeError, RemovedFileNodeError
-from kallithea.lib.vcs.utils import safe_str, safe_unicode
+from kallithea.lib.vcs.utils import safe_bytes, safe_str
from kallithea.lib.vcs.utils.lazy import LazyProperty
@@ -26,10 +27,10 @@ class NodeKind:
class NodeState:
- ADDED = u'added'
- CHANGED = u'changed'
- NOT_CHANGED = u'not changed'
- REMOVED = u'removed'
+ ADDED = 'added'
+ CHANGED = 'changed'
+ NOT_CHANGED = 'not changed'
+ REMOVED = 'removed'
class NodeGeneratorBase(object):
@@ -44,11 +45,9 @@ class NodeGeneratorBase(object):
self.cs = cs
self.current_paths = current_paths
- def __call__(self):
- return [n for n in self]
-
- def __getslice__(self, i, j):
- for p in self.current_paths[i:j]:
+ def __getitem__(self, key):
+ assert isinstance(key, slice), key
+ for p in self.current_paths[key]:
yield self.cs.get_node(p)
def __len__(self):
@@ -81,11 +80,13 @@ class RemovedFileNodesGenerator(NodeGene
for p in self.current_paths:
yield RemovedFileNode(path=p)
- def __getslice__(self, i, j):
- for p in self.current_paths[i:j]:
+ def __getitem__(self, key):
+ assert isinstance(key, slice), key
+ for p in self.current_paths[key]:
yield RemovedFileNode(path=p)
+@functools.total_ordering
class Node(object):
"""
Simplest class representing file or directory on repository. SCM backends
@@ -101,7 +102,7 @@ class Node(object):
if path.startswith('/'):
raise NodeError("Cannot initialize Node objects with slash at "
"the beginning as only relative paths are supported")
- self.path = safe_str(path.rstrip('/')) # we store paths as str
+ self.path = path.rstrip('/')
if path == '' and kind != NodeKind.DIR:
raise NodeError("Only DirNode and its subclasses may be "
"initialized with empty path")
@@ -120,67 +121,34 @@ class Node(object):
return None
@LazyProperty
- def unicode_path(self):
- return safe_unicode(self.path)
-
- @LazyProperty
def name(self):
"""
Returns name of the node so if its path
then only last part is returned.
"""
- return safe_unicode(self.path.rstrip('/').split('/')[-1])
-
- def _get_kind(self):
- return self._kind
-
- def _set_kind(self, kind):
- if hasattr(self, '_kind'):
- raise NodeError("Cannot change node's kind")
- else:
- self._kind = kind
- # Post setter check (path's trailing slash)
- if self.path.endswith('/'):
- raise NodeError("Node's path cannot end with slash")
-
- kind = property(_get_kind, _set_kind)
-
- def __cmp__(self, other):
- """
- Comparator using name of the node, needed for quick list sorting.
- """
- kind_cmp = cmp(self.kind, other.kind)
- if kind_cmp:
- return kind_cmp
- return cmp(self.name, other.name)
+ return self.path.rstrip('/').split('/')[-1]
def __eq__(self, other):
- for attr in ['name', 'path', 'kind']:
- if getattr(self, attr) != getattr(other, attr):
- return False
- if self.is_file():
- if self.content != other.content:
- return False
- else:
- # For DirNode's check without entering each dir
- self_nodes_paths = list(sorted(n.path for n in self.nodes))
- other_nodes_paths = list(sorted(n.path for n in self.nodes))
- if self_nodes_paths != other_nodes_paths:
- return False
- return True
+ if type(self) is not type(other):
+ return False
+ if self.kind != other.kind:
+ return False
+ if self.path != other.path:
+ return False
- def __nq__(self, other):
- return not self.__eq__(other)
+ def __lt__(self, other):
+ if self.kind < other.kind:
+ return True
+ if self.kind > other.kind:
+ return False
+ if self.path < other.path:
+ return True
+ if self.path > other.path:
+ return False
def __repr__(self):
return '<%s %r>' % (self.__class__.__name__, self.path)
- def __str__(self):
- return self.__repr__()
-
- def __unicode__(self):
- return self.name
-
def get_parent_path(self):
"""
Returns node's parent path or empty string if node is root.
@@ -258,8 +226,24 @@ class FileNode(Node):
raise NodeError("Cannot use both content and changeset")
super(FileNode, self).__init__(path, kind=NodeKind.FILE)
self.changeset = changeset
+ if not isinstance(content, bytes) and content is not None:
+ # File content is one thing that inherently must be bytes ... but
+ # VCS module tries to be "user friendly" and support unicode ...
+ content = safe_bytes(content)
self._content = content
- self._mode = mode or 0100644
+ self._mode = mode or 0o100644
+
+ def __eq__(self, other):
+ eq = super(FileNode, self).__eq__(other)
+ if eq is not None:
+ return eq
+ return self.content == other.content
+
+ def __lt__(self, other):
+ lt = super(FileNode, self).__lt__(other)
+ if lt is not None:
+ return lt
+ return self.content < other.content
@LazyProperty
def mode(self):
@@ -273,25 +257,17 @@ class FileNode(Node):
mode = self._mode
return mode
- def _get_content(self):
+ @property
+ def content(self):
+ """
+ Returns lazily byte content of the FileNode.
+ """
if self.changeset:
content = self.changeset.get_file_content(self.path)
else:
content = self._content
return content
- @property
- def content(self):
- """
- Returns lazily content of the FileNode. If possible, would try to
- decode content from UTF-8.
- """
- content = self._get_content()
-
- if bool(content and '\0' in content):
- return content
- return safe_unicode(content)
-
@LazyProperty
def size(self):
if self.changeset:
@@ -361,7 +337,7 @@ class FileNode(Node):
"""
from pygments import lexers
try:
- lexer = lexers.guess_lexer_for_filename(self.name, self.content, stripnl=False)
+ lexer = lexers.guess_lexer_for_filename(self.name, safe_str(self.content), stripnl=False)
except lexers.ClassNotFound:
lexer = lexers.TextLexer(stripnl=False)
# returns first alias
@@ -409,8 +385,7 @@ class FileNode(Node):
"""
Returns True if file has binary content.
"""
- _bin = '\0' in self._get_content()
- return _bin
+ return b'\0' in self.content
def is_browser_compatible_image(self):
return self.mimetype in [
@@ -488,10 +463,23 @@ class DirNode(Node):
self.changeset = changeset
self._nodes = nodes
- @LazyProperty
- def content(self):
- raise NodeError("%s represents a dir and has no ``content`` attribute"
- % self)
+ def __eq__(self, other):
+ eq = super(DirNode, self).__eq__(other)
+ if eq is not None:
+ return eq
+ # check without entering each dir
+ self_nodes_paths = list(sorted(n.path for n in self.nodes))
+ other_nodes_paths = list(sorted(n.path for n in self.nodes))
+ return self_nodes_paths == other_nodes_paths
+
+ def __lt__(self, other):
+ lt = super(DirNode, self).__lt__(other)
+ if lt is not None:
+ return lt
+ # check without entering each dir
+ self_nodes_paths = list(sorted(n.path for n in self.nodes))
+ other_nodes_paths = list(sorted(n.path for n in self.nodes))
+ return self_nodes_paths < other_nodes_paths
@LazyProperty
def nodes(self):
@@ -595,12 +583,13 @@ class SubModuleNode(Node):
size = 0
def __init__(self, name, url, changeset=None, alias=None):
- self.path = name
+ # Note: Doesn't call Node.__init__!
+ self.path = name.rstrip('/')
self.kind = NodeKind.SUBMODULE
self.alias = alias
# we have to use emptyChangeset here since this can point to svn/git/hg
# submodules we cannot get from repository
- self.changeset = EmptyChangeset(str(changeset), alias=alias)
+ self.changeset = EmptyChangeset(changeset, alias=alias)
self.url = url
def __repr__(self):
@@ -613,5 +602,5 @@ class SubModuleNode(Node):
Returns name of the node so if its path
then only last part is returned.
"""
- org = safe_unicode(self.path.rstrip('/').split('/')[-1])
- return u'%s @ %s' % (org, self.changeset.short_id)
+ org = self.path.rstrip('/').rsplit('/', 1)[-1]
+ return '%s @ %s' % (org, self.changeset.short_id)
diff --git a/kallithea/lib/vcs/subprocessio.py b/kallithea/lib/vcs/subprocessio.py
--- a/kallithea/lib/vcs/subprocessio.py
+++ b/kallithea/lib/vcs/subprocessio.py
@@ -44,7 +44,7 @@ class StreamFeeder(threading.Thread):
if type(source) in (type(''), bytes, bytearray): # string-like
self.bytes = bytes(source)
else: # can be either file pointer or file-like
- if type(source) in (int, long): # file pointer it is
+ if isinstance(source, int): # file pointer it is
# converting file descriptor (int) stdin into file-like
source = os.fdopen(source, 'rb', 16384)
# let's see if source is file-like by now
@@ -125,11 +125,7 @@ class InputStreamChunker(threading.Threa
if len(t) > ccm:
kr.clear()
kr.wait(2)
- # # this only works on 2.7.x and up
- # if not kr.wait(10):
- # raise Exception("Timed out while waiting for input to be read.")
- # instead we'll use this
- if len(t) > ccm + 3:
+ if not kr.wait(10):
raise IOError(
"Timed out while waiting for input from subprocess.")
t.append(b)
@@ -178,7 +174,7 @@ class BufferedGenerator(object):
def __iter__(self):
return self
- def next(self):
+ def __next__(self):
while not len(self.data) and not self.worker.EOF.is_set():
self.worker.data_added.clear()
self.worker.data_added.wait(0.2)
@@ -225,17 +221,6 @@ class BufferedGenerator(object):
return not self.worker.keep_reading.is_set()
@property
- def done_reading_event(self):
- """
- Done_reading does not mean that the iterator's buffer is empty.
- Iterator might have done reading from underlying source, but the read
- chunks might still be available for serving through .next() method.
-
- :returns: An threading.Event class instance.
- """
- return self.worker.EOF
-
- @property
def done_reading(self):
"""
Done_reading does not mean that the iterator's buffer is empty.
@@ -286,7 +271,7 @@ class SubprocessIOChunker(object):
- We are multithreaded. Writing in and reading out, err are all sep threads.
- We support concurrent (in and out) stream processing.
- - The output is not a stream. It's a queue of read string (bytes, not unicode)
+ - The output is not a stream. It's a queue of read string (bytes, not str)
chunks. The object behaves as an iterable. You can "for chunk in obj:" us.
- We are non-blocking in more respects than communicate()
(reading from subprocess out pauses when internal buffer is full, but
@@ -367,18 +352,17 @@ class SubprocessIOChunker(object):
and returncode != 0
): # and it failed
bg_out.stop()
- out = ''.join(bg_out)
+ out = b''.join(bg_out)
bg_err.stop()
- err = ''.join(bg_err)
- if (err.strip() == 'fatal: The remote end hung up unexpectedly' and
- out.startswith('0034shallow ')
+ err = b''.join(bg_err)
+ if (err.strip() == b'fatal: The remote end hung up unexpectedly' and
+ out.startswith(b'0034shallow ')
):
# hack inspired by https://github.com/schacon/grack/pull/7
bg_out = iter([out])
_p = None
elif err:
- raise EnvironmentError(
- "Subprocess exited due to an error:\n" + err)
+ raise EnvironmentError("Subprocess exited due to an error: %s" % err)
else:
raise EnvironmentError(
"Subprocess exited with non 0 ret code: %s" % returncode)
@@ -390,7 +374,7 @@ class SubprocessIOChunker(object):
def __iter__(self):
return self
- def next(self):
+ def __next__(self):
if self.process:
returncode = self.process.poll()
if (returncode is not None # process has terminated
@@ -400,7 +384,7 @@ class SubprocessIOChunker(object):
self.error.stop()
err = ''.join(self.error)
raise EnvironmentError("Subprocess exited due to an error:\n" + err)
- return self.output.next()
+ return next(self.output)
def throw(self, type, value=None, traceback=None):
if self.output.length or not self.output.done_reading:
diff --git a/kallithea/lib/vcs/utils/__init__.py b/kallithea/lib/vcs/utils/__init__.py
--- a/kallithea/lib/vcs/utils/__init__.py
+++ b/kallithea/lib/vcs/utils/__init__.py
@@ -1,3 +1,5 @@
+# -*- coding: utf-8 -*-
+
"""
This module provides some useful tools for ``vcs`` like annotate/diff html
output. It also includes some internal helpers.
@@ -25,7 +27,7 @@ def aslist(obj, sep=None, strip=True):
:param sep:
:param strip:
"""
- if isinstance(obj, (basestring)):
+ if isinstance(obj, str):
lst = obj.split(sep)
if strip:
lst = [v.strip() for v in lst]
@@ -66,89 +68,107 @@ def safe_int(val, default=None):
return val
-def safe_unicode(str_, from_encoding=None):
+def safe_str(s):
"""
- safe unicode function. Does few trick to turn str_ into unicode
-
- In case of UnicodeDecode error we try to return it with encoding detected
- by chardet library if it fails fallback to unicode with errors replaced
-
- :param str_: string to decode
- :rtype: unicode
- :returns: unicode object
+ Safe unicode str function. Use a few tricks to turn s into str:
+ In case of UnicodeDecodeError with configured default encodings, try to
+ detect encoding with chardet library, then fall back to first encoding with
+ errors replaced.
"""
- if isinstance(str_, unicode):
- return str_
+ if isinstance(s, str):
+ return s
- if not from_encoding:
- from kallithea.lib.vcs.conf import settings
- from_encoding = settings.DEFAULT_ENCODINGS
-
- if not isinstance(from_encoding, (list, tuple)):
- from_encoding = [from_encoding]
+ if not isinstance(s, bytes): # use __str__ and don't expect UnicodeDecodeError
+ return str(s)
- try:
- return unicode(str_)
- except UnicodeDecodeError:
- pass
-
- for enc in from_encoding:
+ from kallithea.lib.vcs.conf import settings
+ for enc in settings.DEFAULT_ENCODINGS:
try:
- return unicode(str_, enc)
+ return str(s, enc)
except UnicodeDecodeError:
pass
try:
import chardet
- encoding = chardet.detect(str_)['encoding']
- if encoding is None:
- raise Exception()
- return str_.decode(encoding)
- except (ImportError, UnicodeDecodeError, Exception):
- return unicode(str_, from_encoding[0], 'replace')
+ encoding = chardet.detect(s)['encoding']
+ if encoding is not None:
+ return s.decode(encoding)
+ except (ImportError, UnicodeDecodeError):
+ pass
+
+ return str(s, settings.DEFAULT_ENCODINGS[0], 'replace')
-def safe_str(unicode_, to_encoding=None):
+def safe_bytes(s):
"""
- safe str function. Does few trick to turn unicode_ into string
-
- In case of UnicodeEncodeError we try to return it with encoding detected
- by chardet library if it fails fallback to string with errors replaced
-
- :param unicode_: unicode to encode
- :rtype: str
- :returns: str object
+ Safe bytes function. Use a few tricks to turn s into bytes string:
+ In case of UnicodeEncodeError with configured default encodings, fall back
+ to first configured encoding with errors replaced.
"""
+ if isinstance(s, bytes):
+ return s
- # if it's not basestr cast to str
- if not isinstance(unicode_, basestring):
- return str(unicode_)
-
- if isinstance(unicode_, str):
- return unicode_
+ assert isinstance(s, str), repr(s) # bytes cannot coerse with __str__ or handle None or int
- if not to_encoding:
- from kallithea.lib.vcs.conf import settings
- to_encoding = settings.DEFAULT_ENCODINGS
-
- if not isinstance(to_encoding, (list, tuple)):
- to_encoding = [to_encoding]
-
- for enc in to_encoding:
+ from kallithea.lib.vcs.conf import settings
+ for enc in settings.DEFAULT_ENCODINGS:
try:
- return unicode_.encode(enc)
+ return s.encode(enc)
except UnicodeEncodeError:
pass
- try:
- import chardet
- encoding = chardet.detect(unicode_)['encoding']
- if encoding is None:
- raise UnicodeEncodeError()
+ return s.encode(settings.DEFAULT_ENCODINGS[0], 'replace')
+
+
+def ascii_bytes(s):
+ """
+ Simple conversion from str to bytes, *assuming* all codepoints are
+ 7-bit and it thus is pure ASCII.
+ Will fail badly with UnicodeError on invalid input.
+ This should be used where enocding and "safe" ambiguity should be avoided.
+ Where strings already have been encoded in other ways but still are unicode
+ string - for example to hex, base64, json, urlencoding, or are known to be
+ identifiers.
- return unicode_.encode(encoding)
- except (ImportError, UnicodeEncodeError):
- return unicode_.encode(to_encoding[0], 'replace')
+ >>> ascii_bytes('a')
+ b'a'
+ >>> ascii_bytes(u'a')
+ b'a'
+ >>> ascii_bytes('å')
+ Traceback (most recent call last):
+ UnicodeEncodeError: 'ascii' codec can't encode character '\xe5' in position 0: ordinal not in range(128)
+ >>> ascii_bytes('å'.encode('utf8'))
+ Traceback (most recent call last):
+ AssertionError: b'\xc3\xa5'
+ """
+ assert isinstance(s, str), repr(s)
+ return s.encode('ascii')
+
+
+def ascii_str(s):
+ r"""
+ Simple conversion from bytes to str, *assuming* all codepoints are
+ 7-bit and it thus is pure ASCII.
+ Will fail badly with UnicodeError on invalid input.
+ This should be used where enocding and "safe" ambiguity should be avoided.
+ Where strings are encoded but also in other ways are known to be ASCII, and
+ where a unicode string is wanted without caring about encoding. For example
+ to hex, base64, urlencoding, or are known to be identifiers.
+
+ >>> ascii_str(b'a')
+ 'a'
+ >>> ascii_str(u'a')
+ Traceback (most recent call last):
+ AssertionError: 'a'
+ >>> ascii_str('å'.encode('utf8'))
+ Traceback (most recent call last):
+ UnicodeDecodeError: 'ascii' codec can't decode byte 0xc3 in position 0: ordinal not in range(128)
+ >>> ascii_str(u'å')
+ Traceback (most recent call last):
+ AssertionError: 'å'
+ """
+ assert isinstance(s, bytes), repr(s)
+ return s.decode('ascii')
# Regex taken from http://www.regular-expressions.info/email.html
@@ -178,7 +198,7 @@ def author_email(author):
m = email_re.search(author)
if m is None:
return ''
- return safe_str(m.group(0))
+ return m.group(0)
def author_name(author):
diff --git a/kallithea/lib/vcs/utils/annotate.py b/kallithea/lib/vcs/utils/annotate.py
deleted file mode 100644
--- a/kallithea/lib/vcs/utils/annotate.py
+++ /dev/null
@@ -1,179 +0,0 @@
-import StringIO
-
-from pygments import highlight
-from pygments.formatters import HtmlFormatter
-
-from kallithea.lib.vcs.exceptions import VCSError
-from kallithea.lib.vcs.nodes import FileNode
-
-
-def annotate_highlight(filenode, annotate_from_changeset_func=None,
- order=None, headers=None, **options):
- """
- Returns html portion containing annotated table with 3 columns: line
- numbers, changeset information and pygmentized line of code.
-
- :param filenode: FileNode object
- :param annotate_from_changeset_func: function taking changeset and
- returning single annotate cell; needs break line at the end
- :param order: ordered sequence of ``ls`` (line numbers column),
- ``annotate`` (annotate column), ``code`` (code column); Default is
- ``['ls', 'annotate', 'code']``
- :param headers: dictionary with headers (keys are whats in ``order``
- parameter)
- """
- options['linenos'] = True
- formatter = AnnotateHtmlFormatter(filenode=filenode, order=order,
- headers=headers,
- annotate_from_changeset_func=annotate_from_changeset_func, **options)
- lexer = filenode.lexer
- highlighted = highlight(filenode.content, lexer, formatter)
- return highlighted
-
-
-class AnnotateHtmlFormatter(HtmlFormatter):
-
- def __init__(self, filenode, annotate_from_changeset_func=None,
- order=None, **options):
- """
- If ``annotate_from_changeset_func`` is passed it should be a function
- which returns string from the given changeset. For example, we may pass
- following function as ``annotate_from_changeset_func``::
-
- def changeset_to_anchor(changeset):
- return '%s\n' % \
- (changeset.id, changeset.id)
-
- :param annotate_from_changeset_func: see above
- :param order: (default: ``['ls', 'annotate', 'code']``); order of
- columns;
- :param options: standard pygment's HtmlFormatter options, there is
- extra option tough, ``headers``. For instance we can pass::
-
- formatter = AnnotateHtmlFormatter(filenode, headers={
- 'ls': '#',
- 'annotate': 'Annotate',
- 'code': 'Code',
- })
-
- """
- super(AnnotateHtmlFormatter, self).__init__(**options)
- self.annotate_from_changeset_func = annotate_from_changeset_func
- self.order = order or ('ls', 'annotate', 'code')
- headers = options.pop('headers', None)
- if headers and not ('ls' in headers and 'annotate' in headers and
- 'code' in headers
- ):
- raise ValueError("If headers option dict is specified it must "
- "all 'ls', 'annotate' and 'code' keys")
- self.headers = headers
- if isinstance(filenode, FileNode):
- self.filenode = filenode
- else:
- raise VCSError("This formatter expect FileNode parameter, not %r"
- % type(filenode))
-
- def annotate_from_changeset(self, changeset):
- """
- Returns full html line for single changeset per annotated line.
- """
- if self.annotate_from_changeset_func:
- return self.annotate_from_changeset_func(changeset)
- else:
- return ''.join((changeset.id, '\n'))
-
- def _wrap_tablelinenos(self, inner):
- dummyoutfile = StringIO.StringIO()
- lncount = 0
- for t, line in inner:
- if t:
- lncount += 1
- dummyoutfile.write(line)
-
- fl = self.linenostart
- mw = len(str(lncount + fl - 1))
- sp = self.linenospecial
- st = self.linenostep
- la = self.lineanchors
- aln = self.anchorlinenos
- if sp:
- lines = []
-
- for i in range(fl, fl + lncount):
- if i % st == 0:
- if i % sp == 0:
- if aln:
- lines.append(''
- '%*d' %
- (la, i, mw, i))
- else:
- lines.append(''
- '%*d' % (mw, i))
- else:
- if aln:
- lines.append(''
- '%*d' % (la, i, mw, i))
- else:
- lines.append('%*d' % (mw, i))
- else:
- lines.append('')
- ls = '\n'.join(lines)
- else:
- lines = []
- for i in range(fl, fl + lncount):
- if i % st == 0:
- if aln:
- lines.append('%*d'
- % (la, i, mw, i))
- else:
- lines.append('%*d' % (mw, i))
- else:
- lines.append('')
- ls = '\n'.join(lines)
-
- annotate_changesets = [tup[1] for tup in self.filenode.annotate]
- # If pygments cropped last lines break we need do that too
- ln_cs = len(annotate_changesets)
- ln_ = len(ls.splitlines())
- if ln_cs > ln_:
- annotate_changesets = annotate_changesets[:ln_ - ln_cs]
- annotate = ''.join((self.annotate_from_changeset(changeset)
- for changeset in annotate_changesets))
- # in case you wonder about the seemingly redundant
' +
ls + ''
'
' + ls + '
" - "Date: %s
" - "Message: %s") % (author, date, message) + def url_func(changeset): + author = escape(changeset.author) + date = changeset.date + message = escape(changeset.message) + tooltip_html = ("Author: %s
" + "Date: %s
" + "Message: %s") % (author, date, message) - lnk_format = show_id(changeset) - uri = link_to( - lnk_format, - url('changeset_home', repo_name=repo_name, - revision=changeset.raw_id), - style=get_color_string(changeset.raw_id), - **{'data-toggle': 'popover', - 'data-content': tooltip_html} - ) + lnk_format = show_id(changeset) + uri = link_to( + lnk_format, + url('changeset_home', repo_name=repo_name, + revision=changeset.raw_id), + style=get_color_string(changeset.raw_id), + **{'data-toggle': 'popover', + 'data-content': tooltip_html} + ) - uri += '\n' - return uri - return _url_func + uri += '\n' + return uri - return literal(markup_whitespace(annotate_highlight(filenode, url_func(repo_name), **kwargs))) + return literal(markup_whitespace(annotate_highlight(filenode, url_func, **kwargs))) class _Message(object): @@ -424,22 +437,14 @@ class _Message(object): Converting the message to a string returns the message text. Instances also have the following attributes: - * ``message``: the message text. * ``category``: the category specified when the message was created. + * ``message``: the html-safe message text. """ def __init__(self, category, message): self.category = category self.message = message - def __str__(self): - return self.message - - __unicode__ = __str__ - - def __html__(self): - return escape(safe_unicode(self.message)) - def _session_flash_messages(append=None, clear=False): """Manage a message queue in tg.session: return the current message queue @@ -461,7 +466,7 @@ def _session_flash_messages(append=None, return flash_messages -def flash(message, category=None, logf=None): +def flash(message, category, logf=None): """ Show a message to the user _and_ log it through the specified function @@ -471,14 +476,22 @@ def flash(message, category=None, logf=N logf defaults to log.info, unless category equals 'success', in which case logf defaults to log.debug. """ + assert category in ('error', 'success', 'warning'), category + if hasattr(message, '__html__'): + # render to HTML for storing in cookie + safe_message = str(message) + else: + # Apply str - the message might be an exception with __str__ + # Escape, so we can trust the result without further escaping, without any risk of injection + safe_message = html_escape(str(message)) if logf is None: logf = log.info if category == 'success': logf = log.debug - logf('Flash %s: %s', category, message) + logf('Flash %s: %s', category, safe_message) - _session_flash_messages(append=(category, message)) + _session_flash_messages(append=(category, safe_message)) def pop_flash_messages(): @@ -486,14 +499,22 @@ def pop_flash_messages(): The return value is a list of ``Message`` objects. """ - return [_Message(*m) for m in _session_flash_messages(clear=True)] + return [_Message(category, message) for category, message in _session_flash_messages(clear=True)] -age = lambda x, y=False: _age(x, y) -capitalize = lambda x: x.capitalize() +def age(x, y=False): + return _age(x, y) + +def capitalize(x): + return x.capitalize() + email = author_email -short_id = lambda x: x[:12] -hide_credentials = lambda x: ''.join(credentials_filter(x)) + +def short_id(x): + return x[:12] + +def hide_credentials(x): + return ''.join(credentials_filter(x)) def show_id(cs): @@ -516,8 +537,7 @@ def show_id(cs): def fmt_date(date): if date: - return date.strftime("%Y-%m-%d %H:%M:%S").decode('utf-8') - + return date.strftime("%Y-%m-%d %H:%M:%S") return "" @@ -548,7 +568,7 @@ def user_attr_or_none(author, show_attr) email = author_email(author) if email: from kallithea.model.db import User - user = User.get_by_email(email, cache=True) # cache will only use sql_cache_short + user = User.get_by_email(email) if user is not None: return getattr(user, show_attr) return None @@ -590,15 +610,12 @@ def person(author, show_attr="username") def person_by_id(id_, show_attr="username"): from kallithea.model.db import User - # attr to return from fetched user - person_getter = lambda usr: getattr(usr, show_attr) - # maybe it's an ID ? if str(id_).isdigit() or isinstance(id_, int): id_ = int(id_) user = User.get(id_) if user is not None: - return person_getter(user) + return getattr(user, show_attr) return id_ @@ -677,7 +694,7 @@ def action_parser(user_log, feed=False, return _op, _name revs = [] - if len(filter(lambda v: v != '', revs_ids)) > 0: + if len([v for v in revs_ids if v != '']) > 0: repo = None for rev in revs_ids[:revs_top_limit]: _op, _name = _get_op(rev) @@ -850,10 +867,7 @@ def action_parser(user_log, feed=False, .replace('[', '') \ .replace(']', '') - action_params_func = lambda: "" - - if callable(action_str[1]): - action_params_func = action_str[1] + action_params_func = action_str[1] if callable(action_str[1]) else (lambda: "") def action_parser_icon(): action = user_log.action @@ -937,13 +951,13 @@ def gravatar_url(email_address, size=30, if email_address == _def: return default - parsed_url = urlparse.urlparse(url.current(qualified=True)) + parsed_url = urllib.parse.urlparse(url.current(qualified=True)) url = (c.visual.gravatar_url or User.DEFAULT_GRAVATAR_URL) \ .replace('{email}', email_address) \ - .replace('{md5email}', hashlib.md5(safe_str(email_address).lower()).hexdigest()) \ + .replace('{md5email}', hashlib.md5(safe_bytes(email_address).lower()).hexdigest()) \ .replace('{netloc}', parsed_url.netloc) \ .replace('{scheme}', parsed_url.scheme) \ - .replace('{size}', safe_str(size)) + .replace('{size}', str(size)) return url @@ -959,7 +973,7 @@ def changed_tooltip(nodes): suf = '' if len(nodes) > 30: suf = '
' + _(' and %s more') % (len(nodes) - 30) - return literal(pref + '
'.join([safe_unicode(x.path) + return literal(pref + '
'.join([x.path for x in nodes[:30]]) + suf) else: return ': ' + _('No files') @@ -1069,6 +1083,8 @@ def urlify_text(s, repo_name=None, link_ URLs links to what they say. Issues are linked to given issue-server. If link_ is provided, all text not already linking somewhere will link there. + >>> urlify_text("Urlify http://example.com/ and 'https://example.com' *and* markup/b>") + literal('Urlify http://example.com/ and 'https://example.com' *and* <b>markup/b>') """ def _replace(match_obj): @@ -1162,10 +1178,11 @@ def urlify_issues(newtext, repo_name): assert CONFIG['sqlalchemy.url'] # make sure config has been loaded # Build chain of urlify functions, starting with not doing any transformation - tmp_urlify_issues_f = lambda s: s + def tmp_urlify_issues_f(s): + return s issue_pat_re = re.compile(r'issue_pat(.*)') - for k in CONFIG.keys(): + for k in CONFIG: # Find all issue_pat* settings that also have corresponding server_link and prefix configuration m = issue_pat_re.match(k) if m is None: @@ -1214,9 +1231,9 @@ def urlify_issues(newtext, repo_name): 'url': issue_url, 'text': issue_text, } - tmp_urlify_issues_f = (lambda s, - issue_re=issue_re, issues_replace=issues_replace, chain_f=tmp_urlify_issues_f: - issue_re.sub(issues_replace, chain_f(s))) + + def tmp_urlify_issues_f(s, issue_re=issue_re, issues_replace=issues_replace, chain_f=tmp_urlify_issues_f): + return issue_re.sub(issues_replace, chain_f(s)) # Set tmp function globally - atomically _urlify_issues_f = tmp_urlify_issues_f @@ -1229,7 +1246,7 @@ def render_w_mentions(source, repo_name= Render plain text with revision hashes and issue references urlified and with @mention highlighting. """ - s = safe_unicode(source) + s = safe_str(source) s = urlify_text(s, repo_name=repo_name) return literal('
%s
' % s)
diff --git a/kallithea/lib/hooks.py b/kallithea/lib/hooks.py
--- a/kallithea/lib/hooks.py
+++ b/kallithea/lib/hooks.py
@@ -25,17 +25,17 @@ Original author and date, and relevant c
:license: GPLv3, see LICENSE.md for more details.
"""
-import binascii
import os
import sys
import time
+import mercurial.scmutil
+
from kallithea.lib import helpers as h
from kallithea.lib.exceptions import UserCreationError
-from kallithea.lib.utils import action_logger, make_ui, setup_cache_regions
-from kallithea.lib.utils2 import HookEnvironmentError, get_hook_environment, safe_str, safe_unicode
+from kallithea.lib.utils import action_logger, make_ui
+from kallithea.lib.utils2 import HookEnvironmentError, ascii_str, get_hook_environment, safe_bytes, safe_str
from kallithea.lib.vcs.backends.base import EmptyChangeset
-from kallithea.lib.vcs.utils.hgcompat import revrange
from kallithea.model.db import Repository, User
@@ -44,7 +44,7 @@ def _get_scm_size(alias, root_path):
alias += '.'
size_scm, size_root = 0, 0
- for path, dirs, files in os.walk(safe_str(root_path)):
+ for path, dirs, files in os.walk(root_path):
if path.find(alias) != -1:
for f in files:
try:
@@ -66,16 +66,16 @@ def _get_scm_size(alias, root_path):
def repo_size(ui, repo, hooktype=None, **kwargs):
- """Presents size of repository after push"""
- size_hg_f, size_root_f, size_total_f = _get_scm_size('.hg', repo.root)
+ """Show size of Mercurial repository, to be called after push."""
+ size_hg_f, size_root_f, size_total_f = _get_scm_size('.hg', safe_str(repo.root))
last_cs = repo[len(repo) - 1]
msg = ('Repository size .hg: %s Checkout: %s Total: %s\n'
'Last revision is now r%s:%s\n') % (
- size_hg_f, size_root_f, size_total_f, last_cs.rev(), last_cs.hex()[:12]
+ size_hg_f, size_root_f, size_total_f, last_cs.rev(), ascii_str(last_cs.hex())[:12]
)
- ui.status(msg)
+ ui.status(safe_bytes(msg))
def log_pull_action(ui, repo, **kwargs):
@@ -110,8 +110,7 @@ def log_push_action(ui, repo, node, node
Note: This hook is not only logging, but also the side effect invalidating
cahes! The function should perhaps be renamed.
"""
- _h = binascii.hexlify
- revs = [_h(repo[r].node()) for r in revrange(repo, [node + ':' + node_last])]
+ revs = [ascii_str(repo[r].hex()) for r in mercurial.scmutil.revrange(repo, [b'%s:%s' % (node, node_last)])]
process_pushed_raw_ids(revs)
return 0
@@ -303,31 +302,23 @@ def _hook_environment(repo_path):
they thus need enough info to be able to create an app environment and
connect to the database.
"""
- from paste.deploy import appconfig
- from sqlalchemy import engine_from_config
- from kallithea.config.environment import load_environment
- from kallithea.model.base import init_model
+ import paste.deploy
+ import kallithea.config.middleware
extras = get_hook_environment()
- ini_file_path = extras['config']
- #logging.config.fileConfig(ini_file_path) # Note: we are in a different process - don't use configured logging
- app_conf = appconfig('config:%s' % ini_file_path)
- conf = load_environment(app_conf.global_conf, app_conf.local_conf)
- setup_cache_regions(conf)
+ path_to_ini_file = extras['config']
+ kallithea.CONFIG = paste.deploy.appconfig('config:' + path_to_ini_file)
+ #logging.config.fileConfig(ini_file_path) # Note: we are in a different process - don't use configured logging
+ kallithea.config.middleware.make_app(kallithea.CONFIG.global_conf, **kallithea.CONFIG.local_conf)
- engine = engine_from_config(conf, 'sqlalchemy.')
- init_model(engine)
-
- repo_path = safe_unicode(repo_path)
# fix if it's not a bare repo
if repo_path.endswith(os.sep + '.git'):
repo_path = repo_path[:-5]
repo = Repository.get_by_full_path(repo_path)
if not repo:
- raise OSError('Repository %s not found in database'
- % (safe_str(repo_path)))
+ raise OSError('Repository %s not found in database' % repo_path)
baseui = make_ui()
return baseui, repo
@@ -368,19 +359,20 @@ def handle_git_post_receive(repo_path, g
if push_ref['old_rev'] == EmptyChangeset().raw_id:
# update the symbolic ref if we push new repo
if scm_repo.is_empty():
- scm_repo._repo.refs.set_symbolic_ref('HEAD',
- 'refs/heads/%s' % push_ref['name'])
+ scm_repo._repo.refs.set_symbolic_ref(
+ b'HEAD',
+ b'refs/heads/%s' % safe_bytes(push_ref['name']))
# build exclude list without the ref
cmd = ['for-each-ref', '--format=%(refname)', 'refs/heads/*']
- stdout, stderr = scm_repo.run_git_command(cmd)
+ stdout = scm_repo.run_git_command(cmd)
ref = push_ref['ref']
heads = [head for head in stdout.splitlines() if head != ref]
# now list the git revs while excluding from the list
cmd = ['log', push_ref['new_rev'], '--reverse', '--pretty=format:%H']
cmd.append('--not')
cmd.extend(heads) # empty list is ok
- stdout, stderr = scm_repo.run_git_command(cmd)
+ stdout = scm_repo.run_git_command(cmd)
git_revs += stdout.splitlines()
elif push_ref['new_rev'] == EmptyChangeset().raw_id:
@@ -389,7 +381,7 @@ def handle_git_post_receive(repo_path, g
else:
cmd = ['log', '%(old_rev)s..%(new_rev)s' % push_ref,
'--reverse', '--pretty=format:%H']
- stdout, stderr = scm_repo.run_git_command(cmd)
+ stdout = scm_repo.run_git_command(cmd)
git_revs += stdout.splitlines()
elif _type == 'tags':
@@ -404,5 +396,5 @@ def handle_git_post_receive(repo_path, g
def rejectpush(ui, **kwargs):
"""Mercurial hook to be installed as pretxnopen and prepushkey for read-only repos"""
ex = get_hook_environment()
- ui.warn((b"Push access to %r denied\n") % safe_str(ex.repository))
+ ui.warn(safe_bytes("Push access to %r denied\n" % ex.repository))
return 1
diff --git a/kallithea/lib/indexers/__init__.py b/kallithea/lib/indexers/__init__.py
--- a/kallithea/lib/indexers/__init__.py
+++ b/kallithea/lib/indexers/__init__.py
@@ -146,7 +146,7 @@ class WhooshResultWrapper(object):
docnum = self.matcher.id()
chunks = [offsets for offsets in self.get_chunks()]
docs_id.append([docnum, chunks])
- self.matcher.next()
+ self.matcher.next() # this looks like a py2 iterator ... but it isn't
return docs_id
def __str__(self):
@@ -203,7 +203,7 @@ class WhooshResultWrapper(object):
return res
def get_short_content(self, res, chunks):
- return u''.join([res['content'][chunk[0]:chunk[1]] for chunk in chunks])
+ return ''.join([res['content'][chunk[0]:chunk[1]] for chunk in chunks])
def get_chunks(self):
"""
diff --git a/kallithea/lib/indexers/daemon.py b/kallithea/lib/indexers/daemon.py
--- a/kallithea/lib/indexers/daemon.py
+++ b/kallithea/lib/indexers/daemon.py
@@ -39,8 +39,8 @@ from whoosh.qparser import QueryParser
from kallithea.config.conf import INDEX_EXTENSIONS, INDEX_FILENAMES
from kallithea.lib.indexers import CHGSET_IDX_NAME, CHGSETS_SCHEMA, IDX_NAME, SCHEMA
-from kallithea.lib.utils2 import safe_str, safe_unicode
-from kallithea.lib.vcs.exceptions import ChangesetError, NodeDoesNotExistError, RepositoryError
+from kallithea.lib.utils2 import safe_str
+from kallithea.lib.vcs.exceptions import ChangesetDoesNotExistError, ChangesetError, NodeDoesNotExistError, RepositoryError
from kallithea.model.db import Repository
from kallithea.model.scm import ScmModel
@@ -77,8 +77,7 @@ class WhooshIndexingDaemon(object):
# filter repo list
if repo_list:
- # Fix non-ascii repo names to unicode
- repo_list = map(safe_unicode, repo_list)
+ repo_list = set(repo_list)
self.filtered_repo_paths = {}
for repo_name, repo in self.repo_paths.items():
if repo_name in repo_list:
@@ -110,7 +109,7 @@ class WhooshIndexingDaemon(object):
self.initial = False
def _get_index_revision(self, repo):
- db_repo = Repository.get_by_repo_name(repo.name_unicode)
+ db_repo = Repository.get_by_repo_name(repo.name)
landing_rev = 'tip'
if db_repo:
_rev_type, _rev = db_repo.landing_rev
@@ -133,7 +132,7 @@ class WhooshIndexingDaemon(object):
cs = self._get_index_changeset(repo)
for _topnode, _dirs, files in cs.walk('/'):
for f in files:
- index_paths_.add(os.path.join(safe_str(repo.path), safe_str(f.path)))
+ index_paths_.add(os.path.join(repo.path, f.path))
except RepositoryError:
log.debug(traceback.format_exc())
@@ -142,19 +141,16 @@ class WhooshIndexingDaemon(object):
def get_node(self, repo, path, index_rev=None):
"""
- gets a filenode based on given full path. It operates on string for
- hg git compatibility.
+ gets a filenode based on given full path.
:param repo: scm repo instance
:param path: full path including root location
:return: FileNode
"""
# FIXME: paths should be normalized ... or even better: don't include repo.path
- path = safe_str(path)
- repo_path = safe_str(repo.path)
- assert path.startswith(repo_path)
- assert path[len(repo_path)] in (os.path.sep, os.path.altsep)
- node_path = path[len(repo_path) + 1:]
+ assert path.startswith(repo.path)
+ assert path[len(repo.path)] in (os.path.sep, os.path.altsep)
+ node_path = path[len(repo.path) + 1:]
cs = self._get_index_changeset(repo, index_rev=index_rev)
node = cs.get_node(node_path)
return node
@@ -182,27 +178,27 @@ class WhooshIndexingDaemon(object):
indexed = indexed_w_content = 0
if self.is_indexable_node(node):
- u_content = node.content
- if not isinstance(u_content, unicode):
+ bytes_content = node.content
+ if b'\0' in bytes_content:
log.warning(' >> %s - no text content', path)
- u_content = u''
+ u_content = ''
else:
log.debug(' >> %s', path)
+ u_content = safe_str(bytes_content)
indexed_w_content += 1
else:
log.debug(' >> %s - not indexable', path)
# just index file name without it's content
- u_content = u''
+ u_content = ''
indexed += 1
- p = safe_unicode(path)
writer.add_document(
- fileid=p,
- owner=unicode(repo.contact),
- repository_rawname=safe_unicode(repo_name),
- repository=safe_unicode(repo_name),
- path=p,
+ fileid=path,
+ owner=repo.contact,
+ repository_rawname=repo_name,
+ repository=repo_name,
+ path=path,
content=u_content,
modtime=self.get_node_mtime(node),
extension=node.extension
@@ -237,18 +233,18 @@ class WhooshIndexingDaemon(object):
indexed += 1
log.debug(' >> %s %s/%s', cs, indexed, total)
writer.add_document(
- raw_id=unicode(cs.raw_id),
- owner=unicode(repo.contact),
+ raw_id=cs.raw_id,
+ owner=repo.contact,
date=cs._timestamp,
- repository_rawname=safe_unicode(repo_name),
- repository=safe_unicode(repo_name),
+ repository_rawname=repo_name,
+ repository=repo_name,
author=cs.author,
message=cs.message,
last=cs.last,
- added=u' '.join([safe_unicode(node.path) for node in cs.added]).lower(),
- removed=u' '.join([safe_unicode(node.path) for node in cs.removed]).lower(),
- changed=u' '.join([safe_unicode(node.path) for node in cs.changed]).lower(),
- parents=u' '.join([cs.raw_id for cs in cs.parents]),
+ added=' '.join(node.path for node in cs.added).lower(),
+ removed=' '.join(node.path for node in cs.removed).lower(),
+ changed=' '.join(node.path for node in cs.changed).lower(),
+ parents=' '.join(cs.raw_id for cs in cs.parents),
)
return indexed
@@ -291,7 +287,7 @@ class WhooshIndexingDaemon(object):
continue
qp = QueryParser('repository', schema=CHGSETS_SCHEMA)
- q = qp.parse(u"last:t AND %s" % repo_name)
+ q = qp.parse("last:t AND %s" % repo_name)
results = searcher.search(q)
@@ -303,14 +299,18 @@ class WhooshIndexingDaemon(object):
# assuming that there is only one result, if not this
# may require a full re-index.
start_id = results[0]['raw_id']
- last_rev = repo.get_changeset(revision=start_id).revision
+ try:
+ last_rev = repo.get_changeset(revision=start_id).revision
+ except ChangesetDoesNotExistError:
+ log.error('previous last revision %s not found - indexing from scratch', start_id)
+ start_id = None
# there are new changesets to index or a new repo to index
if last_rev == 0 or num_of_revs > last_rev + 1:
# delete the docs in the index for the previous
# last changeset(s)
for hit in results:
- q = qp.parse(u"last:t AND %s AND raw_id:%s" %
+ q = qp.parse("last:t AND %s AND raw_id:%s" %
(repo_name, hit['raw_id']))
writer.delete_by_query(q)
@@ -330,8 +330,8 @@ class WhooshIndexingDaemon(object):
log.debug('>> NOTHING TO COMMIT TO CHANGESET INDEX<<')
def update_file_index(self):
- log.debug((u'STARTING INCREMENTAL INDEXING UPDATE FOR EXTENSIONS %s '
- 'AND REPOS %s') % (INDEX_EXTENSIONS, self.repo_paths.keys()))
+ log.debug('STARTING INCREMENTAL INDEXING UPDATE FOR EXTENSIONS %s '
+ 'AND REPOS %s', INDEX_EXTENSIONS, ' and '.join(self.repo_paths))
idx = open_dir(self.index_location, indexname=self.indexname)
# The set of all paths in the index
@@ -390,9 +390,7 @@ class WhooshIndexingDaemon(object):
ri_cnt = 0 # indexed
riwc_cnt = 0 # indexed with content
for path in self.get_paths(repo):
- path = safe_unicode(path)
if path in to_index or path not in indexed_paths:
-
# This is either a file that's changed, or a new file
# that wasn't indexed before. So index it!
i, iwc = self.add_doc(writer, path, repo, repo_name)
@@ -431,7 +429,7 @@ class WhooshIndexingDaemon(object):
file_idx = create_in(self.index_location, SCHEMA, indexname=IDX_NAME)
file_idx_writer = file_idx.writer()
log.debug('BUILDING INDEX FOR EXTENSIONS %s '
- 'AND REPOS %s' % (INDEX_EXTENSIONS, self.repo_paths.keys()))
+ 'AND REPOS %s', INDEX_EXTENSIONS, ' and '.join(self.repo_paths))
for repo_name, repo in sorted(self.repo_paths.items()):
log.debug('Updating indices for repo %s', repo_name)
diff --git a/kallithea/lib/inifile.py b/kallithea/lib/inifile.py
--- a/kallithea/lib/inifile.py
+++ b/kallithea/lib/inifile.py
@@ -42,6 +42,10 @@ default_variables = {
'uuid': lambda: 'VERY-SECRET',
}
+variable_options = {
+ 'database_engine': ['sqlite', 'postgres', 'mysql'],
+ 'http_server': ['waitress', 'gearbox', 'gevent', 'gunicorn', 'uwsgi'],
+}
def expand(template, mako_variable_values, settings):
"""Expand mako template and tweak it.
@@ -63,16 +67,27 @@ def expand(template, mako_variable_value
... %elif conditional_options == 'option-b':
... some_variable = "never mind - option-b will not be used anyway ..."
... %endif
+ ...
+ ... [comment-section]
+ ... #variable3 = 3.0
+ ... #variable4 = 4.0
+ ... #variable5 = 5.0
+ ... variable5 = 5.1
+ ... #variable6 = 6.0
+ ... #variable6 = 6.1
+ ... #variable7 = 7.0
+ ... variable7 = 7.1
... '''
- >>> selected_mako_conditionals = []
>>> mako_variable_values = {'mako_variable': 'VALUE', 'mako_function': (lambda: 'FUNCTION RESULT'),
- ... 'conditional_options': 'option-a'}
+ ... 'conditional_options': 'option-a', 'http_server': 'nc'}
>>> settings = { # only partially used
... '[first-section]': {'variable2': 'VAL2', 'first_extra': 'EXTRA'},
+ ... '[comment-section]': {'variable3': '3.0', 'variable4': '4.1', 'variable5': '5.2', 'variable6': '6.2', 'variable7': '7.0'},
... '[third-section]': {'third_extra': ' 3'},
... '[fourth-section]': {'fourth_extra': '4', 'fourth': '"four"'},
... }
- >>> print expand(template, mako_variable_values, settings)
+ >>> print(expand(template, mako_variable_values, settings))
+ ERROR: http_server is 'nc' - it should be one of 'waitress', 'gearbox', 'gevent', 'gunicorn', 'uwsgi'
>>> MarkupRenderer.render('''
''', '.md')
- u'
''', 'b.mkd')
- u'yo
''', 'markdown')
- u'yo
'
+ 'yo
'
>>> MarkupRenderer.render('''yo''', 'md')
- u''
+ ''
"""
renderer = cls._detect_renderer(source, filename)
@@ -150,7 +150,11 @@ class MarkupRenderer(object):
@classmethod
def plain(cls, source, universal_newline=True):
- source = safe_unicode(source)
+ """
+ >>> MarkupRenderer.plain('https://example.com/')
+ 'https://example.com/' + """ + source = safe_str(source) if universal_newline: newline = '\n' source = newline.join(source.splitlines()) @@ -168,30 +172,30 @@ class MarkupRenderer(object): with "safe" fall-back to plaintext. Output from this method should be sanitized before use. >>> MarkupRenderer.markdown('''
''')
- u'
''')
- u'
yo
''')
- u'yo
'
+ 'yo
'
>>> MarkupRenderer.markdown('''yo''')
- u''
+ ''
>>> MarkupRenderer.markdown('''## Foo''')
- u'Foo
' - >>> print MarkupRenderer.markdown(''' + 'Foo
' + >>> print(MarkupRenderer.markdown(''' ... #!/bin/bash ... echo "hello" - ... ''') + ... '''))1
2 | #!/bin/bash
echo "hello"
|
- $link_previous ~2~ $link_next
- $link_previous\n~4~$link_next
here:
- # since the content in the other cell also is wrapped in a div,
- # some browsers in some configurations seem to mess up the formatting.
- '''
- yield 0, ('' % self.cssclass +
- '
'
-
- '''
- headers_row = []
- if self.headers:
- headers_row = ['']
- for key in self.order:
- td = ''.join(('', self.headers[key], ' '))
- headers_row.append(td)
- headers_row.append(' ')
-
- body_row_start = ['']
- for key in self.order:
- if key == 'ls':
- body_row_start.append(
- '')
- yield 0, ('' % self.cssclass +
- ''.join(headers_row) +
- ''.join(body_row_start)
- )
- yield 0, dummyoutfile.getvalue()
- yield 0, '
'
diff --git a/kallithea/lib/vcs/utils/archivers.py b/kallithea/lib/vcs/utils/archivers.py
deleted file mode 100644
--- a/kallithea/lib/vcs/utils/archivers.py
+++ /dev/null
@@ -1,67 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
- vcs.utils.archivers
- ~~~~~~~~~~~~~~~~~~~
-
- set of archiver functions for creating archives from repository content
-
- :created_on: Jan 21, 2011
- :copyright: (c) 2010-2011 by Marcin Kuzminski, Lukasz Balcerzak.
-"""
-
-
-class BaseArchiver(object):
-
- def __init__(self):
- self.archive_file = self._get_archive_file()
-
- def addfile(self):
- """
- Adds a file to archive container
- """
- pass
-
- def close(self):
- """
- Closes and finalizes operation of archive container object
- """
- self.archive_file.close()
-
- def _get_archive_file(self):
- """
- Returns container for specific archive
- """
- raise NotImplementedError()
-
-
-class TarArchiver(BaseArchiver):
- pass
-
-
-class Tbz2Archiver(BaseArchiver):
- pass
-
-
-class TgzArchiver(BaseArchiver):
- pass
-
-
-class ZipArchiver(BaseArchiver):
- pass
-
-
-def get_archiver(self, kind):
- """
- Returns instance of archiver class specific to given kind
-
- :param kind: archive kind
- """
-
- archivers = {
- 'tar': TarArchiver,
- 'tbz2': Tbz2Archiver,
- 'tgz': TgzArchiver,
- 'zip': ZipArchiver,
- }
-
- return archivers[kind]()
diff --git a/kallithea/lib/vcs/utils/fakemod.py b/kallithea/lib/vcs/utils/fakemod.py
--- a/kallithea/lib/vcs/utils/fakemod.py
+++ b/kallithea/lib/vcs/utils/fakemod.py
@@ -9,5 +9,5 @@ def create_module(name, path):
"""
module = imp.new_module(name)
module.__file__ = path
- execfile(path, module.__dict__)
+ exec(compile(open(path, "rb").read(), path, 'exec'), module.__dict__)
return module
diff --git a/kallithea/lib/vcs/utils/helpers.py b/kallithea/lib/vcs/utils/helpers.py
--- a/kallithea/lib/vcs/utils/helpers.py
+++ b/kallithea/lib/vcs/utils/helpers.py
@@ -1,7 +1,6 @@
"""
Utilities aimed to help achieve mostly basic tasks.
"""
-from __future__ import division
import datetime
import os
@@ -33,16 +32,14 @@ def get_scm(path, search_up=False, expli
if not os.path.isdir(path):
raise VCSError("Given path %s is not a directory" % path)
- def get_scms(path):
- return [(scm, path) for scm in get_scms_for_path(path)]
-
- found_scms = get_scms(path)
- while not found_scms and search_up:
+ while True:
+ found_scms = [(scm, path) for scm in get_scms_for_path(path)]
+ if found_scms or not search_up:
+ break
newpath = abspath(path, '..')
if newpath == path:
break
path = newpath
- found_scms = get_scms(path)
if len(found_scms) > 1:
for scm in found_scms:
@@ -133,7 +130,7 @@ def parse_changesets(text):
>>> parse_changesets('aaabbb')
{'start': None, 'main': 'aaabbb', 'end': None}
>>> parse_changesets('aaabbb..cccddd')
- {'start': 'aaabbb', 'main': None, 'end': 'cccddd'}
+ {'start': 'aaabbb', 'end': 'cccddd', 'main': None}
"""
text = text.strip()
diff --git a/kallithea/lib/vcs/utils/hgcompat.py b/kallithea/lib/vcs/utils/hgcompat.py
--- a/kallithea/lib/vcs/utils/hgcompat.py
+++ b/kallithea/lib/vcs/utils/hgcompat.py
@@ -2,46 +2,16 @@
Mercurial libs compatibility
"""
-# Mercurial 5.0 550a172a603b renamed memfilectx argument `copied` to `copysource`
-import inspect
-
-import mercurial
-from mercurial import archival, config, demandimport, discovery, httppeer, localrepo
-from mercurial import merge as hg_merge
-from mercurial import obsutil, patch, scmutil, sshpeer, ui, unionrepo
-from mercurial.commands import clone, nullid, pull
-from mercurial.context import memctx, memfilectx
-from mercurial.discovery import findcommonoutgoing
-from mercurial.encoding import tolocal
-from mercurial.error import Abort, RepoError, RepoLookupError
-from mercurial.hg import peer
-from mercurial.hgweb import hgweb_mod
-from mercurial.hgweb.common import get_contact
-from mercurial.match import exact as match_exact
-from mercurial.match import match
-from mercurial.mdiff import diffopts
-from mercurial.node import hex, nullrev
-from mercurial.scmutil import revrange
-from mercurial.tags import tag
-from mercurial.url import httpbasicauthhandler, httpdigestauthhandler
-from mercurial.util import url as hg_url
+import mercurial.encoding
+import mercurial.localrepo
-# patch demandimport, due to bug in mercurial when it always triggers demandimport.enable()
-demandimport.enable = lambda *args, **kwargs: 1
-
-
-# workaround for 3.3 94ac64bcf6fe and not calling largefiles reposetup correctly
-localrepo.localrepository._lfstatuswriters = [lambda *msg, **opts: None]
-# 3.5 7699d3212994 added the invariant that repo.lfstatus must exist before hitting overridearchive
-localrepo.localrepository.lfstatus = False
+def monkey_do():
+ """Apply some Mercurial monkey patching"""
+ # workaround for 3.3 94ac64bcf6fe and not calling largefiles reposetup correctly, and test_archival failing
+ mercurial.localrepo.localrepository._lfstatuswriters = [lambda *msg, **opts: None]
+ # 3.5 7699d3212994 added the invariant that repo.lfstatus must exist before hitting overridearchive
+ mercurial.localrepo.localrepository.lfstatus = False
-if inspect.getargspec(memfilectx.__init__).args[7] != 'copysource':
- assert inspect.getargspec(memfilectx.__init__).args[7] == 'copied', inspect.getargspec(memfilectx.__init__).args
- __org_memfilectx_ = memfilectx
- memfilectx = lambda repo, changectx, path, data, islink=False, isexec=False, copysource=None: \
- __org_memfilectx_(repo, changectx, path, data, islink=islink, isexec=isexec, copied=copysource)
-
-# Mercurial 5.0 dropped exact argument for match in 635a12c53ea6, and 0531dff73d0b made the exact function stable with a single parameter
-if inspect.getargspec(match_exact).args[0] != 'files':
- match_exact = lambda path: match(None, '', [path], exact=True)
+ # Minimize potential impact from custom configuration
+ mercurial.encoding.environ[b'HGPLAIN'] = b'1'
diff --git a/kallithea/lib/vcs/utils/imports.py b/kallithea/lib/vcs/utils/imports.py
--- a/kallithea/lib/vcs/utils/imports.py
+++ b/kallithea/lib/vcs/utils/imports.py
@@ -1,6 +1,3 @@
-from kallithea.lib.vcs.exceptions import VCSError
-
-
def import_class(class_path):
"""
Returns class from the given path.
@@ -8,10 +5,7 @@ def import_class(class_path):
For example, in order to get class located at
``vcs.backends.hg.MercurialRepository``:
- try:
- hgrepo = import_class('vcs.backends.hg.MercurialRepository')
- except VCSError:
- # handle error
+ hgrepo = import_class('vcs.backends.hg.MercurialRepository')
"""
splitted = class_path.split('.')
mod_path = '.'.join(splitted[:-1])
diff --git a/kallithea/lib/vcs/utils/lazy.py b/kallithea/lib/vcs/utils/lazy.py
--- a/kallithea/lib/vcs/utils/lazy.py
+++ b/kallithea/lib/vcs/utils/lazy.py
@@ -1,6 +1,3 @@
-import threading
-
-
class _Missing(object):
def __repr__(self):
@@ -44,21 +41,3 @@ class LazyProperty(object):
value = self._func(obj)
obj.__dict__[self.__name__] = value
return value
-
-
-class ThreadLocalLazyProperty(LazyProperty):
- """
- Same as above but uses thread local dict for cache storage.
- """
-
- def __get__(self, obj, klass=None):
- if obj is None:
- return self
- if not hasattr(obj, '__tl_dict__'):
- obj.__tl_dict__ = threading.local().__dict__
-
- value = obj.__tl_dict__.get(self.__name__, _missing)
- if value is _missing:
- value = self._func(obj)
- obj.__tl_dict__[self.__name__] = value
- return value
diff --git a/kallithea/lib/vcs/utils/paths.py b/kallithea/lib/vcs/utils/paths.py
--- a/kallithea/lib/vcs/utils/paths.py
+++ b/kallithea/lib/vcs/utils/paths.py
@@ -1,7 +1,8 @@
import os
-abspath = lambda * p: os.path.abspath(os.path.join(*p))
+def abspath(*p):
+ return os.path.abspath(os.path.join(*p))
def get_dirs_for_path(*paths):
@@ -11,7 +12,7 @@ def get_dirs_for_path(*paths):
for path in paths:
head = path
while head:
- head, tail = os.path.split(head)
+ head, _tail = os.path.split(head)
if head:
yield head
else:
diff --git a/kallithea/lib/vcs/utils/progressbar.py b/kallithea/lib/vcs/utils/progressbar.py
--- a/kallithea/lib/vcs/utils/progressbar.py
+++ b/kallithea/lib/vcs/utils/progressbar.py
@@ -1,7 +1,5 @@
# encoding: UTF-8
-from __future__ import print_function
-
import datetime
import string
import sys
@@ -43,7 +41,7 @@ class ProgressBar(object):
def __iter__(self):
start = self.step
end = self.steps + 1
- for x in xrange(start, end):
+ for x in range(start, end):
self.render(x)
yield x
@@ -215,7 +213,7 @@ def colorize(text='', opts=(), **kwargs)
code_list = []
if text == '' and len(opts) == 1 and opts[0] == 'reset':
return '\x1b[%sm' % RESET
- for k, v in kwargs.iteritems():
+ for k, v in kwargs.items():
if k == 'fg':
code_list.append(foreground[v])
elif k == 'bg':
@@ -359,7 +357,7 @@ def main():
print("Standard progress bar...")
bar = ProgressBar(30)
- for x in xrange(1, 31):
+ for x in range(1, 31):
bar.render(x)
time.sleep(0.02)
bar.stream.write('\n')
@@ -410,7 +408,7 @@ def main():
bar.width = 50
bar.elements.remove('steps')
bar.elements += ['transfer', 'time', 'eta', 'speed']
- for x in xrange(0, bar.steps, 1024):
+ for x in range(0, bar.steps, 1024):
bar.render(x)
time.sleep(0.01)
now = datetime.datetime.now()
diff --git a/kallithea/lib/vcs/utils/termcolors.py b/kallithea/lib/vcs/utils/termcolors.py
--- a/kallithea/lib/vcs/utils/termcolors.py
+++ b/kallithea/lib/vcs/utils/termcolors.py
@@ -44,7 +44,7 @@ def colorize(text='', opts=(), **kwargs)
code_list = []
if text == '' and len(opts) == 1 and opts[0] == 'reset':
return '\x1b[%sm' % RESET
- for k, v in kwargs.iteritems():
+ for k, v in kwargs.items():
if k == 'fg':
code_list.append(foreground[v])
elif k == 'bg':
@@ -188,7 +188,7 @@ def parse_color_setting(config_string):
definition['bg'] = colors[-1]
# All remaining instructions are options
- opts = tuple(s for s in styles if s in opt_dict.keys())
+ opts = tuple(s for s in styles if s in opt_dict)
if opts:
definition['opts'] = opts
diff --git a/kallithea/model/comment.py b/kallithea/model/comment.py
--- a/kallithea/model/comment.py
+++ b/kallithea/model/comment.py
@@ -31,7 +31,7 @@ from collections import defaultdict
from tg.i18n import ugettext as _
from kallithea.lib import helpers as h
-from kallithea.lib.utils2 import extract_mentioned_users, safe_unicode
+from kallithea.lib.utils2 import extract_mentioned_users
from kallithea.model.db import ChangesetComment, PullRequest, Repository, User
from kallithea.model.meta import Session
from kallithea.model.notification import NotificationModel
@@ -81,11 +81,10 @@ class ChangesetCommentsModel(object):
repo_name=repo.repo_name,
revision=revision,
anchor='comment-%s' % comment.comment_id)
- subj = safe_unicode(
- h.link_to('Re changeset: %(desc)s %(line)s' %
+ subj = h.link_to(
+ 'Re changeset: %(desc)s %(line)s' %
{'desc': desc, 'line': line},
- comment_url)
- )
+ comment_url)
# get the current participants of this changeset
recipients = _list_changeset_commenters(revision)
# add changeset author if it's known locally
@@ -127,13 +126,12 @@ class ChangesetCommentsModel(object):
h.canonical_hostname()))
comment_url = pull_request.url(canonical=True,
anchor='comment-%s' % comment.comment_id)
- subj = safe_unicode(
- h.link_to('Re pull request %(pr_nice_id)s: %(desc)s %(line)s' %
+ subj = h.link_to(
+ 'Re pull request %(pr_nice_id)s: %(desc)s %(line)s' %
{'desc': desc,
'pr_nice_id': comment.pull_request.nice_id(),
'line': line},
- comment_url)
- )
+ comment_url)
# get the current participants of this pull request
recipients = _list_pull_request_commenters(pull_request)
recipients.append(pull_request.owner)
@@ -257,7 +255,7 @@ class ChangesetCommentsModel(object):
paths = defaultdict(lambda: defaultdict(list))
for co in comments:
paths[co.f_path][co.line_no].append(co)
- return paths.items()
+ return sorted(paths.items())
def _get_comments(self, repo_id, revision=None, pull_request=None,
inline=False, f_path=None, line_no=None):
diff --git a/kallithea/model/db.py b/kallithea/model/db.py
--- a/kallithea/model/db.py
+++ b/kallithea/model/db.py
@@ -25,6 +25,7 @@ Original author and date, and relevant c
:license: GPLv3, see LICENSE.md for more details.
"""
+import base64
import collections
import datetime
import functools
@@ -36,22 +37,20 @@ import traceback
import ipaddr
import sqlalchemy
-from beaker.cache import cache_region, region_invalidate
-from sqlalchemy import *
+from sqlalchemy import Boolean, Column, DateTime, Float, ForeignKey, Index, Integer, LargeBinary, String, Unicode, UnicodeText, UniqueConstraint
from sqlalchemy.ext.hybrid import hybrid_property
from sqlalchemy.orm import class_mapper, joinedload, relationship, validates
from tg.i18n import lazy_ugettext as _
from webob.exc import HTTPNotFound
import kallithea
-from kallithea.lib.caching_query import FromCache
-from kallithea.lib.compat import json
+from kallithea.lib import ext_json
from kallithea.lib.exceptions import DefaultUserException
-from kallithea.lib.utils2 import Optional, aslist, get_changeset_safe, get_clone_url, remove_prefix, safe_int, safe_str, safe_unicode, str2bool, urlreadable
+from kallithea.lib.utils2 import (Optional, ascii_bytes, aslist, get_changeset_safe, get_clone_url, remove_prefix, safe_bytes, safe_int, safe_str, str2bool,
+ urlreadable)
from kallithea.lib.vcs import get_backend
from kallithea.lib.vcs.backends.base import EmptyChangeset
from kallithea.lib.vcs.utils.helpers import get_scm
-from kallithea.lib.vcs.utils.lazy import LazyProperty
from kallithea.model.meta import Base, Session
@@ -62,7 +61,8 @@ log = logging.getLogger(__name__)
# BASE CLASSES
#==============================================================================
-_hash_key = lambda k: hashlib.md5(safe_str(k)).hexdigest()
+def _hash_key(k):
+ return hashlib.md5(safe_bytes(k)).hexdigest()
class BaseDbModel(object):
@@ -73,6 +73,7 @@ class BaseDbModel(object):
@classmethod
def _get_keys(cls):
"""return column names for this model """
+ # Note: not a normal dict - iterator gives "users.firstname", but keys gives "firstname"
return class_mapper(cls).c.keys()
def get_dict(self):
@@ -90,7 +91,7 @@ class BaseDbModel(object):
# update with attributes from __json__
if callable(_json_attr):
_json_attr = _json_attr()
- for k, val in _json_attr.iteritems():
+ for k, val in _json_attr.items():
d[k] = val
return d
@@ -135,8 +136,10 @@ class BaseDbModel(object):
return None
if isinstance(value, cls):
return value
- if isinstance(value, (int, long)) or safe_str(value).isdigit():
+ if isinstance(value, int):
return cls.get(value)
+ if isinstance(value, str) and value.isdigit():
+ return cls.get(int(value))
if callback is not None:
return callback(value)
@@ -163,12 +166,6 @@ class BaseDbModel(object):
Session().delete(obj)
def __repr__(self):
- if hasattr(self, '__unicode__'):
- # python repr needs to return str
- try:
- return safe_str(self.__unicode__())
- except UnicodeDecodeError:
- pass
return '' % (self.__class__.__name__)
@@ -185,9 +182,9 @@ class Setting(Base, BaseDbModel):
)
SETTINGS_TYPES = {
- 'str': safe_str,
+ 'str': safe_bytes,
'int': safe_int,
- 'unicode': safe_unicode,
+ 'unicode': safe_str,
'bool': str2bool,
'list': functools.partial(aslist, sep=',')
}
@@ -205,7 +202,7 @@ class Setting(Base, BaseDbModel):
@validates('_app_settings_value')
def validate_settings_value(self, key, val):
- assert type(val) == unicode
+ assert isinstance(val, str)
return val
@hybrid_property
@@ -218,11 +215,9 @@ class Setting(Base, BaseDbModel):
@app_settings_value.setter
def app_settings_value(self, val):
"""
- Setter that will always make sure we use unicode in app_settings_value
-
- :param val:
+ Setter that will always make sure we use str in app_settings_value
"""
- self._app_settings_value = safe_unicode(val)
+ self._app_settings_value = safe_str(val)
@hybrid_property
def app_settings_type(self):
@@ -232,13 +227,13 @@ class Setting(Base, BaseDbModel):
def app_settings_type(self, val):
if val not in self.SETTINGS_TYPES:
raise Exception('type must be one of %s got %s'
- % (self.SETTINGS_TYPES.keys(), val))
+ % (list(self.SETTINGS_TYPES), val))
self._app_settings_type = val
- def __unicode__(self):
- return u"<%s('%s:%s[%s]')>" % (
+ def __repr__(self):
+ return "<%s %s.%s=%r>" % (
self.__class__.__name__,
- self.app_settings_name, self.app_settings_value, self.app_settings_type
+ self.app_settings_name, self.app_settings_type, self.app_settings_value
)
@classmethod
@@ -281,13 +276,9 @@ class Setting(Base, BaseDbModel):
return res
@classmethod
- def get_app_settings(cls, cache=False):
+ def get_app_settings(cls):
ret = cls.query()
-
- if cache:
- ret = ret.options(FromCache("sql_cache_short", "get_hg_settings"))
-
if ret is None:
raise Exception('Could not get application settings !')
settings = {}
@@ -298,7 +289,7 @@ class Setting(Base, BaseDbModel):
return settings
@classmethod
- def get_auth_settings(cls, cache=False):
+ def get_auth_settings(cls):
ret = cls.query() \
.filter(cls.app_settings_name.startswith('auth_')).all()
fd = {}
@@ -307,7 +298,7 @@ class Setting(Base, BaseDbModel):
return fd
@classmethod
- def get_default_repo_settings(cls, cache=False, strip_prefix=False):
+ def get_default_repo_settings(cls, strip_prefix=False):
ret = cls.query() \
.filter(cls.app_settings_name.startswith('default_')).all()
fd = {}
@@ -328,9 +319,9 @@ class Setting(Base, BaseDbModel):
info = {
'modules': sorted(mods, key=lambda k: k[0].lower()),
'py_version': platform.python_version(),
- 'platform': safe_unicode(platform.platform()),
+ 'platform': platform.platform(),
'kallithea_version': kallithea.__version__,
- 'git_version': safe_unicode(check_git_version()),
+ 'git_version': str(check_git_version()),
'git_path': kallithea.CONFIG.get('git_path')
}
return info
@@ -339,9 +330,7 @@ class Setting(Base, BaseDbModel):
class Ui(Base, BaseDbModel):
__tablename__ = 'ui'
__table_args__ = (
- # FIXME: ui_key as key is wrong and should be removed when the corresponding
- # Ui.get_by_key has been replaced by the composite key
- UniqueConstraint('ui_key'),
+ Index('ui_ui_section_ui_key_idx', 'ui_section', 'ui_key'),
UniqueConstraint('ui_section', 'ui_key'),
_table_args_default_dict,
)
@@ -374,6 +363,7 @@ class Ui(Base, BaseDbModel):
q = cls.query()
q = q.filter(cls.ui_key.in_([cls.HOOK_UPDATE, cls.HOOK_REPO_SIZE]))
q = q.filter(cls.ui_section == 'hooks')
+ q = q.order_by(cls.ui_section, cls.ui_key)
return q.all()
@classmethod
@@ -381,6 +371,7 @@ class Ui(Base, BaseDbModel):
q = cls.query()
q = q.filter(~cls.ui_key.in_([cls.HOOK_UPDATE, cls.HOOK_REPO_SIZE]))
q = q.filter(cls.ui_section == 'hooks')
+ q = q.order_by(cls.ui_section, cls.ui_key)
return q.all()
@classmethod
@@ -394,8 +385,9 @@ class Ui(Base, BaseDbModel):
new_ui.ui_value = val
def __repr__(self):
- return '<%s[%s]%s=>%s]>' % (self.__class__.__name__, self.ui_section,
- self.ui_key, self.ui_value)
+ return '<%s %s.%s=%r>' % (
+ self.__class__.__name__,
+ self.ui_section, self.ui_key, self.ui_value)
class User(Base, BaseDbModel):
@@ -406,7 +398,7 @@ class User(Base, BaseDbModel):
_table_args_default_dict,
)
- DEFAULT_USER = 'default'
+ DEFAULT_USER_NAME = 'default'
DEFAULT_GRAVATAR_URL = 'https://secure.gravatar.com/avatar/{md5email}?d=identicon&s={size}'
# The name of the default auth type in extern_type, 'internal' lives in auth_internal.py
DEFAULT_AUTH_TYPE = 'internal'
@@ -512,7 +504,7 @@ class User(Base, BaseDbModel):
@hybrid_property
def is_default_user(self):
- return self.username == User.DEFAULT_USER
+ return self.username == User.DEFAULT_USER_NAME
@hybrid_property
def user_data(self):
@@ -520,20 +512,19 @@ class User(Base, BaseDbModel):
return {}
try:
- return json.loads(self._user_data)
+ return ext_json.loads(self._user_data)
except TypeError:
return {}
@user_data.setter
def user_data(self, val):
try:
- self._user_data = json.dumps(val)
+ self._user_data = ascii_bytes(ext_json.dumps(val))
except Exception:
log.error(traceback.format_exc())
- def __unicode__(self):
- return u"<%s('id:%s:%s')>" % (self.__class__.__name__,
- self.user_id, self.username)
+ def __repr__(self):
+ return "<%s %s: %r>" % (self.__class__.__name__, self.user_id, self.username)
@classmethod
def guess_instance(cls, value):
@@ -551,7 +542,7 @@ class User(Base, BaseDbModel):
return user
@classmethod
- def get_by_username_or_email(cls, username_or_email, case_insensitive=False, cache=False):
+ def get_by_username_or_email(cls, username_or_email, case_insensitive=True):
"""
For anything that looks like an email address, look up by the email address (matching
case insensitively).
@@ -560,35 +551,24 @@ class User(Base, BaseDbModel):
This assumes no normal username can have '@' symbol.
"""
if '@' in username_or_email:
- return User.get_by_email(username_or_email, cache=cache)
+ return User.get_by_email(username_or_email)
else:
- return User.get_by_username(username_or_email, case_insensitive=case_insensitive, cache=cache)
+ return User.get_by_username(username_or_email, case_insensitive=case_insensitive)
@classmethod
- def get_by_username(cls, username, case_insensitive=False, cache=False):
+ def get_by_username(cls, username, case_insensitive=False):
if case_insensitive:
- q = cls.query().filter(func.lower(cls.username) == func.lower(username))
+ q = cls.query().filter(sqlalchemy.func.lower(cls.username) == sqlalchemy.func.lower(username))
else:
q = cls.query().filter(cls.username == username)
-
- if cache:
- q = q.options(FromCache(
- "sql_cache_short",
- "get_user_%s" % _hash_key(username)
- )
- )
return q.scalar()
@classmethod
- def get_by_api_key(cls, api_key, cache=False, fallback=True):
+ def get_by_api_key(cls, api_key, fallback=True):
if len(api_key) != 40 or not api_key.isalnum():
return None
q = cls.query().filter(cls.api_key == api_key)
-
- if cache:
- q = q.options(FromCache("sql_cache_short",
- "get_api_key_%s" % api_key))
res = q.scalar()
if fallback and not res:
@@ -602,21 +582,13 @@ class User(Base, BaseDbModel):
@classmethod
def get_by_email(cls, email, cache=False):
- q = cls.query().filter(func.lower(cls.email) == func.lower(email))
-
- if cache:
- q = q.options(FromCache("sql_cache_short",
- "get_email_key_%s" % email))
-
+ q = cls.query().filter(sqlalchemy.func.lower(cls.email) == sqlalchemy.func.lower(email))
ret = q.scalar()
if ret is None:
q = UserEmailMap.query()
# try fetching in alternate email map
- q = q.filter(func.lower(UserEmailMap.email) == func.lower(email))
+ q = q.filter(sqlalchemy.func.lower(UserEmailMap.email) == sqlalchemy.func.lower(email))
q = q.options(joinedload(UserEmailMap.user))
- if cache:
- q = q.options(FromCache("sql_cache_short",
- "get_email_map_key_%s" % email))
ret = getattr(q.scalar(), 'user', None)
return ret
@@ -654,8 +626,8 @@ class User(Base, BaseDbModel):
return user
@classmethod
- def get_default_user(cls, cache=False):
- user = User.get_by_username(User.DEFAULT_USER, cache=cache)
+ def get_default_user(cls):
+ user = User.get_by_username(User.DEFAULT_USER_NAME)
if user is None:
raise Exception('Missing default account!')
return user
@@ -772,9 +744,8 @@ class UserIpMap(Base, BaseDbModel):
ip_range=self._get_ip_range(self.ip_addr)
)
- def __unicode__(self):
- return u"<%s('user_id:%s=>%s')>" % (self.__class__.__name__,
- self.user_id, self.ip_addr)
+ def __repr__(self):
+ return "<%s %s: %s>" % (self.__class__.__name__, self.user_id, self.ip_addr)
class UserLog(Base, BaseDbModel):
@@ -792,10 +763,10 @@ class UserLog(Base, BaseDbModel):
action = Column(UnicodeText(), nullable=False)
action_date = Column(DateTime(timezone=False), nullable=False)
- def __unicode__(self):
- return u"<%s('id:%s:%s')>" % (self.__class__.__name__,
- self.repository_name,
- self.action)
+ def __repr__(self):
+ return "<%s %r: %r>" % (self.__class__.__name__,
+ self.repository_name,
+ self.action)
@property
def action_as_day(self):
@@ -834,47 +805,37 @@ class UserGroup(Base, BaseDbModel):
return {}
try:
- return json.loads(self._group_data)
+ return ext_json.loads(self._group_data)
except TypeError:
return {}
@group_data.setter
def group_data(self, val):
try:
- self._group_data = json.dumps(val)
+ self._group_data = ascii_bytes(ext_json.dumps(val))
except Exception:
log.error(traceback.format_exc())
- def __unicode__(self):
- return u"<%s('id:%s:%s')>" % (self.__class__.__name__,
- self.users_group_id,
- self.users_group_name)
+ def __repr__(self):
+ return "<%s %s: %r>" % (self.__class__.__name__,
+ self.users_group_id,
+ self.users_group_name)
@classmethod
def guess_instance(cls, value):
return super(UserGroup, cls).guess_instance(value, UserGroup.get_by_group_name)
@classmethod
- def get_by_group_name(cls, group_name, cache=False,
- case_insensitive=False):
+ def get_by_group_name(cls, group_name, case_insensitive=False):
if case_insensitive:
- q = cls.query().filter(func.lower(cls.users_group_name) == func.lower(group_name))
+ q = cls.query().filter(sqlalchemy.func.lower(cls.users_group_name) == sqlalchemy.func.lower(group_name))
else:
q = cls.query().filter(cls.users_group_name == group_name)
- if cache:
- q = q.options(FromCache(
- "sql_cache_short",
- "get_group_%s" % _hash_key(group_name)
- )
- )
return q.scalar()
@classmethod
- def get(cls, user_group_id, cache=False):
+ def get(cls, user_group_id):
user_group = cls.query()
- if cache:
- user_group = user_group.options(FromCache("sql_cache_short",
- "get_users_group_%s" % user_group_id))
return user_group.get(user_group_id)
def get_api_data(self, with_members=True):
@@ -962,9 +923,9 @@ class Repository(Base, BaseDbModel):
DEFAULT_CLONE_URI = '{scheme}://{user}@{netloc}/{repo}'
DEFAULT_CLONE_SSH = 'ssh://{system_user}@{hostname}/{repo}'
- STATE_CREATED = u'repo_state_created'
- STATE_PENDING = u'repo_state_pending'
- STATE_ERROR = u'repo_state_error'
+ STATE_CREATED = 'repo_state_created'
+ STATE_PENDING = 'repo_state_pending'
+ STATE_ERROR = 'repo_state_error'
repo_id = Column(Integer(), primary_key=True)
repo_name = Column(Unicode(255), nullable=False, unique=True)
@@ -1009,9 +970,9 @@ class Repository(Base, BaseDbModel):
primaryjoin='PullRequest.other_repo_id==Repository.repo_id',
cascade="all, delete-orphan")
- def __unicode__(self):
- return u"<%s('%s:%s')>" % (self.__class__.__name__, self.repo_id,
- safe_unicode(self.repo_name))
+ def __repr__(self):
+ return "<%s %s: %r>" % (self.__class__.__name__,
+ self.repo_id, self.repo_name)
@hybrid_property
def landing_rev(self):
@@ -1033,7 +994,7 @@ class Repository(Base, BaseDbModel):
@hybrid_property
def changeset_cache(self):
try:
- cs_cache = json.loads(self._changeset_cache) # might raise on bad data
+ cs_cache = ext_json.loads(self._changeset_cache) # might raise on bad data
cs_cache['raw_id'] # verify data, raise exception on error
return cs_cache
except (TypeError, KeyError, ValueError):
@@ -1042,7 +1003,7 @@ class Repository(Base, BaseDbModel):
@changeset_cache.setter
def changeset_cache(self, val):
try:
- self._changeset_cache = json.dumps(val)
+ self._changeset_cache = ascii_bytes(ext_json.dumps(val))
except Exception:
log.error(traceback.format_exc())
@@ -1055,15 +1016,11 @@ class Repository(Base, BaseDbModel):
q = super(Repository, cls).query()
if sorted:
- q = q.order_by(func.lower(Repository.repo_name))
+ q = q.order_by(sqlalchemy.func.lower(Repository.repo_name))
return q
@classmethod
- def url_sep(cls):
- return URL_SEP
-
- @classmethod
def normalize_repo_name(cls, repo_name):
"""
Normalizes os specific repo_name to the format internally stored inside
@@ -1072,7 +1029,7 @@ class Repository(Base, BaseDbModel):
:param cls:
:param repo_name:
"""
- return cls.url_sep().join(repo_name.split(os.sep))
+ return URL_SEP.join(repo_name.split(os.sep))
@classmethod
def guess_instance(cls, value):
@@ -1083,7 +1040,7 @@ class Repository(Base, BaseDbModel):
"""Get the repo, defaulting to database case sensitivity.
case_insensitive will be slower and should only be specified if necessary."""
if case_insensitive:
- q = Session().query(cls).filter(func.lower(cls.repo_name) == func.lower(repo_name))
+ q = Session().query(cls).filter(sqlalchemy.func.lower(cls.repo_name) == sqlalchemy.func.lower(repo_name))
else:
q = Session().query(cls).filter(cls.repo_name == repo_name)
q = q.options(joinedload(Repository.fork)) \
@@ -1093,7 +1050,7 @@ class Repository(Base, BaseDbModel):
@classmethod
def get_by_full_path(cls, repo_full_path):
- base_full_path = os.path.realpath(cls.base_path())
+ base_full_path = os.path.realpath(kallithea.CONFIG['base_path'])
repo_full_path = os.path.realpath(repo_full_path)
assert repo_full_path.startswith(base_full_path + os.path.sep)
repo_name = repo_full_path[len(base_full_path) + 1:]
@@ -1104,18 +1061,6 @@ class Repository(Base, BaseDbModel):
def get_repo_forks(cls, repo_id):
return cls.query().filter(Repository.fork_id == repo_id)
- @classmethod
- def base_path(cls):
- """
- Returns base path where all repos are stored
-
- :param cls:
- """
- q = Session().query(Ui) \
- .filter(Ui.ui_key == cls.url_sep())
- q = q.options(FromCache("sql_cache_short", "repository_repo_path"))
- return q.one().ui_value
-
@property
def forks(self):
"""
@@ -1132,7 +1077,7 @@ class Repository(Base, BaseDbModel):
@property
def just_name(self):
- return self.repo_name.split(Repository.url_sep())[-1]
+ return self.repo_name.split(URL_SEP)[-1]
@property
def groups_with_parents(self):
@@ -1145,35 +1090,18 @@ class Repository(Base, BaseDbModel):
groups.reverse()
return groups
- @LazyProperty
- def repo_path(self):
- """
- Returns base full path for that repository means where it actually
- exists on a filesystem
- """
- q = Session().query(Ui).filter(Ui.ui_key ==
- Repository.url_sep())
- q = q.options(FromCache("sql_cache_short", "repository_repo_path"))
- return q.one().ui_value
-
@property
def repo_full_path(self):
- p = [self.repo_path]
+ """
+ Returns base full path for the repository - where it actually
+ exists on a filesystem.
+ """
+ p = [kallithea.CONFIG['base_path']]
# we need to split the name by / since this is how we store the
# names in the database, but that eventually needs to be converted
# into a valid system path
- p += self.repo_name.split(Repository.url_sep())
- return os.path.join(*map(safe_unicode, p))
-
- @property
- def cache_keys(self):
- """
- Returns associated cache keys for that repo
- """
- return CacheInvalidation.query() \
- .filter(CacheInvalidation.cache_args == self.repo_name) \
- .order_by(CacheInvalidation.cache_key) \
- .all()
+ p += self.repo_name.split(URL_SEP)
+ return os.path.join(*p)
def get_new_name(self, repo_name):
"""
@@ -1182,7 +1110,7 @@ class Repository(Base, BaseDbModel):
:param group_name:
"""
path_prefix = self.group.full_path_splitted if self.group else []
- return Repository.url_sep().join(path_prefix + [repo_name])
+ return URL_SEP.join(path_prefix + [repo_name])
@property
def _ui(self):
@@ -1190,7 +1118,7 @@ class Repository(Base, BaseDbModel):
Creates an db based ui object for this repository
"""
from kallithea.lib.utils import make_ui
- return make_ui(clear_session=False)
+ return make_ui()
@classmethod
def is_valid(cls, repo_name):
@@ -1202,7 +1130,7 @@ class Repository(Base, BaseDbModel):
"""
from kallithea.lib.utils import is_valid_repo
- return is_valid_repo(repo_name, cls.base_path())
+ return is_valid_repo(repo_name, kallithea.CONFIG['base_path'])
def get_api_data(self, with_revision_names=False,
with_pullrequests=False):
@@ -1397,47 +1325,34 @@ class Repository(Base, BaseDbModel):
def set_invalidate(self):
"""
- Mark caches of this repo as invalid.
+ Flush SA session caches of instances of on disk repo.
"""
- CacheInvalidation.set_invalidate(self.repo_name)
+ try:
+ del self._scm_instance
+ except AttributeError:
+ pass
- _scm_instance = None
+ _scm_instance = None # caching inside lifetime of SA session
@property
def scm_instance(self):
if self._scm_instance is None:
- self._scm_instance = self.scm_instance_cached()
+ return self.scm_instance_no_cache() # will populate self._scm_instance
return self._scm_instance
- def scm_instance_cached(self, valid_cache_keys=None):
- @cache_region('long_term', 'scm_instance_cached')
- def _c(repo_name): # repo_name is just for the cache key
- log.debug('Creating new %s scm_instance and populating cache', repo_name)
- return self.scm_instance_no_cache()
- rn = self.repo_name
-
- valid = CacheInvalidation.test_and_set_valid(rn, None, valid_cache_keys=valid_cache_keys)
- if not valid:
- log.debug('Cache for %s invalidated, getting new object', rn)
- region_invalidate(_c, None, 'scm_instance_cached', rn)
- else:
- log.debug('Trying to get scm_instance of %s from cache', rn)
- return _c(rn)
-
def scm_instance_no_cache(self):
- repo_full_path = safe_str(self.repo_full_path)
+ repo_full_path = self.repo_full_path
alias = get_scm(repo_full_path)[0]
log.debug('Creating instance of %s repository from %s',
alias, self.repo_full_path)
backend = get_backend(alias)
if alias == 'hg':
- repo = backend(repo_full_path, create=False,
- baseui=self._ui)
+ self._scm_instance = backend(repo_full_path, create=False, baseui=self._ui)
else:
- repo = backend(repo_full_path, create=False)
+ self._scm_instance = backend(repo_full_path, create=False)
- return repo
+ return self._scm_instance
def __json__(self):
return dict(
@@ -1476,7 +1391,7 @@ class RepoGroup(Base, BaseDbModel):
q = super(RepoGroup, cls).query()
if sorted:
- q = q.order_by(func.lower(RepoGroup.group_name))
+ q = q.order_by(sqlalchemy.func.lower(RepoGroup.group_name))
return q
@@ -1484,16 +1399,16 @@ class RepoGroup(Base, BaseDbModel):
self.group_name = group_name
self.parent_group = parent_group
- def __unicode__(self):
- return u"<%s('id:%s:%s')>" % (self.__class__.__name__, self.group_id,
- self.group_name)
+ def __repr__(self):
+ return "<%s %s: %s>" % (self.__class__.__name__,
+ self.group_id, self.group_name)
@classmethod
def _generate_choice(cls, repo_group):
"""Return tuple with group_id and name as html literal"""
from webhelpers2.html import literal
if repo_group is None:
- return (-1, u'-- %s --' % _('top level'))
+ return (-1, '-- %s --' % _('top level'))
return repo_group.group_id, literal(cls.SEP.join(repo_group.full_path_splitted))
@classmethod
@@ -1503,28 +1418,18 @@ class RepoGroup(Base, BaseDbModel):
key=lambda c: c[1].split(cls.SEP))
@classmethod
- def url_sep(cls):
- return URL_SEP
-
- @classmethod
def guess_instance(cls, value):
return super(RepoGroup, cls).guess_instance(value, RepoGroup.get_by_group_name)
@classmethod
- def get_by_group_name(cls, group_name, cache=False, case_insensitive=False):
+ def get_by_group_name(cls, group_name, case_insensitive=False):
group_name = group_name.rstrip('/')
if case_insensitive:
gr = cls.query() \
- .filter(func.lower(cls.group_name) == func.lower(group_name))
+ .filter(sqlalchemy.func.lower(cls.group_name) == sqlalchemy.func.lower(group_name))
else:
gr = cls.query() \
.filter(cls.group_name == group_name)
- if cache:
- gr = gr.options(FromCache(
- "sql_cache_short",
- "get_group_%s" % _hash_key(group_name)
- )
- )
return gr.scalar()
@property
@@ -1544,7 +1449,7 @@ class RepoGroup(Base, BaseDbModel):
@property
def name(self):
- return self.group_name.split(RepoGroup.url_sep())[-1]
+ return self.group_name.split(URL_SEP)[-1]
@property
def full_path(self):
@@ -1552,7 +1457,7 @@ class RepoGroup(Base, BaseDbModel):
@property
def full_path_splitted(self):
- return self.group_name.split(RepoGroup.url_sep())
+ return self.group_name.split(URL_SEP)
@property
def repositories(self):
@@ -1607,7 +1512,7 @@ class RepoGroup(Base, BaseDbModel):
"""
path_prefix = (self.parent_group.full_path_splitted if
self.parent_group else [])
- return RepoGroup.url_sep().join(path_prefix + [group_name])
+ return URL_SEP.join(path_prefix + [group_name])
def get_api_data(self):
"""
@@ -1731,8 +1636,8 @@ class Permission(Base, BaseDbModel):
permission_id = Column(Integer(), primary_key=True)
permission_name = Column(String(255), nullable=False)
- def __unicode__(self):
- return u"<%s('%s:%s')>" % (
+ def __repr__(self):
+ return "<%s %s: %r>" % (
self.__class__.__name__, self.permission_id, self.permission_name
)
@@ -1746,27 +1651,27 @@ class Permission(Base, BaseDbModel):
@classmethod
def get_default_perms(cls, default_user_id):
- q = Session().query(UserRepoToPerm, Repository, cls) \
- .join((Repository, UserRepoToPerm.repository_id == Repository.repo_id)) \
- .join((cls, UserRepoToPerm.permission_id == cls.permission_id)) \
+ q = Session().query(UserRepoToPerm) \
+ .options(joinedload(UserRepoToPerm.repository)) \
+ .options(joinedload(UserRepoToPerm.permission)) \
.filter(UserRepoToPerm.user_id == default_user_id)
return q.all()
@classmethod
def get_default_group_perms(cls, default_user_id):
- q = Session().query(UserRepoGroupToPerm, RepoGroup, cls) \
- .join((RepoGroup, UserRepoGroupToPerm.group_id == RepoGroup.group_id)) \
- .join((cls, UserRepoGroupToPerm.permission_id == cls.permission_id)) \
+ q = Session().query(UserRepoGroupToPerm) \
+ .options(joinedload(UserRepoGroupToPerm.group)) \
+ .options(joinedload(UserRepoGroupToPerm.permission)) \
.filter(UserRepoGroupToPerm.user_id == default_user_id)
return q.all()
@classmethod
def get_default_user_group_perms(cls, default_user_id):
- q = Session().query(UserUserGroupToPerm, UserGroup, cls) \
- .join((UserGroup, UserUserGroupToPerm.user_group_id == UserGroup.users_group_id)) \
- .join((cls, UserUserGroupToPerm.permission_id == cls.permission_id)) \
+ q = Session().query(UserUserGroupToPerm) \
+ .options(joinedload(UserUserGroupToPerm.user_group)) \
+ .options(joinedload(UserUserGroupToPerm.permission)) \
.filter(UserUserGroupToPerm.user_id == default_user_id)
return q.all()
@@ -1797,8 +1702,9 @@ class UserRepoToPerm(Base, BaseDbModel):
Session().add(n)
return n
- def __unicode__(self):
- return u'<%s => %s >' % (self.user, self.repository)
+ def __repr__(self):
+ return '<%s %s at %s: %s>' % (
+ self.__class__.__name__, self.user, self.repository, self.permission)
class UserUserGroupToPerm(Base, BaseDbModel):
@@ -1826,8 +1732,9 @@ class UserUserGroupToPerm(Base, BaseDbMo
Session().add(n)
return n
- def __unicode__(self):
- return u'<%s => %s >' % (self.user, self.user_group)
+ def __repr__(self):
+ return '<%s %s at %s: %s>' % (
+ self.__class__.__name__, self.user, self.user_group, self.permission)
class UserToPerm(Base, BaseDbModel):
@@ -1844,8 +1751,9 @@ class UserToPerm(Base, BaseDbModel):
user = relationship('User')
permission = relationship('Permission')
- def __unicode__(self):
- return u'<%s => %s >' % (self.user, self.permission)
+ def __repr__(self):
+ return '<%s %s: %s>' % (
+ self.__class__.__name__, self.user, self.permission)
class UserGroupRepoToPerm(Base, BaseDbModel):
@@ -1873,8 +1781,9 @@ class UserGroupRepoToPerm(Base, BaseDbMo
Session().add(n)
return n
- def __unicode__(self):
- return u' %s >' % (self.users_group, self.repository)
+ def __repr__(self):
+ return '<%s %s at %s: %s>' % (
+ self.__class__.__name__, self.users_group, self.repository, self.permission)
class UserGroupUserGroupToPerm(Base, BaseDbModel):
@@ -1902,8 +1811,9 @@ class UserGroupUserGroupToPerm(Base, Bas
Session().add(n)
return n
- def __unicode__(self):
- return u' %s >' % (self.target_user_group, self.user_group)
+ def __repr__(self):
+ return '<%s %s at %s: %s>' % (
+ self.__class__.__name__, self.user_group, self.target_user_group, self.permission)
class UserGroupToPerm(Base, BaseDbModel):
@@ -2006,137 +1916,13 @@ class UserFollowing(Base, BaseDbModel):
user = relationship('User', primaryjoin='User.user_id==UserFollowing.user_id')
follows_user = relationship('User', primaryjoin='User.user_id==UserFollowing.follows_user_id')
- follows_repository = relationship('Repository', order_by=lambda: func.lower(Repository.repo_name))
+ follows_repository = relationship('Repository', order_by=lambda: sqlalchemy.func.lower(Repository.repo_name))
@classmethod
def get_repo_followers(cls, repo_id):
return cls.query().filter(cls.follows_repository_id == repo_id)
-class CacheInvalidation(Base, BaseDbModel):
- __tablename__ = 'cache_invalidation'
- __table_args__ = (
- Index('key_idx', 'cache_key'),
- _table_args_default_dict,
- )
-
- # cache_id, not used
- cache_id = Column(Integer(), primary_key=True)
- # cache_key as created by _get_cache_key
- cache_key = Column(Unicode(255), nullable=False, unique=True)
- # cache_args is a repo_name
- cache_args = Column(Unicode(255), nullable=False)
- # instance sets cache_active True when it is caching, other instances set
- # cache_active to False to indicate that this cache is invalid
- cache_active = Column(Boolean(), nullable=False, default=False)
-
- def __init__(self, cache_key, repo_name=''):
- self.cache_key = cache_key
- self.cache_args = repo_name
- self.cache_active = False
-
- def __unicode__(self):
- return u"<%s('%s:%s[%s]')>" % (
- self.__class__.__name__,
- self.cache_id, self.cache_key, self.cache_active)
-
- def _cache_key_partition(self):
- prefix, repo_name, suffix = self.cache_key.partition(self.cache_args)
- return prefix, repo_name, suffix
-
- def get_prefix(self):
- """
- get prefix that might have been used in _get_cache_key to
- generate self.cache_key. Only used for informational purposes
- in repo_edit.html.
- """
- # prefix, repo_name, suffix
- return self._cache_key_partition()[0]
-
- def get_suffix(self):
- """
- get suffix that might have been used in _get_cache_key to
- generate self.cache_key. Only used for informational purposes
- in repo_edit.html.
- """
- # prefix, repo_name, suffix
- return self._cache_key_partition()[2]
-
- @classmethod
- def clear_cache(cls):
- """
- Delete all cache keys from database.
- Should only be run when all instances are down and all entries thus stale.
- """
- cls.query().delete()
- Session().commit()
-
- @classmethod
- def _get_cache_key(cls, key):
- """
- Wrapper for generating a unique cache key for this instance and "key".
- key must / will start with a repo_name which will be stored in .cache_args .
- """
- prefix = kallithea.CONFIG.get('instance_id', '')
- return "%s%s" % (prefix, key)
-
- @classmethod
- def set_invalidate(cls, repo_name):
- """
- Mark all caches of a repo as invalid in the database.
- """
- inv_objs = Session().query(cls).filter(cls.cache_args == repo_name).all()
- log.debug('for repo %s got %s invalidation objects',
- safe_str(repo_name), inv_objs)
-
- for inv_obj in inv_objs:
- log.debug('marking %s key for invalidation based on repo_name=%s',
- inv_obj, safe_str(repo_name))
- Session().delete(inv_obj)
- Session().commit()
-
- @classmethod
- def test_and_set_valid(cls, repo_name, kind, valid_cache_keys=None):
- """
- Mark this cache key as active and currently cached.
- Return True if the existing cache registration still was valid.
- Return False to indicate that it had been invalidated and caches should be refreshed.
- """
-
- key = (repo_name + '_' + kind) if kind else repo_name
- cache_key = cls._get_cache_key(key)
-
- if valid_cache_keys and cache_key in valid_cache_keys:
- return True
-
- inv_obj = cls.query().filter(cls.cache_key == cache_key).scalar()
- if inv_obj is None:
- inv_obj = cls(cache_key, repo_name)
- Session().add(inv_obj)
- elif inv_obj.cache_active:
- return True
- inv_obj.cache_active = True
- try:
- Session().commit()
- except sqlalchemy.exc.IntegrityError:
- log.error('commit of CacheInvalidation failed - retrying')
- Session().rollback()
- inv_obj = cls.query().filter(cls.cache_key == cache_key).scalar()
- if inv_obj is None:
- log.error('failed to create CacheInvalidation entry')
- # TODO: fail badly?
- # else: TOCTOU - another thread added the key at the same time; no further action required
- return False
-
- @classmethod
- def get_valid_cache_keys(cls):
- """
- Return opaque object with information of which caches still are valid
- and can be used without checking for invalidation.
- """
- return set(inv_obj.cache_key for inv_obj in cls.query().filter(cls.cache_active).all())
-
-
class ChangesetComment(Base, BaseDbModel):
__tablename__ = 'changeset_comments'
__table_args__ = (
@@ -2225,8 +2011,8 @@ class ChangesetStatus(Base, BaseDbModel)
comment = relationship('ChangesetComment')
pull_request = relationship('PullRequest')
- def __unicode__(self):
- return u"<%s('%s:%s')>" % (
+ def __repr__(self):
+ return "<%s %r by %r>" % (
self.__class__.__name__,
self.status, self.author
)
@@ -2256,8 +2042,8 @@ class PullRequest(Base, BaseDbModel):
)
# values for .status
- STATUS_NEW = u'new'
- STATUS_CLOSED = u'closed'
+ STATUS_NEW = 'new'
+ STATUS_CLOSED = 'closed'
pull_request_id = Column(Integer(), primary_key=True)
title = Column(Unicode(255), nullable=False)
@@ -2278,7 +2064,7 @@ class PullRequest(Base, BaseDbModel):
@revisions.setter
def revisions(self, val):
- self._revisions = safe_unicode(':'.join(val))
+ self._revisions = ':'.join(val)
@property
def org_ref_parts(self):
@@ -2426,9 +2212,9 @@ class Gist(Base, BaseDbModel):
_table_args_default_dict,
)
- GIST_PUBLIC = u'public'
- GIST_PRIVATE = u'private'
- DEFAULT_FILENAME = u'gistfile1.txt'
+ GIST_PUBLIC = 'public'
+ GIST_PRIVATE = 'private'
+ DEFAULT_FILENAME = 'gistfile1.txt'
gist_id = Column(Integer(), primary_key=True)
gist_access_id = Column(Unicode(250), nullable=False)
@@ -2446,7 +2232,9 @@ class Gist(Base, BaseDbModel):
return (self.gist_expires != -1) & (time.time() > self.gist_expires)
def __repr__(self):
- return '' % (self.gist_type, self.gist_access_id)
+ return "<%s %s %s>" % (
+ self.__class__.__name__,
+ self.gist_type, self.gist_access_id)
@classmethod
def guess_instance(cls, value):
@@ -2471,19 +2259,6 @@ class Gist(Base, BaseDbModel):
import kallithea.lib.helpers as h
return h.canonical_url('gist', gist_id=self.gist_access_id)
- @classmethod
- def base_path(cls):
- """
- Returns base path where all gists are stored
-
- :param cls:
- """
- from kallithea.model.gist import GIST_STORE_LOC
- q = Session().query(Ui) \
- .filter(Ui.ui_key == URL_SEP)
- q = q.options(FromCache("sql_cache_short", "repository_repo_path"))
- return os.path.join(q.one().ui_value, GIST_STORE_LOC)
-
def get_api_data(self):
"""
Common function for generating gist related data for API
@@ -2505,14 +2280,15 @@ class Gist(Base, BaseDbModel):
)
data.update(self.get_api_data())
return data
+
## SCM functions
@property
def scm_instance(self):
from kallithea.lib.vcs import get_repo
- base_path = self.base_path()
- return get_repo(os.path.join(*map(safe_str,
- [base_path, self.gist_access_id])))
+ from kallithea.model.gist import GIST_STORE_LOC
+ gist_base_path = os.path.join(kallithea.CONFIG['base_path'], GIST_STORE_LOC)
+ return get_repo(os.path.join(gist_base_path, self.gist_access_id))
class UserSshKeys(Base, BaseDbModel):
@@ -2543,5 +2319,5 @@ class UserSshKeys(Base, BaseDbModel):
# the full public key is too long to be suitable as database key - instead,
# use fingerprints similar to 'ssh-keygen -E sha256 -lf ~/.ssh/id_rsa.pub'
self._public_key = full_key
- enc_key = full_key.split(" ")[1]
- self.fingerprint = hashlib.sha256(enc_key.decode('base64')).digest().encode('base64').replace('\n', '').rstrip('=')
+ enc_key = safe_bytes(full_key.split(" ")[1])
+ self.fingerprint = base64.b64encode(hashlib.sha256(base64.b64decode(enc_key)).digest()).replace(b'\n', b'').rstrip(b'=').decode()
diff --git a/kallithea/model/forms.py b/kallithea/model/forms.py
--- a/kallithea/model/forms.py
+++ b/kallithea/model/forms.py
@@ -238,7 +238,7 @@ def PasswordResetConfirmationForm():
return _PasswordResetConfirmationForm
-def RepoForm(edit=False, old_data=None, supported_backends=BACKENDS.keys(),
+def RepoForm(edit=False, old_data=None, supported_backends=BACKENDS,
repo_groups=None, landing_revs=None):
old_data = old_data or {}
repo_groups = repo_groups or []
@@ -315,7 +315,7 @@ def RepoFieldForm():
return _RepoFieldForm
-def RepoForkForm(edit=False, old_data=None, supported_backends=BACKENDS.keys(),
+def RepoForkForm(edit=False, old_data=None, supported_backends=BACKENDS,
repo_groups=None, landing_revs=None):
old_data = old_data or {}
repo_groups = repo_groups or []
@@ -435,7 +435,7 @@ def CustomDefaultPermissionsForm():
return _CustomDefaultPermissionsForm
-def DefaultsForm(edit=False, old_data=None, supported_backends=BACKENDS.keys()):
+def DefaultsForm(edit=False, old_data=None, supported_backends=BACKENDS):
class _DefaultsForm(formencode.Schema):
allow_extra_fields = True
filter_extra_fields = True
@@ -558,11 +558,11 @@ def GistForm(lifetime_options):
filename = All(v.BasePath()(),
v.UnicodeString(strip=True, required=False))
- description = v.UnicodeString(required=False, if_missing=u'')
+ description = v.UnicodeString(required=False, if_missing='')
lifetime = v.OneOf(lifetime_options)
mimetype = v.UnicodeString(required=False, if_missing=None)
content = v.UnicodeString(required=True, not_empty=True)
- public = v.UnicodeString(required=False, if_missing=u'')
- private = v.UnicodeString(required=False, if_missing=u'')
+ public = v.UnicodeString(required=False, if_missing='')
+ private = v.UnicodeString(required=False, if_missing='')
return _GistForm
diff --git a/kallithea/model/gist.py b/kallithea/model/gist.py
--- a/kallithea/model/gist.py
+++ b/kallithea/model/gist.py
@@ -32,8 +32,8 @@ import shutil
import time
import traceback
-from kallithea.lib.compat import json
-from kallithea.lib.utils2 import AttributeDict, safe_int, safe_unicode, time_to_datetime
+from kallithea.lib import ext_json
+from kallithea.lib.utils2 import AttributeDict, ascii_bytes, safe_int, time_to_datetime
from kallithea.model.db import Gist, Session, User
from kallithea.model.repo import RepoModel
from kallithea.model.scm import ScmModel
@@ -45,12 +45,12 @@ GIST_STORE_LOC = '.rc_gist_store'
GIST_METADATA_FILE = '.rc_gist_metadata'
-def make_gist_id():
+def make_gist_access_id():
"""Generate a random, URL safe, almost certainly unique gist identifier."""
rnd = random.SystemRandom() # use cryptographically secure system PRNG
alphabet = '23456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghjklmnpqrstuvwxyz'
length = 20
- return u''.join(rnd.choice(alphabet) for _ in xrange(length))
+ return ''.join(rnd.choice(alphabet) for _ in range(length))
class GistModel(object):
@@ -82,7 +82,7 @@ class GistModel(object):
'gist_updated': time.time(),
}
with open(os.path.join(repo.path, '.hg', GIST_METADATA_FILE), 'wb') as f:
- f.write(json.dumps(metadata))
+ f.write(ascii_bytes(ext_json.dumps(metadata)))
def get_gist(self, gist):
return Gist.guess_instance(gist)
@@ -108,7 +108,7 @@ class GistModel(object):
:param lifetime: in minutes, -1 == forever
"""
owner = User.guess_instance(owner)
- gist_id = make_gist_id()
+ gist_access_id = make_gist_access_id()
lifetime = safe_int(lifetime, -1)
gist_expires = time.time() + (lifetime * 60) if lifetime != -1 else -1
log.debug('set GIST expiration date to: %s',
@@ -117,21 +117,19 @@ class GistModel(object):
# create the Database version
gist = Gist()
gist.gist_description = description
- gist.gist_access_id = gist_id
+ gist.gist_access_id = gist_access_id
gist.owner_id = owner.user_id
gist.gist_expires = gist_expires
- gist.gist_type = safe_unicode(gist_type)
+ gist.gist_type = gist_type
Session().add(gist)
Session().flush() # make database assign gist.gist_id
if gist_type == Gist.GIST_PUBLIC:
# use DB ID for easy to use GIST ID
- gist_id = safe_unicode(gist.gist_id)
- gist.gist_access_id = gist_id
+ gist.gist_access_id = str(gist.gist_id)
- gist_repo_path = os.path.join(GIST_STORE_LOC, gist_id)
- log.debug('Creating new %s GIST repo in %s', gist_type, gist_repo_path)
+ log.debug('Creating new %s GIST repo %s', gist_type, gist.gist_access_id)
repo = RepoModel()._create_filesystem_repo(
- repo_name=gist_id, repo_type='hg', repo_group=GIST_STORE_LOC)
+ repo_name=gist.gist_access_id, repo_type='hg', repo_group=GIST_STORE_LOC)
processed_mapping = {}
for filename in gist_mapping:
@@ -155,7 +153,7 @@ class GistModel(object):
# fake Kallithea Repository object
fake_repo = AttributeDict(dict(
- repo_name=gist_repo_path,
+ repo_name=os.path.join(GIST_STORE_LOC, gist.gist_access_id),
scm_instance_no_cache=lambda: repo,
))
ScmModel().create_nodes(
@@ -219,7 +217,7 @@ class GistModel(object):
# fake Kallithea Repository object
fake_repo = AttributeDict(dict(
- repo_name=gist_repo.path,
+ repo_name=os.path.join(GIST_STORE_LOC, gist.gist_access_id),
scm_instance_no_cache=lambda: gist_repo,
))
diff --git a/kallithea/model/meta.py b/kallithea/model/meta.py
--- a/kallithea/model/meta.py
+++ b/kallithea/model/meta.py
@@ -18,8 +18,6 @@ from beaker import cache
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import scoped_session, sessionmaker
-from kallithea.lib import caching_query
-
# Beaker CacheManager. A home base for cache configurations.
cache_manager = cache.CacheManager()
@@ -29,18 +27,13 @@ __all__ = ['Base', 'Session']
#
# SQLAlchemy session manager.
#
-session_factory = sessionmaker(
- query_cls=caching_query.query_callable(cache_manager),
- expire_on_commit=True)
+session_factory = sessionmaker(expire_on_commit=True)
Session = scoped_session(session_factory)
# The base class for declarative schemas in db.py
# Engine is injected when model.__init__.init_model() sets meta.Base.metadata.bind
Base = declarative_base()
-# to use cache use this in query:
-# .options(FromCache("sqlalchemy_cache_type", "cachekey"))
-
# Define naming conventions for foreign keys, primary keys, indexes,
# check constraints, and unique constraints, respectively.
diff --git a/kallithea/model/notification.py b/kallithea/model/notification.py
--- a/kallithea/model/notification.py
+++ b/kallithea/model/notification.py
@@ -33,9 +33,7 @@ from tg import app_globals
from tg import tmpl_context as c
from tg.i18n import ugettext as _
-import kallithea
from kallithea.lib import helpers as h
-from kallithea.lib.utils2 import safe_unicode
from kallithea.model.db import User
@@ -44,12 +42,12 @@ log = logging.getLogger(__name__)
class NotificationModel(object):
- TYPE_CHANGESET_COMMENT = u'cs_comment'
- TYPE_MESSAGE = u'message'
- TYPE_MENTION = u'mention' # not used
- TYPE_REGISTRATION = u'registration'
- TYPE_PULL_REQUEST = u'pull_request'
- TYPE_PULL_REQUEST_COMMENT = u'pull_request_comment'
+ TYPE_CHANGESET_COMMENT = 'cs_comment'
+ TYPE_MESSAGE = 'message'
+ TYPE_MENTION = 'mention' # not used
+ TYPE_REGISTRATION = 'registration'
+ TYPE_PULL_REQUEST = 'pull_request'
+ TYPE_PULL_REQUEST_COMMENT = 'pull_request_comment'
def create(self, created_by, subject, body, recipients=None,
type_=TYPE_MESSAGE, with_email=True,
@@ -134,7 +132,8 @@ class NotificationModel(object):
# send email with notification to all other participants
for rec in rec_objs:
tasks.send_email([rec.email], email_subject, email_txt_body,
- email_html_body, headers, author=created_by_obj)
+ email_html_body, headers,
+ from_name=created_by_obj.full_name_or_username)
class EmailNotificationModel(object):
@@ -150,7 +149,6 @@ class EmailNotificationModel(object):
def __init__(self):
super(EmailNotificationModel, self).__init__()
- self._template_root = kallithea.CONFIG['paths']['templates'][0]
self._tmpl_lookup = app_globals.mako_lookup
self.email_types = {
self.TYPE_CHANGESET_COMMENT: 'changeset_comment',
@@ -178,14 +176,21 @@ class EmailNotificationModel(object):
try:
subj = tmpl % kwargs
except KeyError as e:
- log.error('error generating email subject for %r from %s: %s', type_, ','.join(self._subj_map.keys()), e)
+ log.error('error generating email subject for %r from %s: %s', type_, ', '.join(self._subj_map), e)
raise
- l = [safe_unicode(x) for x in [kwargs.get('status_change'), kwargs.get('closing_pr') and _('Closing')] if x]
- if l:
+ # gmail doesn't do proper threading but will ignore leading square
+ # bracket content ... so that is where we put status info
+ bracket_tags = []
+ status_change = kwargs.get('status_change')
+ if status_change:
+ bracket_tags.append(str(status_change)) # apply str to evaluate LazyString before .join
+ if kwargs.get('closing_pr'):
+ bracket_tags.append(_('Closing'))
+ if bracket_tags:
if subj.startswith('['):
- subj = '[' + ', '.join(l) + ': ' + subj[1:]
+ subj = '[' + ', '.join(bracket_tags) + ': ' + subj[1:]
else:
- subj = '[' + ', '.join(l) + '] ' + subj
+ subj = '[' + ', '.join(bracket_tags) + '] ' + subj
return subj
def get_email_tmpl(self, type_, content_type, **kwargs):
diff --git a/kallithea/model/permission.py b/kallithea/model/permission.py
--- a/kallithea/model/permission.py
+++ b/kallithea/model/permission.py
@@ -73,10 +73,8 @@ class PermissionModel(object):
return '.'.join(perm_name.split('.')[:1])
perms = UserToPerm.query().filter(UserToPerm.user == user).all()
- defined_perms_groups = map(_get_group,
- (x.permission.permission_name for x in perms))
+ defined_perms_groups = set(_get_group(x.permission.permission_name) for x in perms)
log.debug('GOT ALREADY DEFINED:%s', perms)
- DEFAULT_PERMS = Permission.DEFAULT_USER_PERMISSIONS
if force:
for perm in perms:
@@ -85,7 +83,7 @@ class PermissionModel(object):
defined_perms_groups = []
# For every default permission that needs to be created, we check if
# its group is already defined. If it's not, we create default permission.
- for perm_name in DEFAULT_PERMS:
+ for perm_name in Permission.DEFAULT_USER_PERMISSIONS:
gr = _get_group(perm_name)
if gr not in defined_perms_groups:
log.debug('GR:%s not found, creating permission %s',
diff --git a/kallithea/model/pull_request.py b/kallithea/model/pull_request.py
--- a/kallithea/model/pull_request.py
+++ b/kallithea/model/pull_request.py
@@ -33,7 +33,7 @@ from tg import request
from tg.i18n import ugettext as _
from kallithea.lib import helpers as h
-from kallithea.lib.utils2 import extract_mentioned_users, safe_str, safe_unicode
+from kallithea.lib.utils2 import ascii_bytes, extract_mentioned_users
from kallithea.model.db import ChangesetStatus, PullRequest, PullRequestReviewer, User
from kallithea.model.meta import Session
from kallithea.model.notification import NotificationModel
@@ -68,14 +68,12 @@ class PullRequestModel(object):
threading = ['%s-pr-%s@%s' % (pr.other_repo.repo_name,
pr.pull_request_id,
h.canonical_hostname())]
- subject = safe_unicode(
- h.link_to(
- _('%(user)s wants you to review pull request %(pr_nice_id)s: %(pr_title)s') %
+ subject = h.link_to(
+ _('%(user)s wants you to review pull request %(pr_nice_id)s: %(pr_title)s') %
{'user': user.username,
'pr_title': pr.title,
'pr_nice_id': pr.nice_id()},
- pr_url)
- )
+ pr_url)
body = pr.description
_org_ref_type, org_ref_name, _org_rev = pr.org_ref.split(':')
_other_ref_type, other_ref_name, _other_rev = pr.other_ref.split(':')
@@ -261,13 +259,13 @@ class CreatePullRequestAction(object):
if self.org_repo.scm_instance.alias == 'git':
# create a ref under refs/pull/ so that commits don't get garbage-collected
- self.org_repo.scm_instance._repo["refs/pull/%d/head" % pr.pull_request_id] = safe_str(self.org_rev)
+ self.org_repo.scm_instance._repo[b"refs/pull/%d/head" % pr.pull_request_id] = ascii_bytes(self.org_rev)
# reset state to under-review
from kallithea.model.changeset_status import ChangesetStatusModel
from kallithea.model.comment import ChangesetCommentsModel
comment = ChangesetCommentsModel().create(
- text=u'',
+ text='',
repo=self.org_repo,
author=created_by,
pull_request=pr,
@@ -362,11 +360,11 @@ class CreatePullRequestIterationAction(o
infos.append(_('No changes found on %s %s since previous iteration.') % (org_ref_type, org_ref_name))
# TODO: fail?
- try:
- title, old_v = re.match(r'(.*)\(v(\d+)\)\s*$', title).groups()
- v = int(old_v) + 1
- except (AttributeError, ValueError):
- v = 2
+ v = 2
+ m = re.match(r'(.*)\(v(\d+)\)\s*$', title)
+ if m is not None:
+ title = m.group(1)
+ v = int(m.group(2)) + 1
self.create_action.title = '%s (v%s)' % (title.strip(), v)
# using a mail-like separator, insert new iteration info in description with latest first
diff --git a/kallithea/model/repo.py b/kallithea/model/repo.py
--- a/kallithea/model/repo.py
+++ b/kallithea/model/repo.py
@@ -35,14 +35,13 @@ from datetime import datetime
import kallithea.lib.utils2
from kallithea.lib import helpers as h
from kallithea.lib.auth import HasRepoPermissionLevel, HasUserGroupPermissionLevel
-from kallithea.lib.caching_query import FromCache
from kallithea.lib.exceptions import AttachedForksError
from kallithea.lib.hooks import log_delete_repository
from kallithea.lib.utils import is_valid_repo_uri, make_ui
-from kallithea.lib.utils2 import LazyProperty, get_current_authuser, obfuscate_url_pw, remove_prefix, safe_str, safe_unicode
+from kallithea.lib.utils2 import LazyProperty, get_current_authuser, obfuscate_url_pw, remove_prefix
from kallithea.lib.vcs.backends import get_backend
-from kallithea.model.db import (
- Permission, RepoGroup, Repository, RepositoryField, Session, Statistics, Ui, User, UserGroup, UserGroupRepoGroupToPerm, UserGroupRepoToPerm, UserRepoGroupToPerm, UserRepoToPerm)
+from kallithea.model.db import (URL_SEP, Permission, RepoGroup, Repository, RepositoryField, Session, Statistics, Ui, User, UserGroup, UserGroupRepoGroupToPerm,
+ UserGroupRepoToPerm, UserRepoGroupToPerm, UserRepoToPerm)
log = logging.getLogger(__name__)
@@ -50,7 +49,7 @@ log = logging.getLogger(__name__)
class RepoModel(object):
- URL_SEPARATOR = Repository.url_sep()
+ URL_SEPARATOR = URL_SEP
def _create_default_perms(self, repository, private):
# create default permission
@@ -81,25 +80,17 @@ class RepoModel(object):
q = Ui.query().filter(Ui.ui_key == '/').one()
return q.ui_value
- def get(self, repo_id, cache=False):
+ def get(self, repo_id):
repo = Repository.query() \
.filter(Repository.repo_id == repo_id)
-
- if cache:
- repo = repo.options(FromCache("sql_cache_short",
- "get_repo_%s" % repo_id))
return repo.scalar()
def get_repo(self, repository):
return Repository.guess_instance(repository)
- def get_by_repo_name(self, repo_name, cache=False):
+ def get_by_repo_name(self, repo_name):
repo = Repository.query() \
.filter(Repository.repo_name == repo_name)
-
- if cache:
- repo = repo.options(FromCache("sql_cache_short",
- "get_repo_%s" % repo_name))
return repo.scalar()
def get_all_user_repos(self, user):
@@ -109,17 +100,15 @@ class RepoModel(object):
:param user:
"""
from kallithea.lib.auth import AuthUser
- user = User.guess_instance(user)
- repos = AuthUser(dbuser=user).permissions['repositories']
- access_check = lambda r: r[1] in ['repository.read',
- 'repository.write',
- 'repository.admin']
- repos = [x[0] for x in filter(access_check, repos.items())]
+ auth_user = AuthUser(dbuser=User.guess_instance(user))
+ repos = [repo_name
+ for repo_name, perm in auth_user.permissions['repositories'].items()
+ if perm in ['repository.read', 'repository.write', 'repository.admin']
+ ]
return Repository.query().filter(Repository.repo_name.in_(repos))
@classmethod
def _render_datatable(cls, tmpl, *args, **kwargs):
- import kallithea
from tg import tmpl_context as c, request, app_globals
from tg.i18n import ugettext as _
@@ -128,7 +117,7 @@ class RepoModel(object):
tmpl = template.get_def(tmpl)
kwargs.update(dict(_=_, h=h, c=c, request=request))
- return tmpl.render(*args, **kwargs)
+ return tmpl.render_unicode(*args, **kwargs)
def get_repos_as_dict(self, repos_list, repo_groups_list=None,
admin=False,
@@ -139,12 +128,16 @@ class RepoModel(object):
admin: return data for action column.
"""
_render = self._render_datatable
- from tg import tmpl_context as c
+ from tg import tmpl_context as c, request
+ from kallithea.model.scm import ScmModel
def repo_lnk(name, rtype, rstate, private, fork_of):
return _render('repo_name', name, rtype, rstate, private, fork_of,
short_name=short_name)
+ def following(repo_id, is_following):
+ return _render('following', repo_id, is_following)
+
def last_change(last_change):
return _render("last_change", last_change)
@@ -189,6 +182,10 @@ class RepoModel(object):
"just_name": repo.just_name,
"name": repo_lnk(repo.repo_name, repo.repo_type,
repo.repo_state, repo.private, repo.fork),
+ "following": following(
+ repo.repo_id,
+ ScmModel().is_following_repo(repo.repo_name, request.authuser.user_id),
+ ),
"last_change_iso": repo.last_db_change.isoformat(),
"last_change": last_change(repo.last_db_change),
"last_changeset": last_rev(repo.repo_name, cs_cache),
@@ -273,7 +270,7 @@ class RepoModel(object):
cur_repo.owner = User.get_by_username(kwargs['owner'])
if 'repo_group' in kwargs:
- assert kwargs['repo_group'] != u'-1', kwargs # RepoForm should have converted to None
+ assert kwargs['repo_group'] != '-1', kwargs # RepoForm should have converted to None
cur_repo.group = RepoGroup.get(kwargs['repo_group'])
cur_repo.repo_name = cur_repo.get_new_name(cur_repo.just_name)
log.debug('Updating repo %s with params:%s', cur_repo, kwargs)
@@ -290,7 +287,7 @@ class RepoModel(object):
# clone_uri is modified - if given a value, check it is valid
if clone_uri != '':
# will raise exception on error
- is_valid_repo_uri(cur_repo.repo_type, clone_uri, make_ui(clear_session=False))
+ is_valid_repo_uri(cur_repo.repo_type, clone_uri, make_ui())
cur_repo.clone_uri = clone_uri
if 'repo_name' in kwargs:
@@ -306,8 +303,7 @@ class RepoModel(object):
repo=cur_repo, user='default', perm=EMPTY_PERM
)
# handle extra fields
- for field in filter(lambda k: k.startswith(RepositoryField.PREFIX),
- kwargs):
+ for field in [k for k in kwargs if k.startswith(RepositoryField.PREFIX)]:
k = RepositoryField.un_prefix_key(field)
ex_field = RepositoryField.get_by_key_name(key=k, repo=cur_repo)
if ex_field:
@@ -339,13 +335,13 @@ class RepoModel(object):
fork_of = Repository.guess_instance(fork_of)
repo_group = RepoGroup.guess_instance(repo_group)
try:
- repo_name = safe_unicode(repo_name)
- description = safe_unicode(description)
+ repo_name = repo_name
+ description = description
# repo name is just a name of repository
# while repo_name_full is a full qualified name that is combined
# with name and path of group
repo_name_full = repo_name
- repo_name = repo_name.split(self.URL_SEPARATOR)[-1]
+ repo_name = repo_name.split(URL_SEP)[-1]
if kallithea.lib.utils2.repo_name_slug(repo_name) != repo_name:
raise Exception('invalid repo name %s' % repo_name)
@@ -360,7 +356,7 @@ class RepoModel(object):
new_repo.private = private
if clone_uri:
# will raise exception on error
- is_valid_repo_uri(repo_type, clone_uri, make_ui(clear_session=False))
+ is_valid_repo_uri(repo_type, clone_uri, make_ui())
new_repo.clone_uri = clone_uri
new_repo.landing_rev = landing_rev
@@ -643,8 +639,7 @@ class RepoModel(object):
_paths = [repo_store_location]
else:
_paths = [self.repos_path, new_parent_path, repo_name]
- # we need to make it str for mercurial
- repo_path = os.path.join(*map(lambda x: safe_str(x), _paths))
+ repo_path = os.path.join(*_paths)
# check if this path is not a repository
if is_valid_repo(repo_path, self.repos_path):
@@ -655,13 +650,13 @@ class RepoModel(object):
raise Exception('This path %s is a valid group' % repo_path)
log.info('creating repo %s in %s from url: `%s`',
- repo_name, safe_unicode(repo_path),
+ repo_name, repo_path,
obfuscate_url_pw(clone_uri))
backend = get_backend(repo_type)
if repo_type == 'hg':
- baseui = make_ui(clear_session=False)
+ baseui = make_ui()
# patch and reset hooks section of UI config to not run any
# hooks on creating remote repo
for k, v in baseui.configitems('hooks'):
@@ -676,7 +671,7 @@ class RepoModel(object):
raise Exception('Not supported repo_type %s expected hg/git' % repo_type)
log.debug('Created repo %s with %s backend',
- safe_unicode(repo_name), safe_unicode(repo_type))
+ repo_name, repo_type)
return repo
def _rename_filesystem_repo(self, old, new):
@@ -688,8 +683,8 @@ class RepoModel(object):
"""
log.info('renaming repo from %s to %s', old, new)
- old_path = safe_str(os.path.join(self.repos_path, old))
- new_path = safe_str(os.path.join(self.repos_path, new))
+ old_path = os.path.join(self.repos_path, old)
+ new_path = os.path.join(self.repos_path, new)
if os.path.isdir(new_path):
raise Exception(
'Was trying to rename to already existing dir %s' % new_path
@@ -704,7 +699,7 @@ class RepoModel(object):
:param repo: repo object
"""
- rm_path = safe_str(os.path.join(self.repos_path, repo.repo_name))
+ rm_path = os.path.join(self.repos_path, repo.repo_name)
log.info("Removing %s", rm_path)
_now = datetime.now()
@@ -715,6 +710,6 @@ class RepoModel(object):
args = repo.group.full_path_splitted + [_d]
_d = os.path.join(*args)
if os.path.exists(rm_path):
- shutil.move(rm_path, safe_str(os.path.join(self.repos_path, _d)))
+ shutil.move(rm_path, os.path.join(self.repos_path, _d))
else:
log.error("Can't find repo to delete in %r", rm_path)
diff --git a/kallithea/model/repo_group.py b/kallithea/model/repo_group.py
--- a/kallithea/model/repo_group.py
+++ b/kallithea/model/repo_group.py
@@ -34,6 +34,7 @@ import traceback
import kallithea.lib.utils2
from kallithea.lib.utils2 import LazyProperty
+from kallithea.model import db
from kallithea.model.db import Permission, RepoGroup, Repository, Session, Ui, User, UserGroup, UserGroupRepoGroupToPerm, UserRepoGroupToPerm
@@ -115,7 +116,7 @@ class RepoGroupModel(object):
:param group: instance of group from database
:param force_delete: use shutil rmtree to remove all objects
"""
- paths = group.full_path.split(RepoGroup.url_sep())
+ paths = group.full_path.split(db.URL_SEP)
paths = os.sep.join(paths)
rm_path = os.path.join(self.repos_path, paths)
@@ -288,7 +289,7 @@ class RepoGroupModel(object):
repo_group.parent_group_id = repo_group_args['parent_group_id']
if 'parent_group_id' in repo_group_args:
- assert repo_group_args['parent_group_id'] != u'-1', repo_group_args # RepoGroupForm should have converted to None
+ assert repo_group_args['parent_group_id'] != '-1', repo_group_args # RepoGroupForm should have converted to None
repo_group.parent_group = RepoGroup.get(repo_group_args['parent_group_id'])
if 'group_name' in repo_group_args:
group_name = repo_group_args['group_name']
diff --git a/kallithea/model/repo_permission.py b/kallithea/model/repo_permission.py
deleted file mode 100644
--- a/kallithea/model/repo_permission.py
+++ /dev/null
@@ -1,100 +0,0 @@
-# -*- coding: utf-8 -*-
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
## EXTRA FOR JS
<%block name="js_extra"/>
-
-
+
## FORM FOR MAKING JS ACTION AS CHANGESET COMMENTS
-
+
%endif
-
-
+
+
<%block name="css_extra">
@@ -34,22 +34,22 @@ ${self.repo_context_bar('changelog')}
-
-
-
+
+
+
<%block name="css_extra">
@@ -70,7 +70,7 @@ ${self.repo_context_bar('files')}
${h.end_form()}
-
-
-
+
+
+
<%block name="css_extra">
@@ -59,7 +59,7 @@ ${self.repo_context_bar('files')}
' + - ls + ' | ' +
- '') - yield 0, dummyoutfile.getvalue() - yield 0, ' |
' + - ls + '
' + - annotate + '
@@ -90,16 +90,14 @@ ${self.repo_context_bar('changelog', c.c
${h.person(c.changeset.author,'full_name_and_username')} - ${h.age(c.changeset.date,True)} ${h.fmt_date(c.changeset.date)}
${h.email_or_none(c.changeset.author)}
- <% rev = c.changeset.extra.get('source') %>
- %if rev:
+ %if c.changeset_graft_source_hash:
${h.email_or_none(c.changeset.author)}
- ${_('Grafted from:')} ${h.link_to(h.short_id(rev),h.url('changeset_home',repo_name=c.repo_name,revision=rev), class_="changeset_hash")}
+ ${_('Grafted from:')} ${h.link_to(h.short_id(c.changeset_graft_source_hash),h.url('changeset_home',repo_name=c.repo_name,revision=c.changeset_graft_source_hash), class_="changeset_hash")}
%endif
- <% rev = c.changeset.extra.get('transplant_source', '').encode('hex') %>
- %if rev:
+ %if c.changeset_transplant_source_hash:
- ${_('Transplanted from:')} ${h.link_to(h.short_id(rev),h.url('changeset_home',repo_name=c.repo_name,revision=rev), class_="changeset_hash")}
+ ${_('Transplanted from:')} ${h.link_to(h.short_id(c.changeset_transplant_source_hash),h.url('changeset_home',repo_name=c.repo_name,revision=c.changeset_transplant_source_hash), class_="changeset_hash")}
%endif
@@ -145,7 +143,7 @@ ${self.repo_context_bar('changelog', c.c
%for fid, url_fid, op, a_path, path, diff, stats in file_diff_data:
- ${h.link_to(h.safe_unicode(path), '#%s' % fid)}
+ ${h.link_to(path, '#%s' % fid)}
@@ -186,9 +184,9 @@ ${self.repo_context_bar('changelog', c.c
${h.fancy_file_stats(stats)}
- ${h.link_to(h.safe_unicode(c.node1.path),h.url('files_home',repo_name=c.repo_name,
- revision=c.cs2.raw_id,f_path=h.safe_unicode(c.node1.path)))}
+ ${h.link_to(c.node1.path,h.url('files_home',repo_name=c.repo_name,
+ revision=c.cs2.raw_id,f_path=c.node1.path))}
-
+
@@ -77,7 +77,7 @@ ${self.repo_context_bar('files')}
-
-
+
%endif
@@ -53,7 +53,7 @@
${h.end_form()}
-
-
+
+
%endif
@@ -90,7 +90,7 @@
${h.end_form()}
-
-
-
+
+
+
<%def name="main()">
@@ -28,8 +28,8 @@ ${self.repo_context_bar('summary')}
-
- %if c.no_data:
+
-
-
%if c.show_stats:
-',),
('mailto:test@example.com',),
('file:///etc/passwd',),
@@ -128,16 +128,16 @@ class TestLoginController(TestController
('non-absolute-path',),
])
def test_login_bad_came_froms(self, url_came_from):
- response = self.app.post(url(controller='login', action='index',
+ response = self.app.post(base.url(controller='login', action='index',
came_from=url_came_from),
- {'username': TEST_USER_ADMIN_LOGIN,
- 'password': TEST_USER_ADMIN_PASS,
+ {'username': base.TEST_USER_ADMIN_LOGIN,
+ 'password': base.TEST_USER_ADMIN_PASS,
'_session_csrf_secret_token': self.session_csrf_secret_token()},
status=400)
def test_login_short_password(self):
- response = self.app.post(url(controller='login', action='index'),
- {'username': TEST_USER_ADMIN_LOGIN,
+ response = self.app.post(base.url(controller='login', action='index'),
+ {'username': base.TEST_USER_ADMIN_LOGIN,
'password': 'as',
'_session_csrf_secret_token': self.session_csrf_secret_token()})
assert response.status == '200 OK'
@@ -145,7 +145,7 @@ class TestLoginController(TestController
response.mustcontain('Enter 3 characters or more')
def test_login_wrong_username_password(self):
- response = self.app.post(url(controller='login', action='index'),
+ response = self.app.post(base.url(controller='login', action='index'),
{'username': 'error',
'password': 'test12',
'_session_csrf_secret_token': self.session_csrf_secret_token()})
@@ -153,8 +153,8 @@ class TestLoginController(TestController
response.mustcontain('Invalid username or password')
def test_login_non_ascii(self):
- response = self.app.post(url(controller='login', action='index'),
- {'username': TEST_USER_REGULAR_LOGIN,
+ response = self.app.post(base.url(controller='login', action='index'),
+ {'username': base.TEST_USER_REGULAR_LOGIN,
'password': 'blåbærgrød',
'_session_csrf_secret_token': self.session_csrf_secret_token()})
@@ -162,63 +162,61 @@ class TestLoginController(TestController
# verify that get arguments are correctly passed along login redirection
- @parametrize('args,args_encoded', [
- ({'foo':'one', 'bar':'two'}, (('foo', 'one'), ('bar', 'two'))),
- ({'blue': u'blå'.encode('utf-8'), 'green':u'grøn'},
- (('blue', u'blå'.encode('utf-8')), ('green', u'grøn'.encode('utf-8')))),
+ @base.parametrize('args', [
+ {'foo':'one', 'bar':'two'},
+ {'blue': 'blå', 'green': 'grøn'},
])
- def test_redirection_to_login_form_preserves_get_args(self, args, args_encoded):
+ def test_redirection_to_login_form_preserves_get_args(self, args):
with fixture.anon_access(False):
- response = self.app.get(url(controller='summary', action='index',
- repo_name=HG_REPO,
+ response = self.app.get(base.url(controller='summary', action='index',
+ repo_name=base.HG_REPO,
**args))
assert response.status == '302 Found'
- came_from = urlparse.parse_qs(urlparse.urlparse(response.location).query)['came_from'][0]
- came_from_qs = urlparse.parse_qsl(urlparse.urlparse(came_from).query)
- for encoded in args_encoded:
- assert encoded in came_from_qs
+ came_from = urllib.parse.parse_qs(urllib.parse.urlparse(response.location).query)['came_from'][0]
+ came_from_qs = urllib.parse.parse_qsl(urllib.parse.urlparse(came_from).query)
+ assert sorted(came_from_qs) == sorted(args.items())
- @parametrize('args,args_encoded', [
+ @base.parametrize('args,args_encoded', [
({'foo':'one', 'bar':'two'}, ('foo=one', 'bar=two')),
- ({'blue': u'blå', 'green':u'grøn'},
+ ({'blue': 'blå', 'green':'grøn'},
('blue=bl%C3%A5', 'green=gr%C3%B8n')),
])
def test_login_form_preserves_get_args(self, args, args_encoded):
- response = self.app.get(url(controller='login', action='index',
- came_from=url('/_admin/users', **args)))
- came_from = urlparse.parse_qs(urlparse.urlparse(response.form.action).query)['came_from'][0]
+ response = self.app.get(base.url(controller='login', action='index',
+ came_from=base.url('/_admin/users', **args)))
+ came_from = urllib.parse.parse_qs(urllib.parse.urlparse(response.form.action).query)['came_from'][0]
for encoded in args_encoded:
assert encoded in came_from
- @parametrize('args,args_encoded', [
+ @base.parametrize('args,args_encoded', [
({'foo':'one', 'bar':'two'}, ('foo=one', 'bar=two')),
- ({'blue': u'blå', 'green':u'grøn'},
+ ({'blue': 'blå', 'green':'grøn'},
('blue=bl%C3%A5', 'green=gr%C3%B8n')),
])
def test_redirection_after_successful_login_preserves_get_args(self, args, args_encoded):
- response = self.app.post(url(controller='login', action='index',
- came_from=url('/_admin/users', **args)),
- {'username': TEST_USER_ADMIN_LOGIN,
- 'password': TEST_USER_ADMIN_PASS,
+ response = self.app.post(base.url(controller='login', action='index',
+ came_from=base.url('/_admin/users', **args)),
+ {'username': base.TEST_USER_ADMIN_LOGIN,
+ 'password': base.TEST_USER_ADMIN_PASS,
'_session_csrf_secret_token': self.session_csrf_secret_token()})
assert response.status == '302 Found'
for encoded in args_encoded:
assert encoded in response.location
- @parametrize('args,args_encoded', [
+ @base.parametrize('args,args_encoded', [
({'foo':'one', 'bar':'two'}, ('foo=one', 'bar=two')),
- ({'blue': u'blå', 'green':u'grøn'},
+ ({'blue': 'blå', 'green':'grøn'},
('blue=bl%C3%A5', 'green=gr%C3%B8n')),
])
def test_login_form_after_incorrect_login_preserves_get_args(self, args, args_encoded):
- response = self.app.post(url(controller='login', action='index',
- came_from=url('/_admin/users', **args)),
+ response = self.app.post(base.url(controller='login', action='index',
+ came_from=base.url('/_admin/users', **args)),
{'username': 'error',
'password': 'test12',
'_session_csrf_secret_token': self.session_csrf_secret_token()})
response.mustcontain('Invalid username or password')
- came_from = urlparse.parse_qs(urlparse.urlparse(response.form.action).query)['came_from'][0]
+ came_from = urllib.parse.parse_qs(urllib.parse.urlparse(response.form.action).query)['came_from'][0]
for encoded in args_encoded:
assert encoded in came_from
@@ -226,12 +224,12 @@ class TestLoginController(TestController
# REGISTRATIONS
#==========================================================================
def test_register(self):
- response = self.app.get(url(controller='login', action='register'))
+ response = self.app.get(base.url(controller='login', action='register'))
response.mustcontain('Sign Up')
def test_register_err_same_username(self):
- uname = TEST_USER_ADMIN_LOGIN
- response = self.app.post(url(controller='login', action='register'),
+ uname = base.TEST_USER_ADMIN_LOGIN
+ response = self.app.post(base.url(controller='login', action='register'),
{'username': uname,
'password': 'test12',
'password_confirmation': 'test12',
@@ -246,11 +244,11 @@ class TestLoginController(TestController
response.mustcontain(msg)
def test_register_err_same_email(self):
- response = self.app.post(url(controller='login', action='register'),
+ response = self.app.post(base.url(controller='login', action='register'),
{'username': 'test_admin_0',
'password': 'test12',
'password_confirmation': 'test12',
- 'email': TEST_USER_ADMIN_EMAIL,
+ 'email': base.TEST_USER_ADMIN_EMAIL,
'firstname': 'test',
'lastname': 'test',
'_session_csrf_secret_token': self.session_csrf_secret_token()})
@@ -260,11 +258,11 @@ class TestLoginController(TestController
response.mustcontain(msg)
def test_register_err_same_email_case_sensitive(self):
- response = self.app.post(url(controller='login', action='register'),
+ response = self.app.post(base.url(controller='login', action='register'),
{'username': 'test_admin_1',
'password': 'test12',
'password_confirmation': 'test12',
- 'email': TEST_USER_ADMIN_EMAIL.title(),
+ 'email': base.TEST_USER_ADMIN_EMAIL.title(),
'firstname': 'test',
'lastname': 'test',
'_session_csrf_secret_token': self.session_csrf_secret_token()})
@@ -273,7 +271,7 @@ class TestLoginController(TestController
response.mustcontain(msg)
def test_register_err_wrong_data(self):
- response = self.app.post(url(controller='login', action='register'),
+ response = self.app.post(base.url(controller='login', action='register'),
{'username': 'xs',
'password': 'test',
'password_confirmation': 'test',
@@ -286,7 +284,7 @@ class TestLoginController(TestController
response.mustcontain('Enter a value 6 characters long or more')
def test_register_err_username(self):
- response = self.app.post(url(controller='login', action='register'),
+ response = self.app.post(base.url(controller='login', action='register'),
{'username': 'error user',
'password': 'test12',
'password_confirmation': 'test12',
@@ -302,8 +300,8 @@ class TestLoginController(TestController
'alphanumeric character')
def test_register_err_case_sensitive(self):
- usr = TEST_USER_ADMIN_LOGIN.title()
- response = self.app.post(url(controller='login', action='register'),
+ usr = base.TEST_USER_ADMIN_LOGIN.title()
+ response = self.app.post(base.url(controller='login', action='register'),
{'username': usr,
'password': 'test12',
'password_confirmation': 'test12',
@@ -319,7 +317,7 @@ class TestLoginController(TestController
response.mustcontain(msg)
def test_register_special_chars(self):
- response = self.app.post(url(controller='login', action='register'),
+ response = self.app.post(base.url(controller='login', action='register'),
{'username': 'xxxaxn',
'password': 'ąćźżąśśśś',
'password_confirmation': 'ąćźżąśśśś',
@@ -333,7 +331,7 @@ class TestLoginController(TestController
response.mustcontain(msg)
def test_register_password_mismatch(self):
- response = self.app.post(url(controller='login', action='register'),
+ response = self.app.post(base.url(controller='login', action='register'),
{'username': 'xs',
'password': '123qwe',
'password_confirmation': 'qwe123',
@@ -352,7 +350,7 @@ class TestLoginController(TestController
name = 'testname'
lastname = 'testlastname'
- response = self.app.post(url(controller='login', action='register'),
+ response = self.app.post(base.url(controller='login', action='register'),
{'username': username,
'password': password,
'password_confirmation': password,
@@ -380,22 +378,22 @@ class TestLoginController(TestController
def test_forgot_password_wrong_mail(self):
bad_email = 'username%wrongmail.org'
response = self.app.post(
- url(controller='login', action='password_reset'),
+ base.url(controller='login', action='password_reset'),
{'email': bad_email,
'_session_csrf_secret_token': self.session_csrf_secret_token()})
response.mustcontain('An email address must contain a single @')
def test_forgot_password(self):
- response = self.app.get(url(controller='login',
+ response = self.app.get(base.url(controller='login',
action='password_reset'))
assert response.status == '200 OK'
username = 'test_password_reset_1'
password = 'qweqwe'
email = 'username@example.com'
- name = u'passwd'
- lastname = u'reset'
+ name = 'passwd'
+ lastname = 'reset'
timestamp = int(time.time())
new = User()
@@ -412,12 +410,12 @@ class TestLoginController(TestController
User.get_by_username(username), timestamp, self.session_csrf_secret_token())
collected = []
- def mock_send_email(recipients, subject, body='', html_body='', headers=None, author=None):
+ def mock_send_email(recipients, subject, body='', html_body='', headers=None, from_name=None):
collected.append((recipients, subject, body, html_body))
with mock.patch.object(kallithea.lib.celerylib.tasks, 'send_email', mock_send_email), \
mock.patch.object(time, 'time', lambda: timestamp):
- response = self.app.post(url(controller='login',
+ response = self.app.post(base.url(controller='login',
action='password_reset'),
{'email': email,
'_session_csrf_secret_token': self.session_csrf_secret_token()})
@@ -431,7 +429,7 @@ class TestLoginController(TestController
(confirmation_url,) = (line for line in body.splitlines() if line.startswith('http://'))
assert ' href="%s"' % confirmation_url.replace('&', '&').replace('@', '%40') in html_body
- d = urlparse.parse_qs(urlparse.urlparse(confirmation_url).query)
+ d = urllib.parse.parse_qs(urllib.parse.urlparse(confirmation_url).query)
assert d['token'] == [token]
assert d['timestamp'] == [str(timestamp)]
assert d['email'] == [email]
@@ -442,7 +440,7 @@ class TestLoginController(TestController
bad_token = "bad"
- response = self.app.post(url(controller='login',
+ response = self.app.post(base.url(controller='login',
action='password_reset_confirmation'),
{'email': email,
'timestamp': timestamp,
@@ -459,14 +457,14 @@ class TestLoginController(TestController
response = self.app.get(confirmation_url)
assert response.status == '200 OK'
response.mustcontain("You are about to set a new password for the email address %s" % email)
- response.mustcontain('
+ %if not c.stats_percentage:
${c.no_data_msg}
%if h.HasPermissionAny('hg.admin')('enable stats on from summary'):
${h.link_to(_('Enable'),h.url('edit_repo',repo_name=c.repo_name),class_="btn btn-default btn-xs")}
@@ -51,19 +51,17 @@ ${self.repo_context_bar('summary')}