kallithea Changeset - 22b40db44a41

Changeset - 22b40db44a41

Parent rev.

Child rev.

[Not reviewed]

default

0 13 0

Mads Kiilerich - 6 years ago 2019-11-25 03:43:15
mads@kiilerich.com

Grafted from: 546b7a8a3341

py3: migrate from urllib2 to urllib

Based on 2to3 urllib, but with unnecessary imports removed.

13 files changed with 58 insertions and 55 deletions:

kallithea/bin/base.py

kallithea/bin/ldap_sync.py

kallithea/lib/auth_modules/auth_crowd.py

kallithea/lib/middleware/simplehg.py

kallithea/lib/recaptcha.py

kallithea/lib/utils2.py

kallithea/lib/vcs/backends/git/repository.py

kallithea/lib/vcs/backends/hg/repository.py

kallithea/tests/functional/test_admin_repos.py

kallithea/tests/functional/test_forks.py

kallithea/tests/other/test_vcs_operations.py

kallithea/tests/scripts/manual_test_crawler.py

kallithea/tests/vcs/test_git.py

0 comments (0 inline, 0 general)

kallithea/bin/base.py

➞

Show inline comments

 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
+#
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.bin.base
 ~~~~~~~~~~~~~~~~~~
 Base utils for shell scripts
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: May 09, 2013
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 import os
 import pprint
 import random
 import sys
-import urllib2
+import urllib.request
 from kallithea.lib import ext_json
 from kallithea.lib.utils2 import ascii_bytes
 CONFIG_NAME = '.config/kallithea'
 FORMAT_PRETTY = 'pretty'
 FORMAT_JSON = 'json'
 def api_call(apikey, apihost, method=None, **kw):
     """
     Api_call wrapper for Kallithea.
     :param apikey:
     :param apihost:
     :param format: formatting, pretty means prints and pprint of json
      json returns unparsed json
     :param method:
     :returns: json response from server
     """
     def _build_data(random_id):
         """
         Builds API data with given random ID
         :param random_id:
         """
         return {
             "id": random_id,
             "api_key": apikey,
             "method": method,
             "args": kw
+        }
     if not method:
         raise Exception('please specify method name !')
     apihost = apihost.rstrip('/')
     id_ = random.randrange(1, 9999)
-    req = urllib2.Request('%s/_admin/api' % apihost,
+    req = urllib.request.Request('%s/_admin/api' % apihost,
                       data=ascii_bytes(ext_json.dumps(_build_data(id_))),
                       headers={'content-type': 'text/plain'})
-    ret = urllib2.urlopen(req)
+    ret = urllib.request.urlopen(req)
     raw_json = ret.read()
     json_data = ext_json.loads(raw_json)
     id_ret = json_data['id']
     if id_ret == id_:
         return json_data
     else:
         _formatted_json = pprint.pformat(json_data)
         raise Exception('something went wrong. '
                         'ID mismatch got %s, expected %s | %s' % (
                                             id_ret, id_, _formatted_json))
 class RcConf(object):
     """
     Kallithea config for API
     conf = RcConf()
     conf['key']
     """
     def __init__(self, config_location=None, autoload=True, autocreate=False,
                  config=None):
         HOME = os.getenv('HOME', os.getenv('USERPROFILE')) or ''
         HOME_CONF = os.path.abspath(os.path.join(HOME, CONFIG_NAME))
         self._conf_name = HOME_CONF if not config_location else config_location
         self._conf = {}
         if autocreate:
             self.make_config(config)
         if autoload:
             self._conf = self.load_config()
     def __getitem__(self, key):
         return self._conf[key]
     def __bool__(self):
         if self._conf:
             return True
         return False
     def __eq__(self, other):
         return self._conf.__eq__(other)
     def __repr__(self):
         return 'RcConf<%s>' % self._conf.__repr__()
     def make_config(self, config):
         """
         Saves given config as a JSON dump in the _conf_name location
         :param config:
         """
         update = False
         if os.path.exists(self._conf_name):
             update = True
         with open(self._conf_name, 'wb') as f:
             ext_json.dump(config, f, indent=4)
             f.write('\n')
         if update:
             sys.stdout.write('Updated config in %s\n' % self._conf_name)
         else:
             sys.stdout.write('Created new config in %s\n' % self._conf_name)
     def update_config(self, new_config):
         """
         Reads the JSON config updates it's values with new_config and
         saves it back as JSON dump
         :param new_config:
         """
         config = {}
         try:
             with open(self._conf_name, 'rb') as conf:
                 config = ext_json.load(conf)
         except IOError as e:
             sys.stderr.write(str(e) + '\n')
         config.update(new_config)
         self.make_config(config)
     def load_config(self):
         """
         Loads config from file and returns loaded JSON object
         """
         try:
             with open(self._conf_name, 'rb') as conf:
                 return ext_json.load(conf)
         except IOError as e:
             #sys.stderr.write(str(e) + '\n')
             pass

kallithea/bin/ldap_sync.py

➞

Show inline comments

 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
+#
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.bin.ldap_sync
 ~~~~~~~~~~~~~~~~~~~~~~~
 LDAP sync script
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Mar 06, 2013
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 from __future__ import print_function
-import urllib2
+import urllib.request
 import uuid
 from configparser import ConfigParser
 import ldap
 from kallithea.lib import ext_json
 from kallithea.lib.utils2 import ascii_bytes
 config = ConfigParser()
 config.read('ldap_sync.conf')
 class InvalidResponseIDError(Exception):
     """ Request and response don't have the same UUID. """
 class ResponseError(Exception):
     """ Response has an error, something went wrong with request execution. """
 class UserAlreadyInGroupError(Exception):
     """ User is already a member of the target group. """
 class UserNotInGroupError(Exception):
     """ User is not a member of the target group. """
 class API(object):
     def __init__(self, url, key):
         self.url = url
         self.key = key
     def get_api_data(self, uid, method, args):
         """Prepare dict for API post."""
         return {
             "id": uid,
             "api_key": self.key,
             "method": method,
             "args": args
+        }
     def post(self, method, args):
         """Send a generic API post to Kallithea.
         This will generate the UUID for validation check after the
         response is returned. Handle errors and get the result back.
         """
         uid = str(uuid.uuid1())
         data = self.get_api_data(uid, method, args)
         data = ascii_bytes(ext_json.dumps(data))
         headers = {'content-type': 'text/plain'}
-        req = urllib2.Request(self.url, data, headers)
+        req = urllib.request.Request(self.url, data, headers)
-        response = urllib2.urlopen(req)
+        response = urllib.request.urlopen(req)
         response = ext_json.load(response)
         if uid != response["id"]:
             raise InvalidResponseIDError("UUID does not match.")
         if response["error"] is not None:
             raise ResponseError(response["error"])
         return response["result"]
     def create_group(self, name, active=True):
         """Create the Kallithea user group."""
         args = {
             "group_name": name,
             "active": str(active)
+        }
         self.post("create_user_group", args)
     def add_membership(self, group, username):
         """Add specific user to a group."""
         args = {
             "usersgroupid": group,
             "userid": username
+        }
         result = self.post("add_user_to_user_group", args)
         if not result["success"]:
             raise UserAlreadyInGroupError("User %s already in group %s." %
                                           (username, group))
     def remove_membership(self, group, username):
         """Remove specific user from a group."""
         args = {
             "usersgroupid": group,
             "userid": username
+        }
         result = self.post("remove_user_from_user_group", args)
         if not result["success"]:
             raise UserNotInGroupError("User %s not in group %s." %
                                       (username, group))
     def get_group_members(self, name):
         """Get the list of member usernames from a user group."""
         args = {"usersgroupid": name}
         members = self.post("get_user_group", args)['members']
         member_list = []
         for member in members:
             member_list.append(member["username"])
         return member_list
     def get_group(self, name):
         """Return group info."""
         args = {"usersgroupid": name}
         return self.post("get_user_group", args)
     def get_user(self, username):
         """Return user info."""
         args = {"userid": username}
         return self.post("get_user", args)
 class LdapClient(object):
     def __init__(self, uri, user, key, base_dn):
         self.client = ldap.initialize(uri, trace_level=0)
         self.client.set_option(ldap.OPT_REFERRALS, 0)
         self.client.simple_bind(user, key)
         self.base_dn = base_dn
     def close(self):
         self.client.unbind()
     def get_groups(self):
         """Get all the groups in form of dict {group_name: group_info,...}."""
         searchFilter = "objectClass=groupOfUniqueNames"
         result = self.client.search_s(self.base_dn, ldap.SCOPE_SUBTREE,
                                       searchFilter)
         groups = {}
         for group in result:
             groups[group[1]['cn'][0]] = group[1]
         return groups
     def get_group_users(self, groups, group):
         """Returns all the users belonging to a single group.
         Based on the list of groups and memberships, returns all the
         users belonging to a single group, searching recursively.
         """
         users = []
         for member in groups[group]["uniqueMember"]:
             member = self.parse_member_string(member)
             if member[0] == "uid":
                 users.append(member[1])
             elif member[0] == "cn":
                 users += self.get_group_users(groups, member[1])

kallithea/lib/auth_modules/auth_crowd.py

➞

Show inline comments

 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
+#
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.lib.auth_modules.auth_crowd
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 Kallithea authentication plugin for Atlassian CROWD
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Created on Nov 17, 2012
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 import base64
 import logging
 import urllib2
 import urllib.parse
 import urllib.request
 from kallithea.lib import auth_modules, ext_json
 from kallithea.lib.compat import hybrid_property
 from kallithea.lib.utils2 import ascii_bytes, ascii_str, safe_bytes
 log = logging.getLogger(__name__)
 class CrowdServer(object):
     def __init__(self, *args, **kwargs):
         """
         Create a new CrowdServer object that points to IP/Address 'host',
         on the given port, and using the given method (https/http). user and
         passwd can be set here or with set_credentials. If unspecified,
         "version" defaults to "latest".
         example::
             cserver = CrowdServer(host="127.0.0.1",
                                   port="8095",
                                   user="some_app",
                                   passwd="some_passwd",
                                   version="1")
         """
         if "port" not in kwargs:
             kwargs["port"] = "8095"
         self._logger = kwargs.get("logger", logging.getLogger(__name__))
         self._uri = "%s://%s:%s/crowd" % (kwargs.get("method", "http"),
                                     kwargs.get("host", "127.0.0.1"),
                                     kwargs.get("port", "8095"))
         self.set_credentials(kwargs.get("user", ""),
                              kwargs.get("passwd", ""))
         self._version = kwargs.get("version", "latest")
         self._url_list = None
         self._appname = "crowd"
     def set_credentials(self, user, passwd):
         self.user = user
         self.passwd = passwd
         self._make_opener()
     def _make_opener(self):
-        mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
+        mgr = urllib.request.HTTPPasswordMgrWithDefaultRealm()
         mgr.add_password(None, self._uri, self.user, self.passwd)
         handler = urllib2.HTTPBasicAuthHandler(mgr)
         self.opener = urllib2.build_opener(handler)
         handler = urllib.request.HTTPBasicAuthHandler(mgr)
         self.opener = urllib.request.build_opener(handler)
     def _request(self, url, body=None, headers=None,
                  method=None, noformat=False,
                  empty_response_ok=False):
         _headers = {"Content-type": "application/json",
                     "Accept": "application/json"}
         if self.user and self.passwd:
             authstring = ascii_str(base64.b64encode(safe_bytes("%s:%s" % (self.user, self.passwd))))
             _headers["Authorization"] = "Basic %s" % authstring
         if headers:
             _headers.update(headers)
         log.debug("Sent to crowd at %s:\nHeaders: %s\nBody:\n%s", url, _headers, body)
-        req = urllib2.Request(url, body, _headers)
+        req = urllib.request.Request(url, body, _headers)
         if method:
             req.get_method = lambda: method
         global msg
         msg = ""
         try:
             rdoc = self.opener.open(req)
             msg = "".join(rdoc.readlines())
             if not msg and empty_response_ok:
                 rval = {}
                 rval["status"] = True
                 rval["error"] = "Response body was empty"
             elif not noformat:
                 rval = ext_json.loads(msg)
                 rval["status"] = True
             else:
                 rval = "".join(rdoc.readlines())
         except Exception as e:
             if not noformat:
                 rval = {"status": False,
                         "body": body,
                         "error": str(e) + "\n" + msg}
             else:
                 rval = None
         return rval
     def user_auth(self, username, password):
         """Authenticate a user against crowd. Returns brief information about
         the user."""
         url = ("%s/rest/usermanagement/%s/authentication?username=%s"
-               % (self._uri, self._version, urllib2.quote(username)))
+               % (self._uri, self._version, urllib.parse.quote(username)))
         body = ascii_bytes(ext_json.dumps({"value": password}))
         return self._request(url, body)
     def user_groups(self, username):
         """Retrieve a list of groups to which this user belongs."""
         url = ("%s/rest/usermanagement/%s/user/group/nested?username=%s"
-               % (self._uri, self._version, urllib2.quote(username)))
+               % (self._uri, self._version, urllib.parse.quote(username)))
         return self._request(url)
 class KallitheaAuthPlugin(auth_modules.KallitheaExternalAuthPlugin):
     def __init__(self):
         self._protocol_values = ["http", "https"]
     @hybrid_property
     def name(self):
         return "crowd"
     def settings(self):
         settings = [
+            {
                 "name": "method",
                 "validator": self.validators.OneOf(self._protocol_values),
                 "type": "select",
                 "values": self._protocol_values,
                 "description": "The protocol used to connect to the Atlassian CROWD server.",
                 "formname": "Protocol"
             },
+            {
                 "name": "host",
                 "validator": self.validators.UnicodeString(strip=True),
                 "type": "string",
                 "description": "The FQDN or IP of the Atlassian CROWD Server",
                 "default": "127.0.0.1",
                 "formname": "Host"
             },
+            {
                 "name": "port",
                 "validator": self.validators.Number(strip=True),
                 "type": "int",
                 "description": "The Port in use by the Atlassian CROWD Server",
                 "default": 8095,
                 "formname": "Port"
             },
+            {
                 "name": "app_name",
                 "validator": self.validators.UnicodeString(strip=True),
                 "type": "string",
                 "description": "The Application Name to authenticate to CROWD",
                 "default": "",
                 "formname": "Application Name"
             },
+            {
                 "name": "app_password",
                 "validator": self.validators.UnicodeString(strip=True),
                 "type": "string",
                 "description": "The password to authenticate to CROWD",
                 "default": "",
                 "formname": "Application Password"
             },
+            {
                 "name": "admin_groups",
                 "validator": self.validators.UnicodeString(strip=True),
                 "type": "string",
                 "description": "A comma separated list of group names that identify users as Kallithea Administrators",
                 "formname": "Admin Groups"
+            }
+        ]
         return settings
     def use_fake_password(self):
         return True
     def auth(self, userobj, username, password, settings, **kwargs):
         """
         Given a user object (which may be null), username, a plaintext password,
         and a settings object (containing all the keys needed as listed in settings()),
         authenticate this user's login attempt.
         Return None on failure. On success, return a dictionary of the form:
             see: KallitheaAuthPluginBase.auth_func_attrs
         This is later validated for correctness
         """
         if not username or not password:
             log.debug('Empty username or password skipping...')
             return None
         log.debug("Crowd settings: %s", settings)
         server = CrowdServer(**settings)
         server.set_credentials(settings["app_name"], settings["app_password"])
         crowd_user = server.user_auth(username, password)
         log.debug("Crowd returned: %s", crowd_user)
         if not crowd_user["status"]:
             log.error('Crowd authentication as %s returned no status', username)
             return None
         if not crowd_user.get('active'):
             log.error('Crowd authentication as %s returned in-active user', username)
             return None
         res = server.user_groups(crowd_user["name"])
         log.debug("Crowd groups: %s", res)

kallithea/lib/middleware/simplehg.py

➞

Show inline comments

 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
+#
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.lib.middleware.simplehg
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 SimpleHg middleware for handling Mercurial protocol requests (push/clone etc.).
 It's implemented with basic auth function
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Apr 28, 2010
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 import logging
 import os
 import urllib
+import urllib.parse
 import mercurial.hgweb
 from kallithea.lib.base import BaseVCSController, get_path_info
 from kallithea.lib.utils import make_ui
 from kallithea.lib.utils2 import safe_bytes, safe_str, safe_unicode
 log = logging.getLogger(__name__)
 def get_header_hgarg(environ):
     """Decode the special Mercurial encoding of big requests over multiple headers.
     >>> get_header_hgarg({})
     ''
     >>> get_header_hgarg({'HTTP_X_HGARG_0': ' ', 'HTTP_X_HGARG_1': 'a','HTTP_X_HGARG_2': '','HTTP_X_HGARG_3': 'b+c %20'})
     'ab+c %20'
     """
     chunks = []
     i = 1
     while True:
         v = environ.get('HTTP_X_HGARG_%d' % i)
         if v is None:
             break
         chunks.append(v)
         i += 1
     return ''.join(chunks)
 cmd_mapping = {
     # 'batch' is not in this list - it is handled explicitly
     'between': 'pull',
     'branches': 'pull',
     'branchmap': 'pull',
     'capabilities': 'pull',
     'changegroup': 'pull',
     'changegroupsubset': 'pull',
     'changesetdata': 'pull',
     'clonebundles': 'pull',
     'debugwireargs': 'pull',
     'filedata': 'pull',
     'getbundle': 'pull',
     'getlfile': 'pull',
     'heads': 'pull',
     'hello': 'pull',
     'known': 'pull',
     'lheads': 'pull',
     'listkeys': 'pull',
     'lookup': 'pull',
     'manifestdata': 'pull',
     'narrow_widen': 'pull',
     'protocaps': 'pull',
     'statlfile': 'pull',
     'stream_out': 'pull',
     'pushkey': 'push',
     'putlfile': 'push',
     'unbundle': 'push',
+    }
 class SimpleHg(BaseVCSController):
     scm_alias = 'hg'
     @classmethod
     def parse_request(cls, environ):
         http_accept = environ.get('HTTP_ACCEPT', '')
         if not http_accept.startswith('application/mercurial'):
             return None
         path_info = get_path_info(environ)
         if not path_info.startswith('/'): # it must!
             return None
         class parsed_request(object):
             repo_name = safe_unicode(path_info[1:].rstrip('/'))
             query_string = environ['QUERY_STRING']
             action = None
             for qry in query_string.split('&'):
                 parts = qry.split('=', 1)
                 if len(parts) == 2 and parts[0] == 'cmd':
                     cmd = parts[1]
                     if cmd == 'batch':
                         hgarg = get_header_hgarg(environ)
                         if not hgarg.startswith('cmds='):
                             action = 'push' # paranoid and safe
                             break
                         action = 'pull'
                         for cmd_arg in hgarg[5:].split(';'):
                             cmd, _args = urllib.unquote_plus(cmd_arg).split(' ', 1)
+                            cmd, _args = urllib.parse.unquote_plus(cmd_arg).split(' ', 1)
                             op = cmd_mapping.get(cmd, 'push')
                             if op != 'pull':
                                 assert op == 'push'
                                 action = 'push'
                                 break
                     else:
                         action = cmd_mapping.get(cmd, 'push')
                     break # only process one cmd
         return parsed_request
     def _make_app(self, parsed_request):
         """
         Make an hgweb wsgi application.
         """
         str_repo_name = safe_str(parsed_request.repo_name)
         repo_path = os.path.join(safe_str(self.basepath), str_repo_name)
         baseui = make_ui(repo_path=repo_path)
         hgweb_app = mercurial.hgweb.hgweb(safe_bytes(repo_path), name=str_repo_name, baseui=baseui)
         def wrapper_app(environ, start_response):
             environ['REPO_NAME'] = str_repo_name # used by mercurial.hgweb.hgweb
             return hgweb_app(environ, start_response)
         return wrapper_app

kallithea/lib/recaptcha.py

➞

Show inline comments

 # -*- coding: utf-8 -*-
 import json
 import urllib
 import urllib2
 import urllib.parse
 import urllib.request
 class RecaptchaResponse(object):
     def __init__(self, is_valid, error_code=None):
         self.is_valid = is_valid
         self.error_code = error_code
     def __repr__(self):
         return '<RecaptchaResponse:%s>' % (self.is_valid)
 def submit(g_recaptcha_response, private_key, remoteip):
     """
     Submits a reCAPTCHA request for verification. Returns RecaptchaResponse for the request
     g_recaptcha_response -- The value of g_recaptcha_response from the form
     private_key -- your reCAPTCHA private key
     remoteip -- the user's IP address
     """
     if not (g_recaptcha_response and len(g_recaptcha_response)):
         return RecaptchaResponse(is_valid=False, error_code='incorrect-captcha-sol')
     def encode_if_necessary(s):
         if isinstance(s, unicode):
             return s.encode('utf-8')
         return s
     params = urllib.urlencode({
+    params = urllib.parse.urlencode({
         'secret': encode_if_necessary(private_key),
         'remoteip': encode_if_necessary(remoteip),
         'response': encode_if_necessary(g_recaptcha_response),
     }).encode('ascii')
-    req = urllib2.Request(
+    req = urllib.request.Request(
         url="https://www.google.com/recaptcha/api/siteverify",
         data=params,
         headers={
             "Content-type": "application/x-www-form-urlencoded",
             "User-agent": "reCAPTCHA Python"
+        }
+    )
-    httpresp = urllib2.urlopen(req)
+    httpresp = urllib.request.urlopen(req)
     return_values = json.loads(httpresp.read())
     httpresp.close()
     if not (isinstance(return_values, dict)):
         return RecaptchaResponse(is_valid=False, error_code='incorrect-captcha-sol')
     elif (("success" in return_values) and ((return_values["success"] is True) or (return_values["success"] == "true"))):
         return RecaptchaResponse(is_valid=True)
     elif (("error-codes" in return_values) and isinstance(return_values["error-codes"], list) and (len(return_values["error-codes"]) > 0)):
         return RecaptchaResponse(is_valid=False, error_code=return_values["error-codes"][0])
     else:
         return RecaptchaResponse(is_valid=False, error_code='incorrect-captcha-sol')

kallithea/lib/utils2.py

➞

Show inline comments

 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
+#
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.lib.utils2
 ~~~~~~~~~~~~~~~~~~~~
 Some simple helper functions.
 Note: all these functions should be independent of Kallithea classes, i.e.
 models, controllers, etc.  to prevent import cycles.
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Jan 5, 2011
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 from __future__ import print_function
 import binascii
 import datetime
 import json
 import os
 import pwd
 import re
 import time
 import urllib
+import urllib.parse
 import urlobject
 from tg.i18n import ugettext as _
 from tg.i18n import ungettext
 from webhelpers2.text import collapse, remove_formatting, strip_tags
 from kallithea.lib.vcs.utils import ascii_bytes, ascii_str, safe_bytes, safe_str, safe_unicode  # re-export
 from kallithea.lib.vcs.utils.lazy import LazyProperty
 def str2bool(_str):
     """
     returns True/False value from given string, it tries to translate the
     string into boolean
     :param _str: string value to translate into boolean
     :rtype: boolean
     :returns: boolean from given string
     """
     if _str is None:
         return False
     if _str in (True, False):
         return _str
     _str = str(_str).strip().lower()
     return _str in ('t', 'true', 'y', 'yes', 'on', '1')
 def aslist(obj, sep=None, strip=True):
     """
     Returns given string separated by sep as list
     :param obj:
     :param sep:
     :param strip:
     """
     if isinstance(obj, (str)):
         lst = obj.split(sep)
         if strip:
             lst = [v.strip() for v in lst]
         return lst
     elif isinstance(obj, (list, tuple)):
         return obj
     elif obj is None:
         return []
     else:
         return [obj]
 def convert_line_endings(line, mode):
     """
     Converts a given line  "line end" according to given mode
     Available modes are::
 - Unix
 - Mac
 - DOS
     :param line: given line to convert
     :param mode: mode to convert to
     :rtype: str
     :return: converted line according to mode
     """
     if mode == 0:
         line = line.replace('\r\n', '\n')
         line = line.replace('\r', '\n')
     elif mode == 1:
         line = line.replace('\r\n', '\r')
         line = line.replace('\n', '\r')
     elif mode == 2:
         line = re.sub("\r(?!\n)|(?<!\r)\n", "\r\n", line)
     return line
 def detect_mode(line, default):
     """
     Detects line break for given line, if line break couldn't be found
     given default value is returned
     :param line: str line
     :param default: default
     :rtype: int
     :return: value of line end on of 0 - Unix, 1 - Mac, 2 - DOS
     """
     if line.endswith('\r\n'):
         return 2
     elif line.endswith('\n'):
         return 0
     elif line.endswith('\r'):
         return 1
     else:
         return default
 def generate_api_key():
     """
     Generates a random (presumably unique) API key.
@@ @@ -229,200 +229,200 @@ def age(prevdate, show_short_version=Fal @@
     if show_short_version:
         if deltas['year'] == 1:
             # ages between 1 and 2 years: show as months
             deltas['month'] += 12
             deltas['year'] = 0
         if deltas['year'] >= 2:
             # ages 2+ years: round
             if deltas['month'] > 6:
                 deltas['year'] += 1
                 deltas['month'] = 0
     # Format the result
     fmt_funcs = {
         'year': lambda d: ungettext(u'%d year', '%d years', d) % d,
         'month': lambda d: ungettext(u'%d month', '%d months', d) % d,
         'day': lambda d: ungettext(u'%d day', '%d days', d) % d,
         'hour': lambda d: ungettext(u'%d hour', '%d hours', d) % d,
         'minute': lambda d: ungettext(u'%d minute', '%d minutes', d) % d,
         'second': lambda d: ungettext(u'%d second', '%d seconds', d) % d,
+    }
     for i, part in enumerate(order):
         value = deltas[part]
         if value == 0:
             continue
         if i < 5:
             sub_part = order[i + 1]
             sub_value = deltas[sub_part]
         else:
             sub_value = 0
         if sub_value == 0 or show_short_version:
             if future:
                 return _('in %s') % fmt_funcs[part](value)
             else:
                 return _('%s ago') % fmt_funcs[part](value)
         if future:
             return _('in %s and %s') % (fmt_funcs[part](value),
                 fmt_funcs[sub_part](sub_value))
         else:
             return _('%s and %s ago') % (fmt_funcs[part](value),
                 fmt_funcs[sub_part](sub_value))
     return _('just now')
 def uri_filter(uri):
     """
     Removes user:password from given url string
     :param uri:
     :rtype: unicode
     :returns: filtered list of strings
     """
     if not uri:
         return []
     proto = ''
     for pat in ('https://', 'http://', 'git://'):
         if uri.startswith(pat):
             uri = uri[len(pat):]
             proto = pat
             break
     # remove passwords and username
     uri = uri[uri.find('@') + 1:]
     # get the port
     cred_pos = uri.find(':')
     if cred_pos == -1:
         host, port = uri, None
     else:
         host, port = uri[:cred_pos], uri[cred_pos + 1:]
     return [_f for _f in [proto, host, port] if _f]
 def credentials_filter(uri):
     """
     Returns a url with removed credentials
     :param uri:
     """
     uri = uri_filter(uri)
     # check if we have port
     if len(uri) > 2 and uri[2]:
         uri[2] = ':' + uri[2]
     return ''.join(uri)
 def get_clone_url(clone_uri_tmpl, prefix_url, repo_name, repo_id, username=None):
     parsed_url = urlobject.URLObject(prefix_url)
     prefix = safe_unicode(urllib.unquote(parsed_url.path.rstrip('/')))
+    prefix = safe_unicode(urllib.parse.unquote(parsed_url.path.rstrip('/')))
     try:
         system_user = pwd.getpwuid(os.getuid()).pw_name
     except Exception: # TODO: support all systems - especially Windows
         system_user = 'kallithea' # hardcoded default value ...
     args = {
         'scheme': parsed_url.scheme,
         'user': safe_unicode(urllib.quote(safe_str(username or ''))),
+        'user': safe_unicode(urllib.parse.quote(safe_str(username or ''))),
         'netloc': parsed_url.netloc + prefix,  # like "hostname:port/prefix" (with optional ":port" and "/prefix")
         'prefix': prefix, # undocumented, empty or starting with /
         'repo': repo_name,
         'repoid': str(repo_id),
         'system_user': safe_unicode(system_user),
         'hostname': parsed_url.hostname,
+    }
     url = re.sub('{([^{}]+)}', lambda m: args.get(m.group(1), m.group(0)), clone_uri_tmpl)
     # remove leading @ sign if it's present. Case of empty user
     url_obj = urlobject.URLObject(url)
     if not url_obj.username:
         url_obj = url_obj.with_username(None)
     return safe_unicode(url_obj)
 def get_changeset_safe(repo, rev):
     """
     Safe version of get_changeset if this changeset doesn't exists for a
     repo it returns a Dummy one instead
     :param repo:
     :param rev:
     """
     from kallithea.lib.vcs.backends.base import BaseRepository
     from kallithea.lib.vcs.exceptions import RepositoryError
     from kallithea.lib.vcs.backends.base import EmptyChangeset
     if not isinstance(repo, BaseRepository):
         raise Exception('You must pass an Repository '
                         'object as first argument got %s' % type(repo))
     try:
         cs = repo.get_changeset(rev)
     except (RepositoryError, LookupError):
         cs = EmptyChangeset(requested_revision=rev)
     return cs
 def datetime_to_time(dt):
     if dt:
         return time.mktime(dt.timetuple())
 def time_to_datetime(tm):
     if tm:
         if isinstance(tm, str):
             try:
                 tm = float(tm)
             except ValueError:
                 return
         return datetime.datetime.fromtimestamp(tm)
 # Must match regexp in kallithea/public/js/base.js MentionsAutoComplete()
 # Check char before @ - it must not look like we are in an email addresses.
 # Matching is greedy so we don't have to look beyond the end.
 MENTIONS_REGEX = re.compile(r'(?:^|(?<=[^a-zA-Z0-9]))@([a-zA-Z0-9][-_.a-zA-Z0-9]*[a-zA-Z0-9])')
 def extract_mentioned_usernames(text):
     r"""
     Returns list of (possible) usernames @mentioned in given text.
     >>> extract_mentioned_usernames('@1-2.a_X,@1234 not@not @ddd@not @n @ee @ff @gg, @gg;@hh @n\n@zz,')
     ['1-2.a_X', '1234', 'ddd', 'ee', 'ff', 'gg', 'gg', 'hh', 'zz']
     """
     return MENTIONS_REGEX.findall(text)
 def extract_mentioned_users(text):
     """ Returns set of actual database Users @mentioned in given text. """
     from kallithea.model.db import User
     result = set()
     for name in extract_mentioned_usernames(text):
         user = User.get_by_username(name, case_insensitive=True)
         if user is not None and not user.is_default_user:
             result.add(user)
     return result
 class AttributeDict(dict):
     def __getattr__(self, attr):
         return self.get(attr, None)
     __setattr__ = dict.__setitem__
     __delattr__ = dict.__delitem__
 def obfuscate_url_pw(engine):
     from sqlalchemy.engine import url as sa_url
     from sqlalchemy.exc import ArgumentError
     try:
         _url = sa_url.make_url(engine or '')
     except ArgumentError:
         return engine
     if _url.password:

kallithea/lib/vcs/backends/git/repository.py

➞

Show inline comments

 # -*- coding: utf-8 -*-
 """
     vcs.backends.git.repository
     ~~~~~~~~~~~~~~~~~~~~~~~~~~~
     Git repository implementation.
     :created_on: Apr 8, 2010
     :copyright: (c) 2010-2011 by Marcin Kuzminski, Lukasz Balcerzak.
 """
 import errno
 import logging
 import os
 import re
 import time
 import urllib
 import urllib2
 import urllib.error
 import urllib.parse
 import urllib.request
 from collections import OrderedDict
 import mercurial.url  # import httpbasicauthhandler, httpdigestauthhandler
 import mercurial.util  # import url as hg_url
 from dulwich.config import ConfigFile
 from dulwich.objects import Tag
 from dulwich.repo import NotGitRepository, Repo
 from kallithea.lib.vcs import subprocessio
 from kallithea.lib.vcs.backends.base import BaseRepository, CollectionGenerator
 from kallithea.lib.vcs.conf import settings
 from kallithea.lib.vcs.exceptions import (
     BranchDoesNotExistError, ChangesetDoesNotExistError, EmptyRepositoryError, RepositoryError, TagAlreadyExistError, TagDoesNotExistError)
 from kallithea.lib.vcs.utils import ascii_str, date_fromtimestamp, makedate, safe_bytes, safe_str, safe_unicode
 from kallithea.lib.vcs.utils.lazy import LazyProperty
 from kallithea.lib.vcs.utils.paths import abspath, get_user_home
 from .changeset import GitChangeset
 from .inmemory import GitInMemoryChangeset
 from .workdir import GitWorkdir
 SHA_PATTERN = re.compile(r'^([0-9a-fA-F]{12}|[0-9a-fA-F]{40})$')
 log = logging.getLogger(__name__)
 class GitRepository(BaseRepository):
     """
     Git repository backend.
     """
     DEFAULT_BRANCH_NAME = 'master'
     scm = 'git'
     def __init__(self, repo_path, create=False, src_url=None,
                  update_after_clone=False, bare=False):
         self.path = safe_unicode(abspath(repo_path))
         self.repo = self._get_repo(create, src_url, update_after_clone, bare)
         self.bare = self.repo.bare
     @property
     def _config_files(self):
         return [
             self.bare and abspath(self.path, 'config')
                       or abspath(self.path, '.git', 'config'),
              abspath(get_user_home(), '.gitconfig'),
+         ]
     @property
     def _repo(self):
         return self.repo
     @property
     def head(self):
         try:
             return self._repo.head()
         except KeyError:
             return None
     @property
     def _empty(self):
         """
         Checks if repository is empty ie. without any changesets
         """
         try:
             self.revisions[0]
         except (KeyError, IndexError):
             return True
         return False
     @LazyProperty
     def revisions(self):
         """
         Returns list of revisions' ids, in ascending order.  Being lazy
         attribute allows external tools to inject shas from cache.
         """
         return self._get_all_revisions()
     @classmethod
     def _run_git_command(cls, cmd, cwd=None):
         """
         Runs given ``cmd`` as git command and returns output bytes in a tuple
         (stdout, stderr) ... or raise RepositoryError.
         :param cmd: git command to be executed
         :param cwd: passed directly to subprocess
         """
         # need to clean fix GIT_DIR !
         gitenv = dict(os.environ)
         gitenv.pop('GIT_DIR', None)
         gitenv['GIT_CONFIG_NOGLOBAL'] = '1'
         assert isinstance(cmd, list), cmd
         cmd = [settings.GIT_EXECUTABLE_PATH, '-c', 'core.quotepath=false'] + cmd
         try:
             p = subprocessio.SubprocessIOChunker(cmd, cwd=cwd, env=gitenv, shell=False)
         except (EnvironmentError, OSError) as err:
             # output from the failing process is in str(EnvironmentError)
             msg = ("Couldn't run git command %s.\n"
                    "Subprocess failed with '%s': %s\n" %
                    (cmd, type(err).__name__, err)
             ).strip()
             log.error(msg)
             raise RepositoryError(msg)
         try:
             stdout = b''.join(p.output)
             stderr = b''.join(p.error)
         finally:
             p.close()
         # TODO: introduce option to make commands fail if they have any stderr output?
         if stderr:
             log.debug('stderr from %s:\n%s', cmd, stderr)
         else:
             log.debug('stderr from %s: None', cmd)
         return stdout, stderr
     def run_git_command(self, cmd):
         """
         Runs given ``cmd`` as git command with cwd set to current repo.
         Returns stdout as unicode str ... or raise RepositoryError.
         """
         cwd = None
         if os.path.isdir(self.path):
             cwd = self.path
         stdout, _stderr = self._run_git_command(cmd, cwd=cwd)
         return safe_unicode(stdout)
     @classmethod
     def _check_url(cls, url):
         """
         Function will check given url and try to verify if it's a valid
         link. Sometimes it may happened that git will issue basic
         auth request that can cause whole API to hang when used from python
         or other external calls.
         On failures it'll raise urllib2.HTTPError, exception is also thrown
         when the return code is non 200
         """
         # check first if it's not an local url
         if os.path.isdir(url) or url.startswith('file:'):
             return True
         if url.startswith('git://'):
             return True
         if '+' in url[:url.find('://')]:
             url = url[url.find('+') + 1:]
         handlers = []
         url_obj = mercurial.util.url(safe_bytes(url))
         test_uri, authinfo = url_obj.authinfo()
         if not test_uri.endswith('info/refs'):
             test_uri = test_uri.rstrip('/') + '/info/refs'
         url_obj.passwd = b'*****'
         cleaned_uri = str(url_obj)
         if authinfo:
             # create a password manager
-            passmgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
+            passmgr = urllib.request.HTTPPasswordMgrWithDefaultRealm()
             passmgr.add_password(*authinfo)
             handlers.extend((mercurial.url.httpbasicauthhandler(passmgr),
                              mercurial.url.httpdigestauthhandler(passmgr)))
-        o = urllib2.build_opener(*handlers)
+        o = urllib.request.build_opener(*handlers)
         o.addheaders = [('User-Agent', 'git/1.7.8.0')]  # fake some git
-        req = urllib2.Request(
+        req = urllib.request.Request(
             "%s?%s" % (
                 test_uri,
                 urllib.urlencode({"service": 'git-upload-pack'})
+                urllib.parse.urlencode({"service": 'git-upload-pack'})
             ))
         try:
             resp = o.open(req)
             if resp.code != 200:
                 raise Exception('Return Code is not 200')
         except Exception as e:
             # means it cannot be cloned
-            raise urllib2.URLError("[%s] org_exc: %s" % (cleaned_uri, e))
+            raise urllib.error.URLError("[%s] org_exc: %s" % (cleaned_uri, e))
         # now detect if it's proper git repo
         gitdata = resp.read()
         if 'service=git-upload-pack' not in gitdata:
-            raise urllib2.URLError(
+            raise urllib.error.URLError(
                 "url [%s] does not look like an git" % cleaned_uri)
         return True
     def _get_repo(self, create, src_url=None, update_after_clone=False,
                   bare=False):
         if create and os.path.exists(self.path):
             raise RepositoryError("Location already exist")
         if src_url and not create:
             raise RepositoryError("Create should be set to True if src_url is "
                                   "given (clone operation creates repository)")
         try:
             if create and src_url:
                 GitRepository._check_url(src_url)
                 self.clone(src_url, update_after_clone, bare)
                 return Repo(self.path)
             elif create:
                 os.makedirs(self.path)
                 if bare:
                     return Repo.init_bare(self.path)
                 else:
                     return Repo.init(self.path)
             else:
                 return Repo(self.path)
         except (NotGitRepository, OSError) as err:
             raise RepositoryError(err)
     def _get_all_revisions(self):
         # we must check if this repo is not empty, since later command
         # fails if it is. And it's cheaper to ask than throw the subprocess
         # errors
         try:
             self._repo.head()
         except KeyError:
             return []
         rev_filter = settings.GIT_REV_FILTER
         cmd = ['rev-list', rev_filter, '--reverse', '--date-order']
         try:
             so = self.run_git_command(cmd)
         except RepositoryError:
             # Can be raised for empty repositories
             return []
         return so.splitlines()
     def _get_all_revisions2(self):
         # alternate implementation using dulwich
         includes = [ascii_str(sha) for key, (sha, type_) in self._parsed_refs.items()
                     if type_ != b'T']
         return [c.commit.id for c in self._repo.get_walker(include=includes)]
     def _get_revision(self, revision):
         """
         Given any revision identifier, returns a 40 char string with revision hash.
         """
         if self._empty:
             raise EmptyRepositoryError("There are no changesets yet")
         if revision in (None, '', 'tip', 'HEAD', 'head', -1):
             revision = -1
         if isinstance(revision, int):
             try:
                 return self.revisions[revision]
             except IndexError:
                 msg = "Revision %r does not exist for %s" % (revision, self.name)
                 raise ChangesetDoesNotExistError(msg)
         if isinstance(revision, (str, unicode)):
             if revision.isdigit() and (len(revision) < 12 or len(revision) == revision.count('0')):
                 try:
                     return self.revisions[int(revision)]
                 except IndexError:
                     msg = "Revision %r does not exist for %s" % (revision, self)
                     raise ChangesetDoesNotExistError(msg)
             # get by branch/tag name
             _ref_revision = self._parsed_refs.get(safe_bytes(revision))
             if _ref_revision:  # and _ref_revision[1] in [b'H', b'RH', b'T']:
                 return ascii_str(_ref_revision[0])
             if revision in self.revisions:
                 return revision
             # maybe it's a tag ? we don't have them in self.revisions
             if revision in self.tags.values():
                 return revision
             if SHA_PATTERN.match(revision):
                 msg = "Revision %r does not exist for %s" % (revision, self.name)
                 raise ChangesetDoesNotExistError(msg)
         raise ChangesetDoesNotExistError("Given revision %r not recognized" % revision)
     def get_ref_revision(self, ref_type, ref_name):
         """

kallithea/lib/vcs/backends/hg/repository.py

➞

Show inline comments

 # -*- coding: utf-8 -*-
 """
     vcs.backends.hg.repository
     ~~~~~~~~~~~~~~~~~~~~~~~~~~
     Mercurial repository implementation.
     :created_on: Apr 8, 2010
     :copyright: (c) 2010-2011 by Marcin Kuzminski, Lukasz Balcerzak.
 """
 import datetime
 import logging
 import os
 import time
 import urllib
 import urllib2
 import urllib.error
 import urllib.parse
 import urllib.request
 from collections import OrderedDict
 import mercurial.commands
 import mercurial.error
 import mercurial.exchange
 import mercurial.hg
 import mercurial.hgweb
 import mercurial.httppeer
 import mercurial.localrepo
 import mercurial.match
 import mercurial.mdiff
 import mercurial.node
 import mercurial.patch
 import mercurial.scmutil
 import mercurial.sshpeer
 import mercurial.tags
 import mercurial.ui
 import mercurial.url
 import mercurial.util
 from kallithea.lib.vcs.backends.base import BaseRepository, CollectionGenerator
 from kallithea.lib.vcs.exceptions import (
     BranchDoesNotExistError, ChangesetDoesNotExistError, EmptyRepositoryError, RepositoryError, TagAlreadyExistError, TagDoesNotExistError, VCSError)
 from kallithea.lib.vcs.utils import ascii_str, author_email, author_name, date_fromtimestamp, makedate, safe_bytes, safe_str, safe_unicode
 from kallithea.lib.vcs.utils.lazy import LazyProperty
 from kallithea.lib.vcs.utils.paths import abspath
 from .changeset import MercurialChangeset
 from .inmemory import MercurialInMemoryChangeset
 from .workdir import MercurialWorkdir
 log = logging.getLogger(__name__)
 class MercurialRepository(BaseRepository):
     """
     Mercurial repository backend
     """
     DEFAULT_BRANCH_NAME = 'default'
     scm = 'hg'
     def __init__(self, repo_path, create=False, baseui=None, src_url=None,
                  update_after_clone=False):
         """
         Raises RepositoryError if repository could not be find at the given
         ``repo_path``.
         :param repo_path: local path of the repository
         :param create=False: if set to True, would try to create repository if
            it does not exist rather than raising exception
         :param baseui=None: user data
         :param src_url=None: would try to clone repository from given location
         :param update_after_clone=False: sets update of working copy after
           making a clone
         """
         if not isinstance(repo_path, str):
             raise VCSError('Mercurial backend requires repository path to '
                            'be instance of <str> got %s instead' %
                            type(repo_path))
         self.path = abspath(repo_path)
         self.baseui = baseui or mercurial.ui.ui()
         # We've set path and ui, now we can set _repo itself
         self._repo = self._get_repo(create, src_url, update_after_clone)
     @property
     def _empty(self):
         """
         Checks if repository is empty ie. without any changesets
         """
         # TODO: Following raises errors when using InMemoryChangeset...
         # return len(self._repo.changelog) == 0
         return len(self.revisions) == 0
     @LazyProperty
     def revisions(self):
         """
         Returns list of revisions' ids, in ascending order.  Being lazy
         attribute allows external tools to inject shas from cache.
         """
         return self._get_all_revisions()
     @LazyProperty
     def name(self):
         return os.path.basename(self.path)
     @LazyProperty
     def branches(self):
         return self._get_branches()
     @LazyProperty
     def closed_branches(self):
         return self._get_branches(normal=False, closed=True)
     @LazyProperty
@@ @@ -221,369 +222,369 @@ class MercurialRepository(BaseRepository @@
     def bookmarks(self):
         """
         Gets bookmarks for this repository
         """
         return self._get_bookmarks()
     def _get_bookmarks(self):
         if self._empty:
             return {}
         return OrderedDict(sorted(
             ((safe_unicode(n), ascii_str(h)) for n, h in self._repo._bookmarks.items()),
             reverse=True,
             key=lambda x: x[0],  # sort by name
         ))
     def _get_all_revisions(self):
         return [ascii_str(self._repo[x].hex()) for x in self._repo.filtered(b'visible').changelog.revs()]
     def get_diff(self, rev1, rev2, path='', ignore_whitespace=False,
                   context=3):
         """
         Returns (git like) *diff*, as plain text. Shows changes introduced by
         ``rev2`` since ``rev1``.
         :param rev1: Entry point from which diff is shown. Can be
           ``self.EMPTY_CHANGESET`` - in this case, patch showing all
           the changes since empty state of the repository until ``rev2``
         :param rev2: Until which revision changes should be shown.
         :param ignore_whitespace: If set to ``True``, would not show whitespace
           changes. Defaults to ``False``.
         :param context: How many lines before/after changed lines should be
           shown. Defaults to ``3``. If negative value is passed-in, it will be
           set to ``0`` instead.
         """
         # Negative context values make no sense, and will result in
         # errors. Ensure this does not happen.
         if context < 0:
             context = 0
         if hasattr(rev1, 'raw_id'):
             rev1 = getattr(rev1, 'raw_id')
         if hasattr(rev2, 'raw_id'):
             rev2 = getattr(rev2, 'raw_id')
         # Check if given revisions are present at repository (may raise
         # ChangesetDoesNotExistError)
         if rev1 != self.EMPTY_CHANGESET:
             self.get_changeset(rev1)
         self.get_changeset(rev2)
         if path:
             file_filter = mercurial.match.exact(path)
         else:
             file_filter = None
         return b''.join(mercurial.patch.diff(self._repo, rev1, rev2, match=file_filter,
                           opts=mercurial.mdiff.diffopts(git=True,
                                         showfunc=True,
                                         ignorews=ignore_whitespace,
                                         context=context)))
     @classmethod
     def _check_url(cls, url, repoui=None):
         """
         Function will check given url and try to verify if it's a valid
         link. Sometimes it may happened that mercurial will issue basic
         auth request that can cause whole API to hang when used from python
         or other external calls.
         On failures it'll raise urllib2.HTTPError, exception is also thrown
         when the return code is non 200
         """
         # check first if it's not an local url
         if os.path.isdir(url) or url.startswith(b'file:'):
             return True
         if url.startswith(b'ssh:'):
             # in case of invalid uri or authentication issues, sshpeer will
             # throw an exception.
             mercurial.sshpeer.instance(repoui or mercurial.ui.ui(), url, False).lookup(b'tip')
             return True
         url_prefix = None
         if b'+' in url[:url.find(b'://')]:
             url_prefix, url = url.split(b'+', 1)
         handlers = []
         url_obj = mercurial.util.url(url)
         test_uri, authinfo = url_obj.authinfo()
         url_obj.passwd = b'*****'
         cleaned_uri = str(url_obj)
         if authinfo:
             # create a password manager
-            passmgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
+            passmgr = urllib.request.HTTPPasswordMgrWithDefaultRealm()
             passmgr.add_password(*authinfo)
             handlers.extend((mercurial.url.httpbasicauthhandler(passmgr),
                              mercurial.url.httpdigestauthhandler(passmgr)))
-        o = urllib2.build_opener(*handlers)
+        o = urllib.request.build_opener(*handlers)
         o.addheaders = [('Content-Type', 'application/mercurial-0.1'),
                         ('Accept', 'application/mercurial-0.1')]
-        req = urllib2.Request(
+        req = urllib.request.Request(
             "%s?%s" % (
                 test_uri,
                 urllib.urlencode({
+                urllib.parse.urlencode({
                     'cmd': 'between',
                     'pairs': "%s-%s" % ('0' * 40, '0' * 40),
                 })
             ))
         try:
             resp = o.open(req)
             if resp.code != 200:
                 raise Exception('Return Code is not 200')
         except Exception as e:
             # means it cannot be cloned
-            raise urllib2.URLError("[%s] org_exc: %s" % (cleaned_uri, e))
+            raise urllib.error.URLError("[%s] org_exc: %s" % (cleaned_uri, e))
         if not url_prefix: # skip svn+http://... (and git+... too)
             # now check if it's a proper hg repo
             try:
                 mercurial.httppeer.instance(repoui or mercurial.ui.ui(), url, False).lookup(b'tip')
             except Exception as e:
-                raise urllib2.URLError(
+                raise urllib.error.URLError(
                     "url [%s] does not look like an hg repo org_exc: %s"
                     % (cleaned_uri, e))
         return True
     def _get_repo(self, create, src_url=None, update_after_clone=False):
         """
         Function will check for mercurial repository in given path and return
         a localrepo object. If there is no repository in that path it will
         raise an exception unless ``create`` parameter is set to True - in
         that case repository would be created and returned.
         If ``src_url`` is given, would try to clone repository from the
         location at given clone_point. Additionally it'll make update to
         working copy accordingly to ``update_after_clone`` flag
         """
         try:
             if src_url:
                 url = safe_bytes(self._get_url(src_url))
                 opts = {}
                 if not update_after_clone:
                     opts.update({'noupdate': True})
                 MercurialRepository._check_url(url, self.baseui)
                 mercurial.commands.clone(self.baseui, url, safe_bytes(self.path), **opts)
                 # Don't try to create if we've already cloned repo
                 create = False
             return mercurial.localrepo.instance(self.baseui, safe_bytes(self.path), create=create)
         except (mercurial.error.Abort, mercurial.error.RepoError) as err:
             if create:
                 msg = "Cannot create repository at %s. Original error was %s" \
                     % (self.name, err)
             else:
                 msg = "Not valid repository at %s. Original error was %s" \
                     % (self.name, err)
             raise RepositoryError(msg)
     @LazyProperty
     def in_memory_changeset(self):
         return MercurialInMemoryChangeset(self)
     @LazyProperty
     def description(self):
         _desc = self._repo.ui.config(b'web', b'description', None, untrusted=True)
         return safe_unicode(_desc or b'unknown')
     @LazyProperty
     def contact(self):
         return safe_unicode(mercurial.hgweb.common.get_contact(self._repo.ui.config)
                             or b'Unknown')
     @LazyProperty
     def last_change(self):
         """
         Returns last change made on this repository as datetime object
         """
         return date_fromtimestamp(self._get_mtime(), makedate()[1])
     def _get_mtime(self):
         try:
             return time.mktime(self.get_changeset().date.timetuple())
         except RepositoryError:
             # fallback to filesystem
             cl_path = os.path.join(self.path, '.hg', "00changelog.i")
             st_path = os.path.join(self.path, '.hg', "store")
             if os.path.exists(cl_path):
                 return os.stat(cl_path).st_mtime
             else:
                 return os.stat(st_path).st_mtime
     def _get_revision(self, revision):
         """
         Given any revision identifier, returns a 40 char string with revision hash.
         :param revision: str or int or None
         """
         if self._empty:
             raise EmptyRepositoryError("There are no changesets yet")
         if revision in [-1, None]:
             revision = b'tip'
         elif isinstance(revision, unicode):
             revision = safe_bytes(revision)
         try:
             if isinstance(revision, int):
                 return ascii_str(self._repo[revision].hex())
             return ascii_str(mercurial.scmutil.revsymbol(self._repo, revision).hex())
         except (IndexError, ValueError, mercurial.error.RepoLookupError, TypeError):
             msg = "Revision %r does not exist for %s" % (safe_unicode(revision), self.name)
             raise ChangesetDoesNotExistError(msg)
         except (LookupError, ):
             msg = "Ambiguous identifier `%s` for %s" % (safe_unicode(revision), self.name)
             raise ChangesetDoesNotExistError(msg)
     def get_ref_revision(self, ref_type, ref_name):
         """
         Returns revision number for the given reference.
         """
         ref_name = safe_str(ref_name)
         if ref_type == 'rev' and not ref_name.strip('0'):
             return self.EMPTY_CHANGESET
         # lookup up the exact node id
         _revset_predicates = {
                 'branch': 'branch',
                 'book': 'bookmark',
                 'tag': 'tag',
                 'rev': 'id',
+            }
         # avoid expensive branch(x) iteration over whole repo
         rev_spec = "%%s & %s(%%s)" % _revset_predicates[ref_type]
         try:
             revs = self._repo.revs(rev_spec, ref_name, ref_name)
         except LookupError:
             msg = "Ambiguous identifier %s:%s for %s" % (ref_type, ref_name, self.name)
             raise ChangesetDoesNotExistError(msg)
         except mercurial.error.RepoLookupError:
             msg = "Revision %s:%s does not exist for %s" % (ref_type, ref_name, self.name)
             raise ChangesetDoesNotExistError(msg)
         if revs:
             revision = revs.last()
         else:
             # TODO: just report 'not found'?
             revision = ref_name
         return self._get_revision(revision)
     def _get_archives(self, archive_name='tip'):
         allowed = self.baseui.configlist(b"web", b"allow_archive",
                                          untrusted=True)
         for name, ext in [(b'zip', '.zip'), (b'gz', '.tar.gz'), (b'bz2', '.tar.bz2')]:
             if name in allowed or self._repo.ui.configbool(b"web",
                                                            b"allow" + name,
                                                            untrusted=True):
                 yield {"type": name, "extension": ext, "node": archive_name}
     def _get_url(self, url):
         """
         Returns normalized url. If schema is not given, would fall
         to filesystem
         (``file:///``) schema.
         """
         url = safe_str(url)
         if url != 'default' and '://' not in url:
             url = "file:" + urllib.pathname2url(url)
+            url = "file:" + urllib.request.pathname2url(url)
         return url
     def get_changeset(self, revision=None):
         """
         Returns ``MercurialChangeset`` object representing repository's
         changeset at the given ``revision``.
         """
         return MercurialChangeset(repository=self, revision=self._get_revision(revision))
     def get_changesets(self, start=None, end=None, start_date=None,
                        end_date=None, branch_name=None, reverse=False, max_revisions=None):
         """
         Returns iterator of ``MercurialChangeset`` objects from start to end
         (both are inclusive)
         :param start: None, str, int or mercurial lookup format
         :param end:  None, str, int or mercurial lookup format
         :param start_date:
         :param end_date:
         :param branch_name:
         :param reversed: return changesets in reversed order
         """
         start_raw_id = self._get_revision(start)
         start_pos = None if start is None else self.revisions.index(start_raw_id)
         end_raw_id = self._get_revision(end)
         end_pos = None if end is None else self.revisions.index(end_raw_id)
         if start_pos is not None and end_pos is not None and start_pos > end_pos:
             raise RepositoryError("Start revision '%s' cannot be "
                                   "after end revision '%s'" % (start, end))
         if branch_name and branch_name not in self.allbranches:
             msg = "Branch %r not found in %s" % (branch_name, self.name)
             raise BranchDoesNotExistError(msg)
         if end_pos is not None:
             end_pos += 1
         # filter branches
         filter_ = []
         if branch_name:
             filter_.append(b'branch("%s")' % safe_bytes(branch_name))
         if start_date:
             filter_.append(b'date(">%s")' % safe_bytes(str(start_date)))
         if end_date:
             filter_.append(b'date("<%s")' % safe_bytes(str(end_date)))
         if filter_ or max_revisions:
             if filter_:
                 revspec = b' and '.join(filter_)
             else:
                 revspec = b'all()'
             if max_revisions:
                 revspec = b'limit(%s, %d)' % (revspec, max_revisions)
             revisions = mercurial.scmutil.revrange(self._repo, [revspec])
         else:
             revisions = self.revisions
         # this is very much a hack to turn this into a list; a better solution
         # would be to get rid of this function entirely and use revsets
         revs = list(revisions)[start_pos:end_pos]
         if reverse:
             revs.reverse()
         return CollectionGenerator(self, revs)
     def pull(self, url):
         """
         Tries to pull changes from external location.
         """
         other = mercurial.hg.peer(self._repo, {}, safe_bytes(self._get_url(url)))
         try:
             mercurial.exchange.pull(self._repo, other, heads=None, force=None)
         except mercurial.error.Abort as err:
             # Propagate error but with vcs's type
             raise RepositoryError(str(err))
     @LazyProperty
     def workdir(self):
         """
         Returns ``Workdir`` instance for this repository.
         """
         return MercurialWorkdir(self)
     def get_config_value(self, section, name=None, config_file=None):
         """
         Returns configuration value for a given [``section``] and ``name``.
         :param section: Section we want to retrieve value from
         :param name: Name of configuration we want to retrieve
         :param config_file: A path to file which should be used to retrieve
           configuration from (might also be a list of file paths)
         """
         if config_file is None:
             config_file = []
         elif isinstance(config_file, str):
             config_file = [config_file]
         config = self._repo.ui

kallithea/tests/functional/test_admin_repos.py

➞

Show inline comments

 # -*- coding: utf-8 -*-
 import os
 import urllib
+import urllib.parse
 import mock
 import pytest
 from kallithea.lib import vcs
 from kallithea.lib.utils2 import safe_str, safe_unicode
 from kallithea.model.db import Permission, RepoGroup, Repository, Ui, User, UserRepoToPerm
 from kallithea.model.meta import Session
 from kallithea.model.repo import RepoModel
 from kallithea.model.repo_group import RepoGroupModel
 from kallithea.model.user import UserModel
 from kallithea.tests import base
 from kallithea.tests.fixture import Fixture, error_function
 fixture = Fixture()
 def _get_permission_for_user(user, repo):
     perm = UserRepoToPerm.query() \
                 .filter(UserRepoToPerm.repository ==
                         Repository.get_by_repo_name(repo)) \
                 .filter(UserRepoToPerm.user == User.get_by_username(user)) \
                 .all()
     return perm
 class _BaseTestCase(base.TestController):
     """
     Write all tests here
     """
     REPO = None
     REPO_TYPE = None
     NEW_REPO = None
     OTHER_TYPE_REPO = None
     OTHER_TYPE = None
     def test_index(self):
         self.log_user()
         response = self.app.get(base.url('repos'))
     def test_create(self):
         self.log_user()
         repo_name = self.NEW_REPO
         description = u'description for newly created repo'
         response = self.app.post(base.url('repos'),
                         fixture._get_repo_create_params(repo_private=False,
                                                 repo_name=repo_name,
                                                 repo_type=self.REPO_TYPE,
                                                 repo_description=description,
                                                 _session_csrf_secret_token=self.session_csrf_secret_token()))
         ## run the check page that triggers the flash message
         response = self.app.get(base.url('repo_check_home', repo_name=repo_name))
         assert response.json == {u'result': True}
         self.checkSessionFlash(response,
                                'Created repository <a href="/%s">%s</a>'
                                % (repo_name, repo_name))
         # test if the repo was created in the database
         new_repo = Session().query(Repository) \
             .filter(Repository.repo_name == repo_name).one()
         assert new_repo.repo_name == repo_name
         assert new_repo.description == description
         # test if the repository is visible in the list ?
         response = self.app.get(base.url('summary_home', repo_name=repo_name))
         response.mustcontain(repo_name)
         response.mustcontain(self.REPO_TYPE)
         # test if the repository was created on filesystem
         try:
             vcs.get_repo(safe_str(os.path.join(Ui.get_by_key('paths', '/').ui_value, repo_name)))
         except vcs.exceptions.VCSError:
             pytest.fail('no repo %s in filesystem' % repo_name)
         RepoModel().delete(repo_name)
         Session().commit()
     def test_case_insensitivity(self):
         self.log_user()
         repo_name = self.NEW_REPO
         description = u'description for newly created repo'
         response = self.app.post(base.url('repos'),
                                  fixture._get_repo_create_params(repo_private=False,
                                                                  repo_name=repo_name,
                                                                  repo_type=self.REPO_TYPE,
                                                                  repo_description=description,
                                                                  _session_csrf_secret_token=self.session_csrf_secret_token()))
         # try to create repo with swapped case
         swapped_repo_name = repo_name.swapcase()
         response = self.app.post(base.url('repos'),
                                  fixture._get_repo_create_params(repo_private=False,
                                                                  repo_name=swapped_repo_name,
                                                                  repo_type=self.REPO_TYPE,
                                                                  repo_description=description,
@@ @@ -317,193 +317,193 @@ class _BaseTestCase(base.TestController) @@
         RepoModel().delete(repo_name_full)
         RepoGroupModel().delete(group_name)
         Session().commit()
     def test_create_remote_repo_wrong_clone_uri(self):
         self.log_user()
         repo_name = self.NEW_REPO
         description = u'description for newly created repo'
         response = self.app.post(base.url('repos'),
                         fixture._get_repo_create_params(repo_private=False,
                                                 repo_name=repo_name,
                                                 repo_type=self.REPO_TYPE,
                                                 repo_description=description,
                                                 clone_uri='http://127.0.0.1/repo',
                                                 _session_csrf_secret_token=self.session_csrf_secret_token()))
         response.mustcontain('Invalid repository URL')
     def test_create_remote_repo_wrong_clone_uri_hg_svn(self):
         self.log_user()
         repo_name = self.NEW_REPO
         description = u'description for newly created repo'
         response = self.app.post(base.url('repos'),
                         fixture._get_repo_create_params(repo_private=False,
                                                 repo_name=repo_name,
                                                 repo_type=self.REPO_TYPE,
                                                 repo_description=description,
                                                 clone_uri='svn+http://127.0.0.1/repo',
                                                 _session_csrf_secret_token=self.session_csrf_secret_token()))
         response.mustcontain('Invalid repository URL')
     def test_delete(self):
         self.log_user()
         repo_name = u'vcs_test_new_to_delete_%s' % self.REPO_TYPE
         description = u'description for newly created repo'
         response = self.app.post(base.url('repos'),
                         fixture._get_repo_create_params(repo_private=False,
                                                 repo_type=self.REPO_TYPE,
                                                 repo_name=repo_name,
                                                 repo_description=description,
                                                 _session_csrf_secret_token=self.session_csrf_secret_token()))
         ## run the check page that triggers the flash message
         response = self.app.get(base.url('repo_check_home', repo_name=repo_name))
         self.checkSessionFlash(response,
                                'Created repository <a href="/%s">%s</a>'
                                % (repo_name, repo_name))
         # test if the repo was created in the database
         new_repo = Session().query(Repository) \
             .filter(Repository.repo_name == repo_name).one()
         assert new_repo.repo_name == repo_name
         assert new_repo.description == description
         # test if the repository is visible in the list ?
         response = self.app.get(base.url('summary_home', repo_name=repo_name))
         response.mustcontain(repo_name)
         response.mustcontain(self.REPO_TYPE)
         # test if the repository was created on filesystem
         try:
             vcs.get_repo(safe_str(os.path.join(Ui.get_by_key('paths', '/').ui_value, repo_name)))
         except vcs.exceptions.VCSError:
             pytest.fail('no repo %s in filesystem' % repo_name)
         response = self.app.post(base.url('delete_repo', repo_name=repo_name),
             params={'_session_csrf_secret_token': self.session_csrf_secret_token()})
         self.checkSessionFlash(response, 'Deleted repository %s' % (repo_name))
         response.follow()
         # check if repo was deleted from db
         deleted_repo = Session().query(Repository) \
             .filter(Repository.repo_name == repo_name).scalar()
         assert deleted_repo is None
         assert os.path.isdir(os.path.join(Ui.get_by_key('paths', '/').ui_value, repo_name)) == False
     def test_delete_non_ascii(self):
         self.log_user()
         non_ascii = "ąęł"
         repo_name = "%s%s" % (safe_str(self.NEW_REPO), non_ascii)
         repo_name_unicode = safe_unicode(repo_name)
         description = 'description for newly created repo' + non_ascii
         description_unicode = safe_unicode(description)
         response = self.app.post(base.url('repos'),
                         fixture._get_repo_create_params(repo_private=False,
                                                 repo_name=repo_name,
                                                 repo_type=self.REPO_TYPE,
                                                 repo_description=description,
                                                 _session_csrf_secret_token=self.session_csrf_secret_token()))
         ## run the check page that triggers the flash message
         response = self.app.get(base.url('repo_check_home', repo_name=repo_name))
         assert response.json == {u'result': True}
         self.checkSessionFlash(response,
                                u'Created repository <a href="/%s">%s</a>'
                                % (urllib.quote(repo_name), repo_name_unicode))
+                               % (urllib.parse.quote(repo_name), repo_name_unicode))
         # test if the repo was created in the database
         new_repo = Session().query(Repository) \
             .filter(Repository.repo_name == repo_name_unicode).one()
         assert new_repo.repo_name == repo_name_unicode
         assert new_repo.description == description_unicode
         # test if the repository is visible in the list ?
         response = self.app.get(base.url('summary_home', repo_name=repo_name))
         response.mustcontain(repo_name)
         response.mustcontain(self.REPO_TYPE)
         # test if the repository was created on filesystem
         try:
             vcs.get_repo(safe_str(os.path.join(Ui.get_by_key('paths', '/').ui_value, repo_name_unicode)))
         except vcs.exceptions.VCSError:
             pytest.fail('no repo %s in filesystem' % repo_name)
         response = self.app.post(base.url('delete_repo', repo_name=repo_name),
             params={'_session_csrf_secret_token': self.session_csrf_secret_token()})
         self.checkSessionFlash(response, 'Deleted repository %s' % (repo_name_unicode))
         response.follow()
         # check if repo was deleted from db
         deleted_repo = Session().query(Repository) \
             .filter(Repository.repo_name == repo_name_unicode).scalar()
         assert deleted_repo is None
         assert os.path.isdir(os.path.join(Ui.get_by_key('paths', '/').ui_value, repo_name_unicode)) == False
     def test_delete_repo_with_group(self):
         # TODO:
         pass
     def test_delete_browser_fakeout(self):
         response = self.app.post(base.url('delete_repo', repo_name=self.REPO),
                                  params=dict(_session_csrf_secret_token=self.session_csrf_secret_token()))
     def test_show(self):
         self.log_user()
         response = self.app.get(base.url('summary_home', repo_name=self.REPO))
     def test_edit(self):
         response = self.app.get(base.url('edit_repo', repo_name=self.REPO))
     def test_set_private_flag_sets_default_to_none(self):
         self.log_user()
         # initially repository perm should be read
         perm = _get_permission_for_user(user='default', repo=self.REPO)
         assert len(perm), 1
         assert perm[0].permission.permission_name == 'repository.read'
         assert Repository.get_by_repo_name(self.REPO).private == False
         response = self.app.post(base.url('update_repo', repo_name=self.REPO),
                         fixture._get_repo_create_params(repo_private=1,
                                                 repo_name=self.REPO,
                                                 repo_type=self.REPO_TYPE,
                                                 owner=base.TEST_USER_ADMIN_LOGIN,
                                                 _session_csrf_secret_token=self.session_csrf_secret_token()))
         self.checkSessionFlash(response,
                                msg='Repository %s updated successfully' % (self.REPO))
         assert Repository.get_by_repo_name(self.REPO).private == True
         # now the repo default permission should be None
         perm = _get_permission_for_user(user='default', repo=self.REPO)
         assert len(perm), 1
         assert perm[0].permission.permission_name == 'repository.none'
         response = self.app.post(base.url('update_repo', repo_name=self.REPO),
                         fixture._get_repo_create_params(repo_private=False,
                                                 repo_name=self.REPO,
                                                 repo_type=self.REPO_TYPE,
                                                 owner=base.TEST_USER_ADMIN_LOGIN,
                                                 _session_csrf_secret_token=self.session_csrf_secret_token()))
         self.checkSessionFlash(response,
                                msg='Repository %s updated successfully' % (self.REPO))
         assert Repository.get_by_repo_name(self.REPO).private == False
         # we turn off private now the repo default permission should stay None
         perm = _get_permission_for_user(user='default', repo=self.REPO)
         assert len(perm), 1
         assert perm[0].permission.permission_name == 'repository.none'
         # update this permission back
         perm[0].permission = Permission.get_by_key('repository.read')
         Session().commit()
     def test_set_repo_fork_has_no_self_id(self):
         self.log_user()
         repo = Repository.get_by_repo_name(self.REPO)
         response = self.app.get(base.url('edit_repo_advanced', repo_name=self.REPO))
         opt = """<option value="%s">%s</option>""" % (repo.repo_id, self.REPO)
         response.mustcontain(no=[opt])
     def test_set_fork_of_other_repo(self):

kallithea/tests/functional/test_forks.py

➞

Show inline comments

 # -*- coding: utf-8 -*-
 import urllib
+import urllib.parse
 from kallithea.lib.utils2 import safe_str, safe_unicode
 from kallithea.model.db import Repository, User
 from kallithea.model.meta import Session
 from kallithea.model.repo import RepoModel
 from kallithea.model.user import UserModel
 from kallithea.tests import base
 from kallithea.tests.fixture import Fixture
 fixture = Fixture()
 class _BaseTestCase(base.TestController):
     """
     Write all tests here
     """
     REPO = None
     REPO_TYPE = None
     NEW_REPO = None
     REPO_FORK = None
     def setup_method(self, method):
         self.username = u'forkuser'
         self.password = u'qweqwe'
         u1 = fixture.create_user(self.username, password=self.password, email=u'fork_king@example.com')
         self.u1_id = u1.user_id
         Session().commit()
     def teardown_method(self, method):
         fixture.destroy_user(self.u1_id)
         Session().commit()
     def test_index(self):
         self.log_user()
         repo_name = self.REPO
         response = self.app.get(base.url(controller='forks', action='forks',
                                     repo_name=repo_name))
         response.mustcontain("""There are no forks yet""")
     def test_no_permissions_to_fork(self):
         self.log_user(base.TEST_USER_REGULAR_LOGIN, base.TEST_USER_REGULAR_PASS)['user_id']
         try:
             user_model = UserModel()
             usr = User.get_default_user()
             user_model.revoke_perm(usr, 'hg.fork.repository')
             user_model.grant_perm(usr, 'hg.fork.none')
             Session().commit()
             # try create a fork
             repo_name = self.REPO
             self.app.post(base.url(controller='forks', action='fork_create',
                               repo_name=repo_name), {'_session_csrf_secret_token': self.session_csrf_secret_token()}, status=403)
         finally:
             usr = User.get_default_user()
             user_model.revoke_perm(usr, 'hg.fork.none')
             user_model.grant_perm(usr, 'hg.fork.repository')
             Session().commit()
     def test_index_with_fork(self):
         self.log_user()
         # create a fork
         fork_name = self.REPO_FORK
         description = 'fork of vcs test'
         repo_name = self.REPO
         org_repo = Repository.get_by_repo_name(repo_name)
         creation_args = {
             'repo_name': fork_name,
             'repo_group': u'-1',
             'fork_parent_id': org_repo.repo_id,
             'repo_type': self.REPO_TYPE,
             'description': description,
             'private': 'False',
             'landing_rev': 'rev:tip',
             '_session_csrf_secret_token': self.session_csrf_secret_token()}
         self.app.post(base.url(controller='forks', action='fork_create',
                           repo_name=repo_name), creation_args)
         response = self.app.get(base.url(controller='forks', action='forks',
                                     repo_name=repo_name))
         response.mustcontain(
             """<a href="/%s">%s</a>""" % (fork_name, fork_name)
+        )
         # remove this fork
         response = self.app.post(base.url('delete_repo', repo_name=fork_name),
             params={'_session_csrf_secret_token': self.session_csrf_secret_token()})
     def test_fork_create_into_group(self):
         self.log_user()
         group = fixture.create_repo_group(u'vc')
         group_id = group.group_id
         fork_name = self.REPO_FORK
         fork_name_full = 'vc/%s' % fork_name
         description = 'fork of vcs test'
         repo_name = self.REPO
         org_repo = Repository.get_by_repo_name(repo_name)
         creation_args = {
             'repo_name': fork_name,
             'repo_group': group_id,
             'fork_parent_id': org_repo.repo_id,
             'repo_type': self.REPO_TYPE,
             'description': description,
             'private': 'False',
             'landing_rev': 'rev:tip',
             '_session_csrf_secret_token': self.session_csrf_secret_token()}
         self.app.post(base.url(controller='forks', action='fork_create',
                           repo_name=repo_name), creation_args)
         repo = Repository.get_by_repo_name(fork_name_full)
         assert repo.fork.repo_name == self.REPO
         ## run the check page that triggers the flash message
         response = self.app.get(base.url('repo_check_home', repo_name=fork_name_full))
         # test if we have a message that fork is ok
         self.checkSessionFlash(response,
                 'Forked repository %s as <a href="/%s">%s</a>'
                 % (repo_name, fork_name_full, fork_name_full))
         # test if the fork was created in the database
         fork_repo = Session().query(Repository) \
             .filter(Repository.repo_name == fork_name_full).one()
         assert fork_repo.repo_name == fork_name_full
         assert fork_repo.fork.repo_name == repo_name
         # test if the repository is visible in the list ?
         response = self.app.get(base.url('summary_home', repo_name=fork_name_full))
         response.mustcontain(fork_name_full)
         response.mustcontain(self.REPO_TYPE)
         response.mustcontain('Fork of "<a href="/%s">%s</a>"' % (repo_name, repo_name))
         fixture.destroy_repo(fork_name_full)
         fixture.destroy_repo_group(group_id)
     def test_fork_unicode(self):
         self.log_user()
         # create a fork
         repo_name = self.REPO
         org_repo = Repository.get_by_repo_name(repo_name)
         fork_name = safe_str(self.REPO_FORK + u'-rødgrød')
         creation_args = {
             'repo_name': fork_name,
             'repo_group': u'-1',
             'fork_parent_id': org_repo.repo_id,
             'repo_type': self.REPO_TYPE,
             'description': 'unicode repo 1',
             'private': 'False',
             'landing_rev': 'rev:tip',
             '_session_csrf_secret_token': self.session_csrf_secret_token()}
         self.app.post(base.url(controller='forks', action='fork_create',
                           repo_name=repo_name), creation_args)
         response = self.app.get(base.url(controller='forks', action='forks',
                                     repo_name=repo_name))
         response.mustcontain(
             """<a href="/%s">%s</a>""" % (urllib.quote(fork_name), fork_name)
+            """<a href="/%s">%s</a>""" % (urllib.parse.quote(fork_name), fork_name)
+        )
         fork_repo = Repository.get_by_repo_name(safe_unicode(fork_name))
         assert fork_repo
         # fork the fork
         fork_name_2 = safe_str(self.REPO_FORK + u'-blåbærgrød')
         creation_args = {
             'repo_name': fork_name_2,
             'repo_group': u'-1',
             'fork_parent_id': fork_repo.repo_id,
             'repo_type': self.REPO_TYPE,
             'description': 'unicode repo 2',
             'private': 'False',
             'landing_rev': 'rev:tip',
             '_session_csrf_secret_token': self.session_csrf_secret_token()}
         self.app.post(base.url(controller='forks', action='fork_create',
                           repo_name=fork_name), creation_args)
         response = self.app.get(base.url(controller='forks', action='forks',
                                     repo_name=fork_name))
         response.mustcontain(
             """<a href="/%s">%s</a>""" % (urllib.quote(fork_name_2), fork_name_2)
+            """<a href="/%s">%s</a>""" % (urllib.parse.quote(fork_name_2), fork_name_2)
+        )
         # remove these forks
         response = self.app.post(base.url('delete_repo', repo_name=fork_name_2),
             params={'_session_csrf_secret_token': self.session_csrf_secret_token()})
         response = self.app.post(base.url('delete_repo', repo_name=fork_name),
             params={'_session_csrf_secret_token': self.session_csrf_secret_token()})
     def test_fork_create_and_permissions(self):
         self.log_user()
         fork_name = self.REPO_FORK
         description = 'fork of vcs test'
         repo_name = self.REPO
         org_repo = Repository.get_by_repo_name(repo_name)
         creation_args = {
             'repo_name': fork_name,
             'repo_group': u'-1',
             'fork_parent_id': org_repo.repo_id,
             'repo_type': self.REPO_TYPE,
             'description': description,
             'private': 'False',
             'landing_rev': 'rev:tip',
             '_session_csrf_secret_token': self.session_csrf_secret_token()}
         self.app.post(base.url(controller='forks', action='fork_create',
                           repo_name=repo_name), creation_args)
         repo = Repository.get_by_repo_name(self.REPO_FORK)
         assert repo.fork.repo_name == self.REPO
         ## run the check page that triggers the flash message
         response = self.app.get(base.url('repo_check_home', repo_name=fork_name))
         # test if we have a message that fork is ok
         self.checkSessionFlash(response,
                 'Forked repository %s as <a href="/%s">%s</a>'
                 % (repo_name, fork_name, fork_name))
         # test if the fork was created in the database
         fork_repo = Session().query(Repository) \
             .filter(Repository.repo_name == fork_name).one()
         assert fork_repo.repo_name == fork_name
         assert fork_repo.fork.repo_name == repo_name
         # test if the repository is visible in the list ?
         response = self.app.get(base.url('summary_home', repo_name=fork_name))
         response.mustcontain(fork_name)
         response.mustcontain(self.REPO_TYPE)
         response.mustcontain('Fork of "<a href="/%s">%s</a>"' % (repo_name, repo_name))
         usr = self.log_user(self.username, self.password)['user_id']
         forks = Repository.query() \
             .filter(Repository.repo_type == self.REPO_TYPE) \
             .filter(Repository.fork_id != None).all()
         assert 1 == len(forks)
         # set read permissions for this
         RepoModel().grant_user_permission(repo=forks[0],
                                           user=usr,
                                           perm='repository.read')
         Session().commit()
         response = self.app.get(base.url(controller='forks', action='forks',
                                     repo_name=repo_name))
         response.mustcontain('<div>fork of vcs test</div>')
         # remove permissions
         default_user = User.get_default_user()
         try:
             RepoModel().grant_user_permission(repo=forks[0],
                                               user=usr, perm='repository.none')
             RepoModel().grant_user_permission(repo=forks[0],
                                               user=default_user, perm='repository.none')
             Session().commit()
             # fork shouldn't be visible
             response = self.app.get(base.url(controller='forks', action='forks',
                                         repo_name=repo_name))
             response.mustcontain('There are no forks yet')
         finally:
             RepoModel().grant_user_permission(repo=forks[0],
                                               user=usr, perm='repository.read')
             RepoModel().grant_user_permission(repo=forks[0],
                                               user=default_user, perm='repository.read')
             RepoModel().delete(repo=forks[0])
 class TestGIT(_BaseTestCase):
     REPO = base.GIT_REPO
     NEW_REPO = base.NEW_GIT_REPO
     REPO_TYPE = 'git'
     REPO_FORK = base.GIT_FORK
 class TestHG(_BaseTestCase):

kallithea/tests/other/test_vcs_operations.py

➞

Show inline comments

 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
+#
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 Test suite for vcs push/pull operations.
 The tests need Git > 1.8.1.
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Dec 30, 2010
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 from __future__ import print_function
 import json
 import os
 import re
 import tempfile
 import time
-import urllib2
+import urllib.request
 from subprocess import PIPE, Popen
 from tempfile import _RandomNameSequence
 import pytest
 from kallithea import CONFIG
 from kallithea.lib.utils2 import ascii_bytes
 from kallithea.model.db import CacheInvalidation, Repository, Ui, User, UserIpMap, UserLog
 from kallithea.model.meta import Session
 from kallithea.model.ssh_key import SshKeyModel
 from kallithea.model.user import UserModel
 from kallithea.tests import base
 from kallithea.tests.fixture import Fixture
 DEBUG = True
 HOST = '127.0.0.1:4999'  # test host
 fixture = Fixture()
 # Parameterize different kinds of VCS testing - both the kind of VCS and the
 # access method (HTTP/SSH)
 # Mixin for using HTTP and SSH URLs
 class HttpVcsTest(object):
     @staticmethod
     def repo_url_param(webserver, repo_name, **kwargs):
         return webserver.repo_url(repo_name, **kwargs)
 class SshVcsTest(object):
     public_keys = {
         base.TEST_USER_REGULAR_LOGIN: u'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAgQC6Ycnc2oUZHQnQwuqgZqTTdMDZD7ataf3JM7oG2Fw8JR6cdmz4QZLe5mfDwaFwG2pWHLRpVqzfrD/Pn3rIO++bgCJH5ydczrl1WScfryV1hYMJ/4EzLGM657J1/q5EI+b9SntKjf4ax+KP322L0TNQGbZUHLbfG2MwHMrYBQpHUQ== kallithea@localhost',
         base.TEST_USER_ADMIN_LOGIN: u'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAgQC6Ycnc2oUZHQnQwuqgZqTTdMDZD7ataf3JM7oG2Fw8JR6cdmz4QZLe5mfDwaFwG2pWHLRpVqzfrD/Pn3rIO++bgCJH5ydczrl1WScfryV1hYMJ/4EzLGM657J1/q5EI+b9SntKjf4ax+KP322L0TNQGbZUHLbfG2MwHMrYBQpHUq== kallithea@localhost',
+    }
     @classmethod
     def repo_url_param(cls, webserver, repo_name, username=base.TEST_USER_ADMIN_LOGIN, password=base.TEST_USER_ADMIN_PASS, client_ip=base.IP_ADDR):
         user = User.get_by_username(username)
         if user.ssh_keys:
             ssh_key = user.ssh_keys[0]
         else:
             sshkeymodel = SshKeyModel()
             ssh_key = sshkeymodel.create(user, u'test key', cls.public_keys[user.username])
             Session().commit()
         return cls._ssh_param(repo_name, user, ssh_key, client_ip)
 # Mixins for using Mercurial and Git
 class HgVcsTest(object):
     repo_type = 'hg'
     repo_name = base.HG_REPO
 class GitVcsTest(object):
     repo_type = 'git'
     repo_name = base.GIT_REPO
 # Combine mixins to give the combinations we want to parameterize tests with
 class HgHttpVcsTest(HgVcsTest, HttpVcsTest):
     pass
 class GitHttpVcsTest(GitVcsTest, HttpVcsTest):
     pass
 class HgSshVcsTest(HgVcsTest, SshVcsTest):
     @staticmethod
     def _ssh_param(repo_name, user, ssh_key, client_ip):
         # Specify a custom ssh command on the command line
         return r"""--config ui.ssh="bash -c 'SSH_ORIGINAL_COMMAND=\"\$2\" SSH_CONNECTION=\"%s 1024 127.0.0.1 22\" kallithea-cli ssh-serve -c %s %s %s' --" ssh://someuser@somehost/%s""" % (
             client_ip,
             CONFIG['__file__'],
             user.user_id,
             ssh_key.user_ssh_key_id,
             repo_name)
 class GitSshVcsTest(GitVcsTest, SshVcsTest):
     @staticmethod
     def _ssh_param(repo_name, user, ssh_key, client_ip):
         # Set a custom ssh command in the global environment
         os.environ['GIT_SSH_COMMAND'] = r"""bash -c 'SSH_ORIGINAL_COMMAND="$2" SSH_CONNECTION="%s 1024 127.0.0.1 22" kallithea-cli ssh-serve -c %s %s %s' --""" % (
             client_ip,
             CONFIG['__file__'],
             user.user_id,
             ssh_key.user_ssh_key_id)
         return "ssh://someuser@somehost/%s""" % repo_name
 parametrize_vcs_test = base.parametrize('vt', [
     HgHttpVcsTest,
     GitHttpVcsTest,
     HgSshVcsTest,
     GitSshVcsTest,
 ])
 parametrize_vcs_test_hg = base.parametrize('vt', [
     HgHttpVcsTest,
     HgSshVcsTest,
 ])
@@ @@ -236,197 +236,197 @@ def set_anonymous_access(enable=True): @@
 def _check_proper_git_push(stdout, stderr):
     assert 'fatal' not in stderr
     assert 'rejected' not in stderr
     assert 'Pushing to' in stderr
     assert 'master -> master' in stderr
 @pytest.mark.usefixtures("test_context_fixture")
 class TestVCSOperations(base.TestController):
     @classmethod
     def setup_class(cls):
         # DISABLE ANONYMOUS ACCESS
         set_anonymous_access(False)
     @pytest.fixture()
     def testhook_cleanup(self):
         yield
         # remove hook
         for hook in ['prechangegroup', 'pretxnchangegroup', 'preoutgoing', 'changegroup', 'outgoing', 'incoming']:
             entry = Ui.get_by_key('hooks', '%s.testhook' % hook)
             if entry:
                 Session().delete(entry)
         Session().commit()
     @pytest.fixture(scope="module")
     def testfork(self):
         # create fork so the repo stays untouched
         git_fork_name = u'%s_fork%s' % (base.GIT_REPO, next(_RandomNameSequence()))
         fixture.create_fork(base.GIT_REPO, git_fork_name)
         hg_fork_name = u'%s_fork%s' % (base.HG_REPO, next(_RandomNameSequence()))
         fixture.create_fork(base.HG_REPO, hg_fork_name)
         return {'git': git_fork_name, 'hg': hg_fork_name}
     @parametrize_vcs_test
     def test_clone_repo_by_admin(self, webserver, vt):
         clone_url = vt.repo_url_param(webserver, vt.repo_name)
         stdout, stderr = Command(base.TESTS_TMP_PATH).execute(vt.repo_type, 'clone', clone_url, _get_tmp_dir())
         if vt.repo_type == 'git':
             assert 'Cloning into' in stdout + stderr
             assert stderr == '' or stdout == ''
         elif vt.repo_type == 'hg':
             assert 'requesting all changes' in stdout
             assert 'adding changesets' in stdout
             assert 'adding manifests' in stdout
             assert 'adding file changes' in stdout
             assert stderr == ''
     @parametrize_vcs_test_http
     def test_clone_wrong_credentials(self, webserver, vt):
         clone_url = vt.repo_url_param(webserver, vt.repo_name, password='bad!')
         stdout, stderr = Command(base.TESTS_TMP_PATH).execute(vt.repo_type, 'clone', clone_url, _get_tmp_dir(), ignoreReturnCode=True)
         if vt.repo_type == 'git':
             assert 'fatal: Authentication failed' in stderr
         elif vt.repo_type == 'hg':
             assert 'abort: authorization failed' in stderr
     def test_clone_git_dir_as_hg(self, webserver):
         clone_url = HgHttpVcsTest.repo_url_param(webserver, base.GIT_REPO)
         stdout, stderr = Command(base.TESTS_TMP_PATH).execute('hg clone', clone_url, _get_tmp_dir(), ignoreReturnCode=True)
         assert 'HTTP Error 404: Not Found' in stderr or "not a valid repository" in stdout and 'abort:' in stderr
     def test_clone_hg_repo_as_git(self, webserver):
         clone_url = GitHttpVcsTest.repo_url_param(webserver, base.HG_REPO)
         stdout, stderr = Command(base.TESTS_TMP_PATH).execute('git clone', clone_url, _get_tmp_dir(), ignoreReturnCode=True)
         assert 'not found' in stderr
     @parametrize_vcs_test
     def test_clone_non_existing_path(self, webserver, vt):
         clone_url = vt.repo_url_param(webserver, 'trololo')
         stdout, stderr = Command(base.TESTS_TMP_PATH).execute(vt.repo_type, 'clone', clone_url, _get_tmp_dir(), ignoreReturnCode=True)
         if vt.repo_type == 'git':
             assert 'not found' in stderr or 'abort: Access to %r denied' % 'trololo' in stderr
         elif vt.repo_type == 'hg':
             assert 'HTTP Error 404: Not Found' in stderr or 'abort: no suitable response from remote hg' in stderr and 'remote: abort: Access to %r denied' % 'trololo' in stdout
     @parametrize_vcs_test
     def test_push_new_repo(self, webserver, vt):
         # Clear the log so we know what is added
         UserLog.query().delete()
         Session().commit()
         # Create an empty server repo using the API
         repo_name = u'new_%s_%s' % (vt.repo_type, next(_RandomNameSequence()))
         usr = User.get_by_username(base.TEST_USER_ADMIN_LOGIN)
         params = {
             "id": 7,
             "api_key": usr.api_key,
             "method": 'create_repo',
             "args": dict(repo_name=repo_name,
                          owner=base.TEST_USER_ADMIN_LOGIN,
                          repo_type=vt.repo_type),
+        }
-        req = urllib2.Request(
+        req = urllib.request.Request(
             'http://%s:%s/_admin/api' % webserver.server_address,
             data=ascii_bytes(json.dumps(params)),
             headers={'content-type': 'application/json'})
-        response = urllib2.urlopen(req)
+        response = urllib.request.urlopen(req)
         result = json.loads(response.read())
         # Expect something like:
         # {u'result': {u'msg': u'Created new repository `new_XXX`', u'task': None, u'success': True}, u'id': 7, u'error': None}
         assert result[u'result'][u'success']
         # Create local clone of the empty server repo
         local_clone_dir = _get_tmp_dir()
         clone_url = vt.repo_url_param(webserver, repo_name)
         stdout, stderr = Command(base.TESTS_TMP_PATH).execute(vt.repo_type, 'clone', clone_url, local_clone_dir)
         # Make 3 commits and push to the empty server repo.
         # The server repo doesn't have any other heads than the
         # refs/heads/master we are pushing, but the `git log` in the push hook
         # should still list the 3 commits.
         stdout, stderr = _add_files_and_push(webserver, vt, local_clone_dir, clone_url=clone_url)
         if vt.repo_type == 'git':
             _check_proper_git_push(stdout, stderr)
         elif vt.repo_type == 'hg':
             assert 'pushing to ' in stdout
             assert 'remote: added ' in stdout
         # Verify that we got the right events in UserLog. Expect something like:
         # <UserLog('id:new_git_XXX:started_following_repo')>
         # <UserLog('id:new_git_XXX:user_created_repo')>
         # <UserLog('id:new_git_XXX:pull')>
         # <UserLog('id:new_git_XXX:push:aed9d4c1732a1927da3be42c47eb9afdc200d427,d38b083a07af10a9f44193486959a96a23db78da,4841ff9a2b385bec995f4679ef649adb3f437622')>
         action_parts = [ul.action.split(':', 1) for ul in UserLog.query().order_by(UserLog.user_log_id)]
         assert [(t[0], (t[1].count(',') + 1) if len(t) == 2 else 0) for t in action_parts] == ([
             (u'started_following_repo', 0),
             (u'user_created_repo', 0),
             (u'pull', 0),
             (u'push', 3)]
             if vt.repo_type == 'git' else [
             (u'started_following_repo', 0),
             (u'user_created_repo', 0),
             # (u'pull', 0), # Mercurial outgoing hook is not called for empty clones
             (u'push', 3)])
     @parametrize_vcs_test
     def test_push_new_file(self, webserver, testfork, vt):
         UserLog.query().delete()
         Session().commit()
         dest_dir = _get_tmp_dir()
         clone_url = vt.repo_url_param(webserver, vt.repo_name)
         stdout, stderr = Command(base.TESTS_TMP_PATH).execute(vt.repo_type, 'clone', clone_url, dest_dir)
         clone_url = vt.repo_url_param(webserver, testfork[vt.repo_type])
         stdout, stderr = _add_files_and_push(webserver, vt, dest_dir, clone_url=clone_url)
         if vt.repo_type == 'git':
             _check_proper_git_push(stdout, stderr)
         elif vt.repo_type == 'hg':
             assert 'pushing to' in stdout
             assert 'Repository size' in stdout
             assert 'Last revision is now' in stdout
         action_parts = [ul.action.split(':', 1) for ul in UserLog.query().order_by(UserLog.user_log_id)]
         assert [(t[0], (t[1].count(',') + 1) if len(t) == 2 else 0) for t in action_parts] == \
             [(u'pull', 0), (u'push', 3)]
     @parametrize_vcs_test
     def test_pull(self, webserver, testfork, vt):
         UserLog.query().delete()
         Session().commit()
         dest_dir = _get_tmp_dir()
         stdout, stderr = Command(base.TESTS_TMP_PATH).execute(vt.repo_type, 'init', dest_dir)
         clone_url = vt.repo_url_param(webserver, vt.repo_name)
         stdout, stderr = Command(dest_dir).execute(vt.repo_type, 'pull', clone_url)
         if vt.repo_type == 'git':
             assert 'FETCH_HEAD' in stderr
         elif vt.repo_type == 'hg':
             assert 'new changesets' in stdout
         action_parts = [ul.action for ul in UserLog.query().order_by(UserLog.user_log_id)]
         assert action_parts == [u'pull']
         # Test handling of URLs with extra '/' around repo_name
         stdout, stderr = Command(dest_dir).execute(vt.repo_type, 'pull', clone_url.replace('/' + vt.repo_name, '/./%s/' % vt.repo_name), ignoreReturnCode=True)
         if issubclass(vt, HttpVcsTest):
             if vt.repo_type == 'git':
                 # NOTE: when pulling from http://hostname/./vcs_test_git/ , the git client will normalize that and issue an HTTP request to /vcs_test_git/info/refs
                 assert 'Already up to date.' in stdout
             else:
                 assert vt.repo_type == 'hg'
                 assert "abort: HTTP Error 404: Not Found" in stderr
         else:
             assert issubclass(vt, SshVcsTest)
             if vt.repo_type == 'git':
                 assert "abort: Access to './%s' denied" % vt.repo_name in stderr
             else:
                 assert "abort: Access to './%s' denied" % vt.repo_name in stdout

kallithea/tests/scripts/manual_test_crawler.py

➞

Show inline comments

 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
+#
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
+#
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 kallithea.tests.scripts.manual_test_crawler
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 Test for crawling a project for memory usage
 This should be runned just as regular script together
 with a watch script that will show memory usage.
 watch -n1 ./kallithea/tests/mem_watch
 This file was forked by the Kallithea project in July 2014.
 Original author and date, and relevant copyright and licensing information is below:
 :created_on: Apr 21, 2010
 :author: marcink
 :copyright: (c) 2013 RhodeCode GmbH, and others.
 :license: GPLv3, see LICENSE.md for more details.
 """
 from __future__ import print_function
 import http.cookiejar
 import os
 import sys
 import tempfile
 import time
 import urllib
 import urllib2
 import urllib.parse
 import urllib.request
 from os.path import dirname
 from kallithea.lib import vcs
 from kallithea.lib.compat import OrderedSet
 from kallithea.lib.vcs.exceptions import RepositoryError
 __here__ = os.path.abspath(__file__)
 __root__ = dirname(dirname(dirname(__here__)))
 sys.path.append(__root__)
 PASES = 3
 HOST = 'http://127.0.0.1'
 PORT = 5000
 BASE_URI = '%s:%s/' % (HOST, PORT)
 if len(sys.argv) == 2:
     BASE_URI = sys.argv[1]
 if not BASE_URI.endswith('/'):
     BASE_URI += '/'
 print('Crawling @ %s' % BASE_URI)
 BASE_URI += '%s'
 PROJECT_PATH = os.path.join('/', 'home', 'username', 'repos')
 PROJECTS = [
     # 'linux-magx-pbranch',
     'CPython',
     'kallithea',
+]
 cj = http.cookiejar.FileCookieJar(os.path.join(tempfile.gettempdir(), 'rc_test_cookie.txt'))
-o = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj))
+o = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(cj))
 o.addheaders = [
     ('User-agent', 'kallithea-crawler'),
     ('Accept-Language', 'en - us, en;q = 0.5')
+]
-urllib2.install_opener(o)
+urllib.request.install_opener(o)
 def _get_repo(proj):
     if isinstance(proj, str):
         repo = vcs.get_repo(os.path.join(PROJECT_PATH, proj))
         proj = proj
     else:
         repo = proj
         proj = repo.name
     return repo, proj
 def test_changelog_walk(proj, pages=100):
     repo, proj = _get_repo(proj)
     total_time = 0
     for i in range(1, pages):
         page = '/'.join((proj, 'changelog',))
         full_uri = (BASE_URI % page) + '?' + urllib.urlencode({'page': i})
+        full_uri = (BASE_URI % page) + '?' + urllib.parse.urlencode({'page': i})
         s = time.time()
         f = o.open(full_uri)
         assert f.url == full_uri, 'URL:%s does not match %s' % (f.url, full_uri)
         size = len(f.read())
         e = time.time() - s
         total_time += e
         print('visited %s size:%s req:%s ms' % (full_uri, size, e))
     print('total_time', total_time)
     print('average on req', total_time / float(pages))
 def test_changeset_walk(proj, limit=None):
     repo, proj = _get_repo(proj)
     print('processing', os.path.join(PROJECT_PATH, proj))
     total_time = 0
     cnt = 0
     for i in repo:
         cnt += 1
         raw_cs = '/'.join((proj, 'changeset', i.raw_id))
         if limit and limit == cnt:
             break
         full_uri = (BASE_URI % raw_cs)
         print('%s visiting %s/%s' % (cnt, full_uri, i))
         s = time.time()
         f = o.open(full_uri)
         size = len(f.read())
         e = time.time() - s
         total_time += e
         print('%s visited %s/%s size:%s req:%s ms' % (cnt, full_uri, i, size, e))
     print('total_time', total_time)
     print('average on req', total_time / float(cnt))
 def test_files_walk(proj, limit=100):
     repo, proj = _get_repo(proj)
     print('processing', os.path.join(PROJECT_PATH, proj))
     total_time = 0
     paths_ = OrderedSet([''])
     try:
         tip = repo.get_changeset('tip')
         for topnode, dirs, files in tip.walk('/'):
             for dir in dirs:
                 paths_.add(dir.path)
                 for f in dir:
                     paths_.add(f.path)
             for f in files:
                 paths_.add(f.path)
     except RepositoryError as e:
         pass
     cnt = 0
     for f in paths_:
         cnt += 1
         if limit and limit == cnt:
             break
         file_path = '/'.join((proj, 'files', 'tip', f))
         full_uri = (BASE_URI % file_path)
         print('%s visiting %s' % (cnt, full_uri))
         s = time.time()
         f = o.open(full_uri)
         size = len(f.read())
         e = time.time() - s
         total_time += e
         print('%s visited OK size:%s req:%s ms' % (cnt, size, e))
     print('total_time', total_time)
     print('average on req', total_time / float(cnt))
 if __name__ == '__main__':
     for path in PROJECTS:
         repo = vcs.get_repo(os.path.join(PROJECT_PATH, path))
         for i in range(PASES):
             print('PASS %s/%s' % (i, PASES))
             test_changelog_walk(repo, pages=80)
             test_changeset_walk(repo, limit=100)
             test_files_walk(repo, limit=100)

kallithea/tests/vcs/test_git.py

➞

Show inline comments

 import datetime
 import os
 import sys
-import urllib2
+import urllib.error
 import mock
 import pytest
 from kallithea.lib.vcs.backends.git import GitChangeset, GitRepository
 from kallithea.lib.vcs.exceptions import NodeDoesNotExistError, RepositoryError, VCSError
 from kallithea.lib.vcs.nodes import DirNode, FileNode, NodeKind, NodeState
 from kallithea.model.scm import ScmModel
 from kallithea.tests.vcs.base import _BackendTestMixin
 from kallithea.tests.vcs.conf import TEST_GIT_REPO, TEST_GIT_REPO_CLONE, TESTS_TMP_PATH, get_new_dir
 class TestGitRepository(object):
     def __check_for_existing_repo(self):
         if os.path.exists(TEST_GIT_REPO_CLONE):
             pytest.fail('Cannot test git clone repo as location %s already '
                       'exists. You should manually remove it first.'
                       % TEST_GIT_REPO_CLONE)
     def setup_method(self):
         self.repo = GitRepository(TEST_GIT_REPO)
     def test_wrong_repo_path(self):
         wrong_repo_path = os.path.join(TESTS_TMP_PATH, 'errorrepo')
         with pytest.raises(RepositoryError):
             GitRepository(wrong_repo_path)
     def test_git_cmd_injection(self):
         repo_inject_path = TEST_GIT_REPO + '; echo "Cake";'
-        with pytest.raises(urllib2.URLError):
+        with pytest.raises(urllib.error.URLError):
             # Should fail because URL will contain the parts after ; too
             GitRepository(get_new_dir('injection-repo'), src_url=repo_inject_path, update_after_clone=True, create=True)
         with pytest.raises(RepositoryError):
             # Should fail on direct clone call, which as of this writing does not happen outside of class
             clone_fail_repo = GitRepository(get_new_dir('injection-repo'), create=True)
             clone_fail_repo.clone(repo_inject_path, update_after_clone=True,)
         # Verify correct quoting of evil characters that should work on posix file systems
         if sys.platform == 'win32':
             # windows does not allow '"' in dir names
             # and some versions of the git client don't like ` and '
             tricky_path = get_new_dir("tricky-path-repo-$")
         else:
             tricky_path = get_new_dir("tricky-path-repo-$'\"`")
         successfully_cloned = GitRepository(tricky_path, src_url=TEST_GIT_REPO, update_after_clone=True, create=True)
         # Repo should have been created
         assert not successfully_cloned._repo.bare
         if sys.platform == 'win32':
             # windows does not allow '"' in dir names
             # and some versions of the git client don't like ` and '
             tricky_path_2 = get_new_dir("tricky-path-2-repo-$")
         else:
             tricky_path_2 = get_new_dir("tricky-path-2-repo-$'\"`")
         successfully_cloned2 = GitRepository(tricky_path_2, src_url=tricky_path, bare=True, create=True)
         # Repo should have been created and thus used correct quoting for clone
         assert successfully_cloned2._repo.bare
         # Should pass because URL has been properly quoted
         successfully_cloned.pull(tricky_path_2)
         successfully_cloned2.fetch(tricky_path)
     def test_repo_create_with_spaces_in_path(self):
         repo_path = get_new_dir("path with spaces")
         repo = GitRepository(repo_path, src_url=None, bare=True, create=True)
         # Repo should have been created
         assert repo._repo.bare
     def test_repo_clone(self):
         self.__check_for_existing_repo()
         repo = GitRepository(TEST_GIT_REPO)
         repo_clone = GitRepository(TEST_GIT_REPO_CLONE,
             src_url=TEST_GIT_REPO, create=True, update_after_clone=True)
         assert len(repo.revisions) == len(repo_clone.revisions)
         # Checking hashes of changesets should be enough
         for changeset in repo.get_changesets():
             raw_id = changeset.raw_id
             assert raw_id == repo_clone.get_changeset(raw_id).raw_id
     def test_repo_clone_with_spaces_in_path(self):
         repo_path = get_new_dir("path with spaces")
         successfully_cloned = GitRepository(repo_path, src_url=TEST_GIT_REPO, update_after_clone=True, create=True)
         # Repo should have been created
         assert not successfully_cloned._repo.bare
         successfully_cloned.pull(TEST_GIT_REPO)
         self.repo.fetch(repo_path)
     def test_repo_clone_without_create(self):
         with pytest.raises(RepositoryError):
             GitRepository(TEST_GIT_REPO_CLONE + '_wo_create', src_url=TEST_GIT_REPO)
     def test_repo_clone_with_update(self):
         repo = GitRepository(TEST_GIT_REPO)
         clone_path = TEST_GIT_REPO_CLONE + '_with_update'
         repo_clone = GitRepository(clone_path,
             create=True, src_url=TEST_GIT_REPO, update_after_clone=True)
         assert len(repo.revisions) == len(repo_clone.revisions)
         # check if current workdir was updated
         fpath = os.path.join(clone_path, 'MANIFEST.in')
         assert os.path.isfile(fpath) == True, 'Repo was cloned and updated but file %s could not be found' % fpath
     def test_repo_clone_without_update(self):
         repo = GitRepository(TEST_GIT_REPO)
         clone_path = TEST_GIT_REPO_CLONE + '_without_update'
         repo_clone = GitRepository(clone_path,
             create=True, src_url=TEST_GIT_REPO, update_after_clone=False)
         assert len(repo.revisions) == len(repo_clone.revisions)
         # check if current workdir was *NOT* updated
         fpath = os.path.join(clone_path, 'MANIFEST.in')
         # Make sure it's not bare repo
         assert not repo_clone._repo.bare
         assert os.path.isfile(fpath) == False, 'Repo was cloned and updated but file %s was found' % fpath
     def test_repo_clone_into_bare_repo(self):
         repo = GitRepository(TEST_GIT_REPO)
         clone_path = TEST_GIT_REPO_CLONE + '_bare.git'
         repo_clone = GitRepository(clone_path, create=True,
             src_url=repo.path, bare=True)
         assert repo_clone._repo.bare
     def test_create_repo_is_not_bare_by_default(self):
         repo = GitRepository(get_new_dir('not-bare-by-default'), create=True)
         assert not repo._repo.bare

0 comments (0 inline, 0 general)