Changeset - 22b40db44a41
[Not reviewed]
default
0 13 0
Mads Kiilerich - 6 years ago 2019-11-25 03:43:15
mads@kiilerich.com
Grafted from: 546b7a8a3341
py3: migrate from urllib2 to urllib

Based on 2to3 urllib, but with unnecessary imports removed.
13 files changed with 58 insertions and 55 deletions:
0 comments (0 inline, 0 general)
kallithea/bin/base.py
Show inline comments
 
@@ -26,13 +26,13 @@ Original author and date, and relevant c
 
"""
 

	
 
import os
 
import pprint
 
import random
 
import sys
 
import urllib2
 
import urllib.request
 

	
 
from kallithea.lib import ext_json
 
from kallithea.lib.utils2 import ascii_bytes
 

	
 

	
 
CONFIG_NAME = '.config/kallithea'
 
@@ -65,16 +65,16 @@ def api_call(apikey, apihost, method=Non
 
        }
 

	
 
    if not method:
 
        raise Exception('please specify method name !')
 
    apihost = apihost.rstrip('/')
 
    id_ = random.randrange(1, 9999)
 
    req = urllib2.Request('%s/_admin/api' % apihost,
 
    req = urllib.request.Request('%s/_admin/api' % apihost,
 
                      data=ascii_bytes(ext_json.dumps(_build_data(id_))),
 
                      headers={'content-type': 'text/plain'})
 
    ret = urllib2.urlopen(req)
 
    ret = urllib.request.urlopen(req)
 
    raw_json = ret.read()
 
    json_data = ext_json.loads(raw_json)
 
    id_ret = json_data['id']
 
    if id_ret == id_:
 
        return json_data
 

	
kallithea/bin/ldap_sync.py
Show inline comments
 
@@ -24,13 +24,13 @@ Original author and date, and relevant c
 
:copyright: (c) 2013 RhodeCode GmbH, and others.
 
:license: GPLv3, see LICENSE.md for more details.
 
"""
 

	
 
from __future__ import print_function
 

	
 
import urllib2
 
import urllib.request
 
import uuid
 
from configparser import ConfigParser
 

	
 
import ldap
 

	
 
from kallithea.lib import ext_json
 
@@ -80,15 +80,15 @@ class API(object):
 
        """
 
        uid = str(uuid.uuid1())
 
        data = self.get_api_data(uid, method, args)
 

	
 
        data = ascii_bytes(ext_json.dumps(data))
 
        headers = {'content-type': 'text/plain'}
 
        req = urllib2.Request(self.url, data, headers)
 
        req = urllib.request.Request(self.url, data, headers)
 

	
 
        response = urllib2.urlopen(req)
 
        response = urllib.request.urlopen(req)
 
        response = ext_json.load(response)
 

	
 
        if uid != response["id"]:
 
            raise InvalidResponseIDError("UUID does not match.")
 

	
 
        if response["error"] is not None:
kallithea/lib/auth_modules/auth_crowd.py
Show inline comments
 
@@ -25,13 +25,14 @@ Original author and date, and relevant c
 
:license: GPLv3, see LICENSE.md for more details.
 
"""
 

	
 

	
 
import base64
 
import logging
 
import urllib2
 
import urllib.parse
 
import urllib.request
 

	
 
from kallithea.lib import auth_modules, ext_json
 
from kallithea.lib.compat import hybrid_property
 
from kallithea.lib.utils2 import ascii_bytes, ascii_str, safe_bytes
 

	
 

	
 
@@ -69,29 +70,29 @@ class CrowdServer(object):
 
    def set_credentials(self, user, passwd):
 
        self.user = user
 
        self.passwd = passwd
 
        self._make_opener()
 

	
 
    def _make_opener(self):
 
        mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
 
        mgr = urllib.request.HTTPPasswordMgrWithDefaultRealm()
 
        mgr.add_password(None, self._uri, self.user, self.passwd)
 
        handler = urllib2.HTTPBasicAuthHandler(mgr)
 
        self.opener = urllib2.build_opener(handler)
 
        handler = urllib.request.HTTPBasicAuthHandler(mgr)
 
        self.opener = urllib.request.build_opener(handler)
 

	
 
    def _request(self, url, body=None, headers=None,
 
                 method=None, noformat=False,
 
                 empty_response_ok=False):
 
        _headers = {"Content-type": "application/json",
 
                    "Accept": "application/json"}
 
        if self.user and self.passwd:
 
            authstring = ascii_str(base64.b64encode(safe_bytes("%s:%s" % (self.user, self.passwd))))
 
            _headers["Authorization"] = "Basic %s" % authstring
 
        if headers:
 
            _headers.update(headers)
 
        log.debug("Sent to crowd at %s:\nHeaders: %s\nBody:\n%s", url, _headers, body)
 
        req = urllib2.Request(url, body, _headers)
 
        req = urllib.request.Request(url, body, _headers)
 
        if method:
 
            req.get_method = lambda: method
 

	
 
        global msg
 
        msg = ""
 
        try:
 
@@ -116,20 +117,20 @@ class CrowdServer(object):
 
        return rval
 

	
 
    def user_auth(self, username, password):
 
        """Authenticate a user against crowd. Returns brief information about
 
        the user."""
 
        url = ("%s/rest/usermanagement/%s/authentication?username=%s"
 
               % (self._uri, self._version, urllib2.quote(username)))
 
               % (self._uri, self._version, urllib.parse.quote(username)))
 
        body = ascii_bytes(ext_json.dumps({"value": password}))
 
        return self._request(url, body)
 

	
 
    def user_groups(self, username):
 
        """Retrieve a list of groups to which this user belongs."""
 
        url = ("%s/rest/usermanagement/%s/user/group/nested?username=%s"
 
               % (self._uri, self._version, urllib2.quote(username)))
 
               % (self._uri, self._version, urllib.parse.quote(username)))
 
        return self._request(url)
 

	
 

	
 
class KallitheaAuthPlugin(auth_modules.KallitheaExternalAuthPlugin):
 
    def __init__(self):
 
        self._protocol_values = ["http", "https"]
kallithea/lib/middleware/simplehg.py
Show inline comments
 
@@ -27,13 +27,13 @@ Original author and date, and relevant c
 

	
 
"""
 

	
 

	
 
import logging
 
import os
 
import urllib
 
import urllib.parse
 

	
 
import mercurial.hgweb
 

	
 
from kallithea.lib.base import BaseVCSController, get_path_info
 
from kallithea.lib.utils import make_ui
 
from kallithea.lib.utils2 import safe_bytes, safe_str, safe_unicode
 
@@ -118,13 +118,13 @@ class SimpleHg(BaseVCSController):
 
                        hgarg = get_header_hgarg(environ)
 
                        if not hgarg.startswith('cmds='):
 
                            action = 'push' # paranoid and safe
 
                            break
 
                        action = 'pull'
 
                        for cmd_arg in hgarg[5:].split(';'):
 
                            cmd, _args = urllib.unquote_plus(cmd_arg).split(' ', 1)
 
                            cmd, _args = urllib.parse.unquote_plus(cmd_arg).split(' ', 1)
 
                            op = cmd_mapping.get(cmd, 'push')
 
                            if op != 'pull':
 
                                assert op == 'push'
 
                                action = 'push'
 
                                break
 
                    else:
kallithea/lib/recaptcha.py
Show inline comments
 
# -*- coding: utf-8 -*-
 
import json
 
import urllib
 
import urllib2
 
import urllib.parse
 
import urllib.request
 

	
 

	
 
class RecaptchaResponse(object):
 
    def __init__(self, is_valid, error_code=None):
 
        self.is_valid = is_valid
 
        self.error_code = error_code
 
@@ -27,28 +27,28 @@ def submit(g_recaptcha_response, private
 

	
 
    def encode_if_necessary(s):
 
        if isinstance(s, unicode):
 
            return s.encode('utf-8')
 
        return s
 

	
 
    params = urllib.urlencode({
 
    params = urllib.parse.urlencode({
 
        'secret': encode_if_necessary(private_key),
 
        'remoteip': encode_if_necessary(remoteip),
 
        'response': encode_if_necessary(g_recaptcha_response),
 
    }).encode('ascii')
 

	
 
    req = urllib2.Request(
 
    req = urllib.request.Request(
 
        url="https://www.google.com/recaptcha/api/siteverify",
 
        data=params,
 
        headers={
 
            "Content-type": "application/x-www-form-urlencoded",
 
            "User-agent": "reCAPTCHA Python"
 
        }
 
    )
 

	
 
    httpresp = urllib2.urlopen(req)
 
    httpresp = urllib.request.urlopen(req)
 
    return_values = json.loads(httpresp.read())
 
    httpresp.close()
 

	
 
    if not (isinstance(return_values, dict)):
 
        return RecaptchaResponse(is_valid=False, error_code='incorrect-captcha-sol')
 
    elif (("success" in return_values) and ((return_values["success"] is True) or (return_values["success"] == "true"))):
kallithea/lib/utils2.py
Show inline comments
 
@@ -33,13 +33,13 @@ import binascii
 
import datetime
 
import json
 
import os
 
import pwd
 
import re
 
import time
 
import urllib
 
import urllib.parse
 

	
 
import urlobject
 
from tg.i18n import ugettext as _
 
from tg.i18n import ungettext
 
from webhelpers2.text import collapse, remove_formatting, strip_tags
 

	
 
@@ -319,20 +319,20 @@ def credentials_filter(uri):
 

	
 
    return ''.join(uri)
 

	
 

	
 
def get_clone_url(clone_uri_tmpl, prefix_url, repo_name, repo_id, username=None):
 
    parsed_url = urlobject.URLObject(prefix_url)
 
    prefix = safe_unicode(urllib.unquote(parsed_url.path.rstrip('/')))
 
    prefix = safe_unicode(urllib.parse.unquote(parsed_url.path.rstrip('/')))
 
    try:
 
        system_user = pwd.getpwuid(os.getuid()).pw_name
 
    except Exception: # TODO: support all systems - especially Windows
 
        system_user = 'kallithea' # hardcoded default value ...
 
    args = {
 
        'scheme': parsed_url.scheme,
 
        'user': safe_unicode(urllib.quote(safe_str(username or ''))),
 
        'user': safe_unicode(urllib.parse.quote(safe_str(username or ''))),
 
        'netloc': parsed_url.netloc + prefix,  # like "hostname:port/prefix" (with optional ":port" and "/prefix")
 
        'prefix': prefix, # undocumented, empty or starting with /
 
        'repo': repo_name,
 
        'repoid': str(repo_id),
 
        'system_user': safe_unicode(system_user),
 
        'hostname': parsed_url.hostname,
kallithea/lib/vcs/backends/git/repository.py
Show inline comments
 
@@ -11,14 +11,15 @@
 

	
 
import errno
 
import logging
 
import os
 
import re
 
import time
 
import urllib
 
import urllib2
 
import urllib.error
 
import urllib.parse
 
import urllib.request
 
from collections import OrderedDict
 

	
 
import mercurial.url  # import httpbasicauthhandler, httpdigestauthhandler
 
import mercurial.util  # import url as hg_url
 
from dulwich.config import ConfigFile
 
from dulwich.objects import Tag
 
@@ -175,39 +176,39 @@ class GitRepository(BaseRepository):
 

	
 
        url_obj.passwd = b'*****'
 
        cleaned_uri = str(url_obj)
 

	
 
        if authinfo:
 
            # create a password manager
 
            passmgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
 
            passmgr = urllib.request.HTTPPasswordMgrWithDefaultRealm()
 
            passmgr.add_password(*authinfo)
 

	
 
            handlers.extend((mercurial.url.httpbasicauthhandler(passmgr),
 
                             mercurial.url.httpdigestauthhandler(passmgr)))
 

	
 
        o = urllib2.build_opener(*handlers)
 
        o = urllib.request.build_opener(*handlers)
 
        o.addheaders = [('User-Agent', 'git/1.7.8.0')]  # fake some git
 

	
 
        req = urllib2.Request(
 
        req = urllib.request.Request(
 
            "%s?%s" % (
 
                test_uri,
 
                urllib.urlencode({"service": 'git-upload-pack'})
 
                urllib.parse.urlencode({"service": 'git-upload-pack'})
 
            ))
 

	
 
        try:
 
            resp = o.open(req)
 
            if resp.code != 200:
 
                raise Exception('Return Code is not 200')
 
        except Exception as e:
 
            # means it cannot be cloned
 
            raise urllib2.URLError("[%s] org_exc: %s" % (cleaned_uri, e))
 
            raise urllib.error.URLError("[%s] org_exc: %s" % (cleaned_uri, e))
 

	
 
        # now detect if it's proper git repo
 
        gitdata = resp.read()
 
        if 'service=git-upload-pack' not in gitdata:
 
            raise urllib2.URLError(
 
            raise urllib.error.URLError(
 
                "url [%s] does not look like an git" % cleaned_uri)
 

	
 
        return True
 

	
 
    def _get_repo(self, create, src_url=None, update_after_clone=False,
 
                  bare=False):
kallithea/lib/vcs/backends/hg/repository.py
Show inline comments
 
@@ -10,14 +10,15 @@
 
"""
 

	
 
import datetime
 
import logging
 
import os
 
import time
 
import urllib
 
import urllib2
 
import urllib.error
 
import urllib.parse
 
import urllib.request
 
from collections import OrderedDict
 

	
 
import mercurial.commands
 
import mercurial.error
 
import mercurial.exchange
 
import mercurial.hg
 
@@ -311,45 +312,45 @@ class MercurialRepository(BaseRepository
 
        test_uri, authinfo = url_obj.authinfo()
 
        url_obj.passwd = b'*****'
 
        cleaned_uri = str(url_obj)
 

	
 
        if authinfo:
 
            # create a password manager
 
            passmgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
 
            passmgr = urllib.request.HTTPPasswordMgrWithDefaultRealm()
 
            passmgr.add_password(*authinfo)
 

	
 
            handlers.extend((mercurial.url.httpbasicauthhandler(passmgr),
 
                             mercurial.url.httpdigestauthhandler(passmgr)))
 

	
 
        o = urllib2.build_opener(*handlers)
 
        o = urllib.request.build_opener(*handlers)
 
        o.addheaders = [('Content-Type', 'application/mercurial-0.1'),
 
                        ('Accept', 'application/mercurial-0.1')]
 

	
 
        req = urllib2.Request(
 
        req = urllib.request.Request(
 
            "%s?%s" % (
 
                test_uri,
 
                urllib.urlencode({
 
                urllib.parse.urlencode({
 
                    'cmd': 'between',
 
                    'pairs': "%s-%s" % ('0' * 40, '0' * 40),
 
                })
 
            ))
 

	
 
        try:
 
            resp = o.open(req)
 
            if resp.code != 200:
 
                raise Exception('Return Code is not 200')
 
        except Exception as e:
 
            # means it cannot be cloned
 
            raise urllib2.URLError("[%s] org_exc: %s" % (cleaned_uri, e))
 
            raise urllib.error.URLError("[%s] org_exc: %s" % (cleaned_uri, e))
 

	
 
        if not url_prefix: # skip svn+http://... (and git+... too)
 
            # now check if it's a proper hg repo
 
            try:
 
                mercurial.httppeer.instance(repoui or mercurial.ui.ui(), url, False).lookup(b'tip')
 
            except Exception as e:
 
                raise urllib2.URLError(
 
                raise urllib.error.URLError(
 
                    "url [%s] does not look like an hg repo org_exc: %s"
 
                    % (cleaned_uri, e))
 

	
 
        return True
 

	
 
    def _get_repo(self, create, src_url=None, update_after_clone=False):
 
@@ -487,13 +488,13 @@ class MercurialRepository(BaseRepository
 
        Returns normalized url. If schema is not given, would fall
 
        to filesystem
 
        (``file:///``) schema.
 
        """
 
        url = safe_str(url)
 
        if url != 'default' and '://' not in url:
 
            url = "file:" + urllib.pathname2url(url)
 
            url = "file:" + urllib.request.pathname2url(url)
 
        return url
 

	
 
    def get_changeset(self, revision=None):
 
        """
 
        Returns ``MercurialChangeset`` object representing repository's
 
        changeset at the given ``revision``.
kallithea/tests/functional/test_admin_repos.py
Show inline comments
 
# -*- coding: utf-8 -*-
 

	
 
import os
 
import urllib
 
import urllib.parse
 

	
 
import mock
 
import pytest
 

	
 
from kallithea.lib import vcs
 
from kallithea.lib.utils2 import safe_str, safe_unicode
 
@@ -407,13 +407,13 @@ class _BaseTestCase(base.TestController)
 
                                                _session_csrf_secret_token=self.session_csrf_secret_token()))
 
        ## run the check page that triggers the flash message
 
        response = self.app.get(base.url('repo_check_home', repo_name=repo_name))
 
        assert response.json == {u'result': True}
 
        self.checkSessionFlash(response,
 
                               u'Created repository <a href="/%s">%s</a>'
 
                               % (urllib.quote(repo_name), repo_name_unicode))
 
                               % (urllib.parse.quote(repo_name), repo_name_unicode))
 
        # test if the repo was created in the database
 
        new_repo = Session().query(Repository) \
 
            .filter(Repository.repo_name == repo_name_unicode).one()
 

	
 
        assert new_repo.repo_name == repo_name_unicode
 
        assert new_repo.description == description_unicode
kallithea/tests/functional/test_forks.py
Show inline comments
 
# -*- coding: utf-8 -*-
 

	
 
import urllib
 
import urllib.parse
 

	
 
from kallithea.lib.utils2 import safe_str, safe_unicode
 
from kallithea.model.db import Repository, User
 
from kallithea.model.meta import Session
 
from kallithea.model.repo import RepoModel
 
from kallithea.model.user import UserModel
 
@@ -156,13 +156,13 @@ class _BaseTestCase(base.TestController)
 
            '_session_csrf_secret_token': self.session_csrf_secret_token()}
 
        self.app.post(base.url(controller='forks', action='fork_create',
 
                          repo_name=repo_name), creation_args)
 
        response = self.app.get(base.url(controller='forks', action='forks',
 
                                    repo_name=repo_name))
 
        response.mustcontain(
 
            """<a href="/%s">%s</a>""" % (urllib.quote(fork_name), fork_name)
 
            """<a href="/%s">%s</a>""" % (urllib.parse.quote(fork_name), fork_name)
 
        )
 
        fork_repo = Repository.get_by_repo_name(safe_unicode(fork_name))
 
        assert fork_repo
 

	
 
        # fork the fork
 
        fork_name_2 = safe_str(self.REPO_FORK + u'-blåbærgrød')
 
@@ -177,13 +177,13 @@ class _BaseTestCase(base.TestController)
 
            '_session_csrf_secret_token': self.session_csrf_secret_token()}
 
        self.app.post(base.url(controller='forks', action='fork_create',
 
                          repo_name=fork_name), creation_args)
 
        response = self.app.get(base.url(controller='forks', action='forks',
 
                                    repo_name=fork_name))
 
        response.mustcontain(
 
            """<a href="/%s">%s</a>""" % (urllib.quote(fork_name_2), fork_name_2)
 
            """<a href="/%s">%s</a>""" % (urllib.parse.quote(fork_name_2), fork_name_2)
 
        )
 

	
 
        # remove these forks
 
        response = self.app.post(base.url('delete_repo', repo_name=fork_name_2),
 
            params={'_session_csrf_secret_token': self.session_csrf_secret_token()})
 
        response = self.app.post(base.url('delete_repo', repo_name=fork_name),
kallithea/tests/other/test_vcs_operations.py
Show inline comments
 
@@ -29,13 +29,13 @@ from __future__ import print_function
 

	
 
import json
 
import os
 
import re
 
import tempfile
 
import time
 
import urllib2
 
import urllib.request
 
from subprocess import PIPE, Popen
 
from tempfile import _RandomNameSequence
 

	
 
import pytest
 

	
 
from kallithea import CONFIG
 
@@ -326,17 +326,17 @@ class TestVCSOperations(base.TestControl
 
            "api_key": usr.api_key,
 
            "method": 'create_repo',
 
            "args": dict(repo_name=repo_name,
 
                         owner=base.TEST_USER_ADMIN_LOGIN,
 
                         repo_type=vt.repo_type),
 
        }
 
        req = urllib2.Request(
 
        req = urllib.request.Request(
 
            'http://%s:%s/_admin/api' % webserver.server_address,
 
            data=ascii_bytes(json.dumps(params)),
 
            headers={'content-type': 'application/json'})
 
        response = urllib2.urlopen(req)
 
        response = urllib.request.urlopen(req)
 
        result = json.loads(response.read())
 
        # Expect something like:
 
        # {u'result': {u'msg': u'Created new repository `new_XXX`', u'task': None, u'success': True}, u'id': 7, u'error': None}
 
        assert result[u'result'][u'success']
 

	
 
        # Create local clone of the empty server repo
kallithea/tests/scripts/manual_test_crawler.py
Show inline comments
 
@@ -34,14 +34,14 @@ from __future__ import print_function
 

	
 
import http.cookiejar
 
import os
 
import sys
 
import tempfile
 
import time
 
import urllib
 
import urllib2
 
import urllib.parse
 
import urllib.request
 
from os.path import dirname
 

	
 
from kallithea.lib import vcs
 
from kallithea.lib.compat import OrderedSet
 
from kallithea.lib.vcs.exceptions import RepositoryError
 

	
 
@@ -70,19 +70,19 @@ PROJECTS = [
 
    'CPython',
 
    'kallithea',
 
]
 

	
 

	
 
cj = http.cookiejar.FileCookieJar(os.path.join(tempfile.gettempdir(), 'rc_test_cookie.txt'))
 
o = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj))
 
o = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(cj))
 
o.addheaders = [
 
    ('User-agent', 'kallithea-crawler'),
 
    ('Accept-Language', 'en - us, en;q = 0.5')
 
]
 

	
 
urllib2.install_opener(o)
 
urllib.request.install_opener(o)
 

	
 

	
 
def _get_repo(proj):
 
    if isinstance(proj, str):
 
        repo = vcs.get_repo(os.path.join(PROJECT_PATH, proj))
 
        proj = proj
 
@@ -98,13 +98,13 @@ def test_changelog_walk(proj, pages=100)
 

	
 
    total_time = 0
 
    for i in range(1, pages):
 

	
 
        page = '/'.join((proj, 'changelog',))
 

	
 
        full_uri = (BASE_URI % page) + '?' + urllib.urlencode({'page': i})
 
        full_uri = (BASE_URI % page) + '?' + urllib.parse.urlencode({'page': i})
 
        s = time.time()
 
        f = o.open(full_uri)
 

	
 
        assert f.url == full_uri, 'URL:%s does not match %s' % (f.url, full_uri)
 

	
 
        size = len(f.read())
kallithea/tests/vcs/test_git.py
Show inline comments
 
import datetime
 
import os
 
import sys
 
import urllib2
 
import urllib.error
 

	
 
import mock
 
import pytest
 

	
 
from kallithea.lib.vcs.backends.git import GitChangeset, GitRepository
 
from kallithea.lib.vcs.exceptions import NodeDoesNotExistError, RepositoryError, VCSError
 
@@ -29,13 +29,13 @@ class TestGitRepository(object):
 
        wrong_repo_path = os.path.join(TESTS_TMP_PATH, 'errorrepo')
 
        with pytest.raises(RepositoryError):
 
            GitRepository(wrong_repo_path)
 

	
 
    def test_git_cmd_injection(self):
 
        repo_inject_path = TEST_GIT_REPO + '; echo "Cake";'
 
        with pytest.raises(urllib2.URLError):
 
        with pytest.raises(urllib.error.URLError):
 
            # Should fail because URL will contain the parts after ; too
 
            GitRepository(get_new_dir('injection-repo'), src_url=repo_inject_path, update_after_clone=True, create=True)
 

	
 
        with pytest.raises(RepositoryError):
 
            # Should fail on direct clone call, which as of this writing does not happen outside of class
 
            clone_fail_repo = GitRepository(get_new_dir('injection-repo'), create=True)
0 comments (0 inline, 0 general)