kallithea Changeset - ce04e6ef80c0

Changeset - ce04e6ef80c0

Parent rev.

Child rev.

[Not reviewed]

beta

0 1 0

Marcin Kuzminski - 14 years ago 2012-05-03 00:59:42
marcin@python-works.com

improvements for rhodecode crawler

1 file changed with 39 insertions and 14 deletions:

rhodecode/tests/rhodecode_crawler.py

0 comments (0 inline, 0 general)

rhodecode/tests/rhodecode_crawler.py

➞

Show inline comments

@@ @@ -33,96 +33,121 @@ import urllib @@
 import urllib2
 import time
 import os
 import sys
 from os.path import join as jn
 from os.path import dirname as dn
 __here__ = os.path.abspath(__file__)
 __root__ = dn(dn(dn(__here__)))
 sys.path.append(__root__)
 from rhodecode.lib import vcs
 from rhodecode.lib.compat import OrderedSet
 from rhodecode.lib.vcs.exceptions import RepositoryError
 BASE_URI = 'http://127.0.0.1:5001/%s'
 PASES = 3
 HOST = 'http://127.0.0.1'
 PORT = 5000
 BASE_URI = '%s:%s/' % (HOST, PORT)
 if len(sys.argv) == 2:
     BASE_URI = sys.argv[1]
 if not BASE_URI.endswith('/'):
     BASE_URI += '/'
 print 'Crawling @ %s' % BASE_URI
 BASE_URI += '%s'
 PROJECT_PATH = jn('/', 'home', 'marcink', 'hg_repos')
 PROJECTS = [
     'linux-magx-pbranch',
     'CPython',
     'rhodecode_tip',
+]
 cj = cookielib.FileCookieJar('/tmp/rc_test_cookie.txt')
 o = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj))
 o.addheaders = [
     ('User-agent', 'rhodecode-crawler'),
     ('Accept-Language', 'en - us, en;q = 0.5')
+]
 urllib2.install_opener(o)
 def _get_repo(proj):
     if isinstance(proj, basestring):
         repo = vcs.get_repo(jn(PROJECT_PATH, proj))
         proj = proj
     else:
         repo = proj
         proj = repo.name
     return repo, proj
 def test_changelog_walk(proj, pages=100):
     repo, proj = _get_repo(proj)
     total_time = 0
     for i in range(1, pages):
         page = '/'.join((proj, 'changelog',))
         full_uri = (BASE_URI % page) + '?' + urllib.urlencode({'page':i})
         s = time.time()
         f = o.open(full_uri)
         size = len(f.read())
         e = time.time() - s
         total_time += e
         print 'visited %s size:%s req:%s ms' % (full_uri, size, e)
     print 'total_time', total_time
     print 'average on req', total_time / float(pages)
 def test_changeset_walk(proj, limit=None):
     repo, proj = _get_repo(proj)
     print 'processing', jn(PROJECT_PATH, proj)
     total_time = 0
     repo = vcs.get_repo(jn(PROJECT_PATH, proj))
     cnt = 0
     for i in repo:
         cnt += 1
         raw_cs = '/'.join((proj, 'changeset', i.raw_id))
         if limit and limit == cnt:
             break
         full_uri = (BASE_URI % raw_cs)
         print '%s visiting %s\%s' % (cnt, full_uri, i)
         s = time.time()
         f = o.open(full_uri)
         size = len(f.read())
         e = time.time() - s
         total_time += e
         print '%s visited %s\%s size:%s req:%s ms' % (cnt, full_uri, i, size, e)
     print 'total_time', total_time
     print 'average on req', total_time / float(cnt)
 def test_files_walk(proj, limit=100):
     repo, proj = _get_repo(proj)
     print 'processing', jn(PROJECT_PATH, proj)
     total_time = 0
     repo = vcs.get_repo(jn(PROJECT_PATH, proj))
     from rhodecode.lib.compat import OrderedSet
     from rhodecode.lib.vcs.exceptions import RepositoryError
     paths_ = OrderedSet([''])
     try:
         tip = repo.get_changeset('tip')
         for topnode, dirs, files in tip.walk('/'):
             for dir in dirs:
                 paths_.add(dir.path)
                 for f in dir:
                     paths_.add(f.path)
             for f in files:
                 paths_.add(f.path)
@@ @@ -141,19 +166,19 @@ def test_files_walk(proj, limit=100): @@
         print '%s visiting %s' % (cnt, full_uri)
         s = time.time()
         f = o.open(full_uri)
         size = len(f.read())
         e = time.time() - s
         total_time += e
         print '%s visited OK size:%s req:%s ms' % (cnt, size, e)
     print 'total_time', total_time
     print 'average on req', total_time / float(cnt)
 if __name__ == '__main__':
     for p in PROJECTS:
         test_changelog_walk(p, 40)
         time.sleep(2)
         test_changeset_walk(p, limit=100)
         time.sleep(2)
         test_files_walk(p, 100)
     for path in PROJECTS:
         repo = vcs.get_repo(jn(PROJECT_PATH, path))
         for i in range(PASES):
             print 'PASS %s/%s' % (i, PASES)
             test_changelog_walk(repo, pages=80)
             test_changeset_walk(repo, limit=100)
             test_files_walk(repo, limit=100)

0 comments (0 inline, 0 general)