Changeset - f61adead3520
[Not reviewed]
beta
0 4 0
Marcin Kuzminski - 13 years ago 2012-11-24 01:59:14
marcin@python-works.com
Added option to ini files for controlling full cache of VCS instances.
This can increase performance for huge repositories like 300K changesets
4 files changed with 7 insertions and 1 deletions:
0 comments (0 inline, 0 general)
development.ini
Show inline comments
 
@@ -15,96 +15,97 @@ pdebug = false
 
################################################################################
 
#email_to = admin@localhost
 
#error_email_from = paste_error@localhost
 
#app_email_from = rhodecode-noreply@localhost
 
#error_message =
 
#email_prefix = [RhodeCode]
 

	
 
#smtp_server = mail.server.com
 
#smtp_username = 
 
#smtp_password = 
 
#smtp_port = 
 
#smtp_use_tls = false
 
#smtp_use_ssl = true
 
# Specify available auth parameters here (e.g. LOGIN PLAIN CRAM-MD5, etc.)
 
#smtp_auth = 
 

	
 
[server:main]
 
##nr of threads to spawn
 
#threadpool_workers = 5
 

	
 
##max request before thread respawn
 
#threadpool_max_requests = 10
 

	
 
##option to use threads of process
 
#use_threadpool = true
 

	
 
#use = egg:Paste#http
 
use = egg:waitress#main
 
host = 0.0.0.0
 
port = 5000
 

	
 
[filter:proxy-prefix]
 
# prefix middleware for rc
 
use = egg:PasteDeploy#prefix
 
prefix = /<your-prefix>
 

	
 
[app:main]
 
use = egg:rhodecode
 
#filter-with = proxy-prefix
 
full_stack = true
 
static_files = true
 
# Optional Languages
 
# en, fr, ja, pt_BR, zh_CN, zh_TW
 
lang = en
 
cache_dir = %(here)s/data
 
index_dir = %(here)s/data/index
 
app_instance_uuid = rc-develop
 
cut_off_limit = 256000
 
vcs_full_cache = True
 
force_https = false
 
commit_parse_limit = 25
 
use_gravatar = true
 

	
 
## alternative_gravatar_url allows you to use your own avatar server application
 
## the following parts of the URL will be replaced
 
## {email}        user email
 
## {md5email}     md5 hash of the user email (like at gravatar.com)
 
## {size}         size of the image that is expected from the server application
 
## {scheme}       http/https from RhodeCode server
 
## {netloc}       network location from RhodeCode server
 
#alternative_gravatar_url = http://myavatarserver.com/getbyemail/{email}/{size}
 
#alternative_gravatar_url = http://myavatarserver.com/getbymd5/{md5email}?s={size}
 

	
 
container_auth_enabled = false
 
proxypass_auth_enabled = false
 
## default encoding used to convert from and to unicode
 
## can be also a comma seperated list of encoding in case of mixed encodings
 
default_encoding = utf8
 

	
 
## overwrite schema of clone url
 
## available vars:
 
## scheme - http/https
 
## user - current user
 
## pass - password 
 
## netloc - network location
 
## path - usually repo_name
 

	
 
#clone_uri = {scheme}://{user}{pass}{netloc}{path}
 

	
 
## issue tracking mapping for commits messages
 
## comment out issue_pat, issue_server, issue_prefix to enable
 

	
 
## pattern to get the issues from commit messages
 
## default one used here is #<numbers> with a regex passive group for `#`
 
## {id} will be all groups matched from this pattern
 

	
 
issue_pat = (?:\s*#)(\d+)
 

	
 
## server url to the issue, each {id} will be replaced with match
 
## fetched from the regex and {repo} is replaced with full repository name
 
## including groups {repo_name} is replaced with just name of repo
 

	
 
issue_server_link = https://myissueserver.com/{repo}/issue/{id}
 

	
 
## prefix to add to link to indicate it's an url
 
## #314 will be replaced by <issue_prefix><id>
 

	
production.ini
Show inline comments
 
@@ -15,96 +15,97 @@ pdebug = false
 
################################################################################
 
#email_to = admin@localhost
 
#error_email_from = paste_error@localhost
 
#app_email_from = rhodecode-noreply@localhost
 
#error_message =
 
#email_prefix = [RhodeCode]
 

	
 
#smtp_server = mail.server.com
 
#smtp_username = 
 
#smtp_password = 
 
#smtp_port = 
 
#smtp_use_tls = false
 
#smtp_use_ssl = true
 
# Specify available auth parameters here (e.g. LOGIN PLAIN CRAM-MD5, etc.)
 
#smtp_auth = 
 

	
 
[server:main]
 
##nr of threads to spawn
 
#threadpool_workers = 5
 

	
 
##max request before thread respawn
 
#threadpool_max_requests = 10
 

	
 
##option to use threads of process
 
#use_threadpool = true
 

	
 
#use = egg:Paste#http
 
use = egg:waitress#main
 
host = 127.0.0.1
 
port = 8001
 

	
 
[filter:proxy-prefix]
 
# prefix middleware for rc
 
use = egg:PasteDeploy#prefix
 
prefix = /<your-prefix>
 

	
 
[app:main]
 
use = egg:rhodecode
 
#filter-with = proxy-prefix
 
full_stack = true
 
static_files = true
 
# Optional Languages
 
# en, fr, ja, pt_BR, zh_CN, zh_TW
 
lang = en
 
cache_dir = %(here)s/data
 
index_dir = %(here)s/data/index
 
app_instance_uuid = rc-production
 
cut_off_limit = 256000
 
vcs_full_cache = True
 
force_https = false
 
commit_parse_limit = 50
 
use_gravatar = true
 

	
 
## alternative_gravatar_url allows you to use your own avatar server application
 
## the following parts of the URL will be replaced
 
## {email}        user email
 
## {md5email}     md5 hash of the user email (like at gravatar.com)
 
## {size}         size of the image that is expected from the server application
 
## {scheme}       http/https from RhodeCode server
 
## {netloc}       network location from RhodeCode server
 
#alternative_gravatar_url = http://myavatarserver.com/getbyemail/{email}/{size}
 
#alternative_gravatar_url = http://myavatarserver.com/getbymd5/{md5email}?s={size}
 

	
 
container_auth_enabled = false
 
proxypass_auth_enabled = false
 
## default encoding used to convert from and to unicode
 
## can be also a comma seperated list of encoding in case of mixed encodings
 
default_encoding = utf8
 

	
 
## overwrite schema of clone url
 
## available vars:
 
## scheme - http/https
 
## user - current user
 
## pass - password 
 
## netloc - network location
 
## path - usually repo_name
 

	
 
#clone_uri = {scheme}://{user}{pass}{netloc}{path}
 

	
 
## issue tracking mapping for commits messages
 
## comment out issue_pat, issue_server, issue_prefix to enable
 

	
 
## pattern to get the issues from commit messages
 
## default one used here is #<numbers> with a regex passive group for `#`
 
## {id} will be all groups matched from this pattern
 

	
 
issue_pat = (?:\s*#)(\d+)
 

	
 
## server url to the issue, each {id} will be replaced with match
 
## fetched from the regex and {repo} is replaced with full repository name
 
## including groups {repo_name} is replaced with just name of repo
 

	
 
issue_server_link = https://myissueserver.com/{repo}/issue/{id}
 

	
 
## prefix to add to link to indicate it's an url
 
## #314 will be replaced by <issue_prefix><id>
 

	
rhodecode/config/deployment.ini_tmpl
Show inline comments
 
@@ -15,96 +15,97 @@ pdebug = false
 
################################################################################
 
#email_to = admin@localhost
 
#error_email_from = paste_error@localhost
 
#app_email_from = rhodecode-noreply@localhost
 
#error_message =
 
#email_prefix = [RhodeCode]
 

	
 
#smtp_server = mail.server.com
 
#smtp_username = 
 
#smtp_password = 
 
#smtp_port = 
 
#smtp_use_tls = false
 
#smtp_use_ssl = true
 
# Specify available auth parameters here (e.g. LOGIN PLAIN CRAM-MD5, etc.)
 
#smtp_auth = 
 

	
 
[server:main]
 
##nr of threads to spawn
 
#threadpool_workers = 5
 

	
 
##max request before thread respawn
 
#threadpool_max_requests = 10
 

	
 
##option to use threads of process
 
#use_threadpool = true
 

	
 
#use = egg:Paste#http
 
use = egg:waitress#main
 
host = 127.0.0.1
 
port = 5000
 

	
 
[filter:proxy-prefix]
 
# prefix middleware for rc
 
use = egg:PasteDeploy#prefix
 
prefix = /<your-prefix>
 

	
 
[app:main]
 
use = egg:rhodecode
 
#filter-with = proxy-prefix
 
full_stack = true
 
static_files = true
 
# Optional Languages
 
# en, fr, ja, pt_BR, zh_CN, zh_TW
 
lang = en
 
cache_dir = %(here)s/data
 
index_dir = %(here)s/data/index
 
app_instance_uuid = ${app_instance_uuid}
 
cut_off_limit = 256000
 
vcs_full_cache = True
 
force_https = false
 
commit_parse_limit = 50
 
use_gravatar = true
 

	
 
## alternative_gravatar_url allows you to use your own avatar server application
 
## the following parts of the URL will be replaced
 
## {email}        user email
 
## {md5email}     md5 hash of the user email (like at gravatar.com)
 
## {size}         size of the image that is expected from the server application
 
## {scheme}       http/https from RhodeCode server
 
## {netloc}       network location from RhodeCode server
 
#alternative_gravatar_url = http://myavatarserver.com/getbyemail/{email}/{size}
 
#alternative_gravatar_url = http://myavatarserver.com/getbymd5/{md5email}?s={size}
 

	
 
container_auth_enabled = false
 
proxypass_auth_enabled = false
 
## default encoding used to convert from and to unicode
 
## can be also a comma seperated list of encoding in case of mixed encodings
 
default_encoding = utf8
 

	
 
## overwrite schema of clone url
 
## available vars:
 
## scheme - http/https
 
## user - current user
 
## pass - password 
 
## netloc - network location
 
## path - usually repo_name
 

	
 
#clone_uri = {scheme}://{user}{pass}{netloc}{path}
 

	
 
## issue tracking mapping for commits messages
 
## comment out issue_pat, issue_server, issue_prefix to enable
 

	
 
## pattern to get the issues from commit messages
 
## default one used here is #<numbers> with a regex passive group for `#`
 
## {id} will be all groups matched from this pattern
 

	
 
issue_pat = (?:\s*#)(\d+)
 

	
 
## server url to the issue, each {id} will be replaced with match
 
## fetched from the regex and {repo} is replaced with full repository name
 
## including groups {repo_name} is replaced with just name of repo
 

	
 
issue_server_link = https://myissueserver.com/{repo}/issue/{id}
 

	
 
## prefix to add to link to indicate it's an url
 
## #314 will be replaced by <issue_prefix><id>
 

	
rhodecode/model/db.py
Show inline comments
 
@@ -896,97 +896,100 @@ class Repository(Base, BaseModel):
 
        """
 
        Returns statuses for this repository
 

	
 
        :param revisions: list of revisions to get statuses for
 
        :type revisions: list
 
        """
 

	
 
        statuses = ChangesetStatus.query()\
 
            .filter(ChangesetStatus.repo == self)\
 
            .filter(ChangesetStatus.version == 0)
 
        if revisions:
 
            statuses = statuses.filter(ChangesetStatus.revision.in_(revisions))
 
        grouped = {}
 

	
 
        #maybe we have open new pullrequest without a status ?
 
        stat = ChangesetStatus.STATUS_UNDER_REVIEW
 
        status_lbl = ChangesetStatus.get_status_lbl(stat)
 
        for pr in PullRequest.query().filter(PullRequest.org_repo == self).all():
 
            for rev in pr.revisions:
 
                pr_id = pr.pull_request_id
 
                pr_repo = pr.other_repo.repo_name
 
                grouped[rev] = [stat, status_lbl, pr_id, pr_repo]
 

	
 
        for stat in statuses.all():
 
            pr_id = pr_repo = None
 
            if stat.pull_request:
 
                pr_id = stat.pull_request.pull_request_id
 
                pr_repo = stat.pull_request.other_repo.repo_name
 
            grouped[stat.revision] = [str(stat.status), stat.status_lbl,
 
                                      pr_id, pr_repo]
 
        return grouped
 

	
 
    #==========================================================================
 
    # SCM CACHE INSTANCE
 
    #==========================================================================
 

	
 
    @property
 
    def invalidate(self):
 
        return CacheInvalidation.invalidate(self.repo_name)
 

	
 
    def set_invalidate(self):
 
        """
 
        set a cache for invalidation for this instance
 
        """
 
        CacheInvalidation.set_invalidate(repo_name=self.repo_name)
 

	
 
    @LazyProperty
 
    def scm_instance(self):
 
        return self.scm_instance_cached()
 
        import rhodecode
 
        full_cache = str2bool(rhodecode.CONFIG.get('vcs_full_cache'))
 
        if full_cache:
 
            return self.scm_instance_cached()
 
        return self.__get_instance()
 

	
 
    def scm_instance_cached(self, cache_map=None):
 
        @cache_region('long_term')
 
        def _c(repo_name):
 
            return self.__get_instance()
 
        rn = self.repo_name
 
        log.debug('Getting cached instance of repo')
 

	
 
        if cache_map:
 
            # get using prefilled cache_map
 
            invalidate_repo = cache_map[self.repo_name]
 
            if invalidate_repo:
 
                invalidate_repo = (None if invalidate_repo.cache_active
 
                                   else invalidate_repo)
 
        else:
 
            # get from invalidate
 
            invalidate_repo = self.invalidate
 

	
 
        if invalidate_repo is not None:
 
            region_invalidate(_c, None, rn)
 
            # update our cache
 
            CacheInvalidation.set_valid(invalidate_repo.cache_key)
 
        return _c(rn)
 

	
 
    def __get_instance(self):
 
        repo_full_path = self.repo_full_path
 
        try:
 
            alias = get_scm(repo_full_path)[0]
 
            log.debug('Creating instance of %s repository' % alias)
 
            backend = get_backend(alias)
 
        except VCSError:
 
            log.error(traceback.format_exc())
 
            log.error('Perhaps this repository is in db and not in '
 
                      'filesystem run rescan repositories with '
 
                      '"destroy old data " option from admin panel')
 
            return
 

	
 
        if alias == 'hg':
 

	
 
            repo = backend(safe_str(repo_full_path), create=False,
 
                           baseui=self._ui)
 
            # skip hidden web repository
 
            if repo._get_hidden():
 
                return
 
        else:
 
            repo = backend(repo_full_path, create=False)
 

	
0 comments (0 inline, 0 general)