Changeset - e886f91fcb71
[Not reviewed]
beta
0 7 0
Marcin Kuzminski - 14 years ago 2011-10-28 02:40:22
marcin@python-works.com
Cached readme generation
- cleaned up cache system
7 files changed with 123 insertions and 87 deletions:
0 comments (0 inline, 0 general)
development.ini
Show inline comments
 
@@ -70,63 +70,69 @@ celery.result.backend = amqp
 
celery.result.dburi = amqp://
 
celery.result.serialier = json
 

	
 
#celery.send.task.error.emails = true
 
#celery.amqp.task.result.expires = 18000
 

	
 
celeryd.concurrency = 2
 
#celeryd.log.file = celeryd.log
 
celeryd.log.level = debug
 
celeryd.max.tasks.per.child = 1
 

	
 
#tasks will never be sent to the queue, but executed locally instead.
 
celery.always.eager = false
 

	
 
####################################
 
###         BEAKER CACHE        ####
 
####################################
 
beaker.cache.data_dir=%(here)s/data/cache/data
 
beaker.cache.lock_dir=%(here)s/data/cache/lock
 

	
 
beaker.cache.regions=super_short_term,short_term,long_term,sql_cache_short,sql_cache_med,sql_cache_long
 

	
 
beaker.cache.super_short_term.type=memory
 
beaker.cache.super_short_term.expire=10
 
beaker.cache.super_short_term.key_length = 256
 

	
 
beaker.cache.short_term.type=memory
 
beaker.cache.short_term.expire=60
 
beaker.cache.short_term.key_length = 256
 

	
 
beaker.cache.long_term.type=memory
 
beaker.cache.long_term.expire=36000
 
beaker.cache.long_term.key_length = 256
 

	
 
beaker.cache.sql_cache_short.type=memory
 
beaker.cache.sql_cache_short.expire=10
 
beaker.cache.sql_cache_short.key_length = 256
 

	
 
beaker.cache.sql_cache_med.type=memory
 
beaker.cache.sql_cache_med.expire=360
 
beaker.cache.sql_cache_med.key_length = 256
 

	
 
beaker.cache.sql_cache_long.type=file
 
beaker.cache.sql_cache_long.expire=3600
 
beaker.cache.sql_cache_long.key_length = 256
 

	
 
####################################
 
###       BEAKER SESSION        ####
 
####################################
 
## Type of storage used for the session, current types are 
 
## dbm, file, memcached, database, and memory. 
 
## The storage uses the Container API 
 
##that is also used by the cache system.
 
beaker.session.type = file
 

	
 
beaker.session.key = rhodecode
 
beaker.session.secret = g654dcno0-9873jhgfreyu
 
beaker.session.timeout = 36000
 

	
 
##auto save the session to not to use .save()
 
beaker.session.auto = False
 

	
 
##true exire at browser close
 
#beaker.session.cookie_expires = 3600
 

	
 
    
 
################################################################################
 
## WARNING: *THE LINE BELOW MUST BE UNCOMMENTED ON A PRODUCTION ENVIRONMENT*  ##
 
## Debug mode will enable the interactive debugging tool, allowing ANYONE to  ##
production.ini
Show inline comments
 
@@ -70,63 +70,69 @@ celery.result.backend = amqp
 
celery.result.dburi = amqp://
 
celery.result.serialier = json
 

	
 
#celery.send.task.error.emails = true
 
#celery.amqp.task.result.expires = 18000
 

	
 
celeryd.concurrency = 2
 
#celeryd.log.file = celeryd.log
 
celeryd.log.level = debug
 
celeryd.max.tasks.per.child = 1
 

	
 
#tasks will never be sent to the queue, but executed locally instead.
 
celery.always.eager = false
 

	
 
####################################
 
###         BEAKER CACHE        ####
 
####################################
 
beaker.cache.data_dir=%(here)s/data/cache/data
 
beaker.cache.lock_dir=%(here)s/data/cache/lock
 

	
 
beaker.cache.regions=super_short_term,short_term,long_term,sql_cache_short,sql_cache_med,sql_cache_long
 

	
 
beaker.cache.super_short_term.type=memory
 
beaker.cache.super_short_term.expire=10
 
beaker.cache.super_short_term.key_length = 256
 

	
 
beaker.cache.short_term.type=memory
 
beaker.cache.short_term.expire=60
 
beaker.cache.short_term.key_length = 256
 

	
 
beaker.cache.long_term.type=memory
 
beaker.cache.long_term.expire=36000
 
beaker.cache.long_term.key_length = 256
 

	
 
beaker.cache.sql_cache_short.type=memory
 
beaker.cache.sql_cache_short.expire=10
 
beaker.cache.sql_cache_short.key_length = 256
 

	
 
beaker.cache.sql_cache_med.type=memory
 
beaker.cache.sql_cache_med.expire=360
 
beaker.cache.sql_cache_med.key_length = 256
 

	
 
beaker.cache.sql_cache_long.type=file
 
beaker.cache.sql_cache_long.expire=3600
 
beaker.cache.sql_cache_long.key_length = 256
 

	
 
####################################
 
###       BEAKER SESSION        ####
 
####################################
 
## Type of storage used for the session, current types are 
 
## dbm, file, memcached, database, and memory. 
 
## The storage uses the Container API 
 
##that is also used by the cache system.
 
beaker.session.type = file
 

	
 
beaker.session.key = rhodecode
 
beaker.session.secret = g654dcno0-9873jhgfreyu
 
beaker.session.timeout = 36000
 

	
 
##auto save the session to not to use .save()
 
beaker.session.auto = False
 

	
 
##true exire at browser close
 
#beaker.session.cookie_expires = 3600
 

	
 
    
 
################################################################################
 
## WARNING: *THE LINE BELOW MUST BE UNCOMMENTED ON A PRODUCTION ENVIRONMENT*  ##
 
## Debug mode will enable the interactive debugging tool, allowing ANYONE to  ##
rhodecode/config/deployment.ini_tmpl
Show inline comments
 
@@ -70,111 +70,118 @@ celery.result.backend = amqp
 
celery.result.dburi = amqp://
 
celery.result.serialier = json
 

	
 
#celery.send.task.error.emails = true
 
#celery.amqp.task.result.expires = 18000
 

	
 
celeryd.concurrency = 2
 
#celeryd.log.file = celeryd.log
 
celeryd.log.level = debug
 
celeryd.max.tasks.per.child = 1
 

	
 
#tasks will never be sent to the queue, but executed locally instead.
 
celery.always.eager = false
 

	
 
####################################
 
###         BEAKER CACHE        ####
 
####################################
 
beaker.cache.data_dir=%(here)s/data/cache/data
 
beaker.cache.lock_dir=%(here)s/data/cache/lock
 

	
 
beaker.cache.regions=super_short_term,short_term,long_term,sql_cache_short,sql_cache_med,sql_cache_long
 

	
 
beaker.cache.super_short_term.type=memory
 
beaker.cache.super_short_term.expire=10
 
beaker.cache.super_short_term.key_length = 256
 

	
 
beaker.cache.short_term.type=memory
 
beaker.cache.short_term.expire=60
 
beaker.cache.short_term.key_length = 256
 

	
 
beaker.cache.long_term.type=memory
 
beaker.cache.long_term.expire=36000
 
beaker.cache.long_term.key_length = 256
 

	
 
beaker.cache.sql_cache_short.type=memory
 
beaker.cache.sql_cache_short.expire=10
 
beaker.cache.sql_cache_short.key_length = 256
 

	
 
beaker.cache.sql_cache_med.type=memory
 
beaker.cache.sql_cache_med.expire=360
 
beaker.cache.sql_cache_med.key_length = 256
 

	
 
beaker.cache.sql_cache_long.type=file
 
beaker.cache.sql_cache_long.expire=3600
 
beaker.cache.sql_cache_long.key_length = 256
 

	
 
####################################
 
###       BEAKER SESSION        ####
 
####################################
 
## Type of storage used for the session, current types are 
 
## dbm, file, memcached, database, and memory. 
 
## The storage uses the Container API 
 
##that is also used by the cache system.
 
beaker.session.type = file
 

	
 
beaker.session.key = rhodecode
 
beaker.session.secret = ${app_instance_secret}
 
beaker.session.timeout = 36000
 

	
 
##auto save the session to not to use .save()
 
beaker.session.auto = False
 

	
 
##true exire at browser close
 
#beaker.session.cookie_expires = 3600
 

	
 
    
 
################################################################################
 
## WARNING: *THE LINE BELOW MUST BE UNCOMMENTED ON A PRODUCTION ENVIRONMENT*  ##
 
## Debug mode will enable the interactive debugging tool, allowing ANYONE to  ##
 
## execute malicious code after an exception is raised.                       ##
 
################################################################################
 
set debug = false
 

	
 
##################################
 
###       LOGVIEW CONFIG       ###
 
##################################
 
logview.sqlalchemy = #faa
 
logview.pylons.templating = #bfb
 
logview.pylons.util = #eee
 

	
 
#########################################################
 
### DB CONFIGS - EACH DB WILL HAVE IT'S OWN CONFIG    ###
 
#########################################################
 

	
 
# SQLITE [default]
 
sqlalchemy.db1.url = sqlite:///%(here)s/rhodecode.db
 
 
 
# POSTGRESQL
 
# sqlalchemy.db1.url = postgresql://user:pass@localhost/rhodecode
 

	
 
# MySQL
 
# sqlalchemy.db1.url = mysql://user:pass@localhost/rhodecode
 

	
 
# see sqlalchemy docs for others
 

	
 
sqlalchemy.db1.echo = false
 
sqlalchemy.db1.pool_recycle = 3600
 
sqlalchemy.convert_unicode = true
 

	
 
################################
 
### LOGGING CONFIGURATION   ####
 
################################
 
[loggers]
 
keys = root, routes, rhodecode, sqlalchemy, beaker, templates
 

	
 
[handlers]
 
keys = console, console_sql
 

	
 
[formatters]
 
keys = generic, color_formatter, color_formatter_sql
 

	
 
#############
 
## LOGGERS ##
 
#############
 
[logger_root]
 
level = NOTSET
 
handlers = console
 

	
rhodecode/controllers/summary.py
Show inline comments
 
@@ -15,63 +15,65 @@
 
# the Free Software Foundation, either version 3 of the License, or
 
# (at your option) any later version.
 
#
 
# This program is distributed in the hope that it will be useful,
 
# but WITHOUT ANY WARRANTY; without even the implied warranty of
 
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 
# GNU General Public License for more details.
 
#
 
# You should have received a copy of the GNU General Public License
 
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
 

	
 
import traceback
 
import calendar
 
import logging
 
from time import mktime
 
from datetime import timedelta, date
 
from itertools import product
 

	
 
from vcs.exceptions import ChangesetError, EmptyRepositoryError, \
 
    NodeDoesNotExistError
 

	
 
from pylons import tmpl_context as c, request, url
 
from pylons.i18n.translation import _
 

	
 
from rhodecode.model.db import Statistics
 
from beaker.cache import cache_region, region_invalidate
 

	
 
from rhodecode.model.db import Statistics, CacheInvalidation
 
from rhodecode.lib import ALL_READMES, ALL_EXTS
 
from rhodecode.lib.auth import LoginRequired, HasRepoPermissionAnyDecorator
 
from rhodecode.lib.base import BaseRepoController, render
 
from rhodecode.lib.utils import EmptyChangeset
 
from rhodecode.lib.markup_renderer import MarkupRenderer
 
from rhodecode.lib.celerylib import run_task
 
from rhodecode.lib.celerylib.tasks import get_commits_stats, \
 
    LANGUAGES_EXTENSIONS_MAP
 
from rhodecode.lib.helpers import RepoPage
 
from rhodecode.lib.compat import json, OrderedDict
 

	
 
log = logging.getLogger(__name__)
 

	
 
README_FILES = [''.join([x[0][0], x[1][0]]) for x in 
 
README_FILES = [''.join([x[0][0], x[1][0]]) for x in
 
                    sorted(list(product(ALL_READMES, ALL_EXTS)),
 
                           key=lambda y:y[0][1] + y[1][1])]
 

	
 
class SummaryController(BaseRepoController):
 

	
 
    @LoginRequired()
 
    @HasRepoPermissionAnyDecorator('repository.read', 'repository.write',
 
                                   'repository.admin')
 
    def __before__(self):
 
        super(SummaryController, self).__before__()
 

	
 
    def index(self, repo_name):
 

	
 
        e = request.environ
 
        c.dbrepo = dbrepo = c.rhodecode_db_repo
 

	
 
        c.following = self.scm_model.is_following_repo(repo_name,
 
                                                self.rhodecode_user.user_id)
 

	
 
        def url_generator(**kw):
 
            return url('shortlog_home', repo_name=repo_name, size=10, **kw)
 

	
 
        c.repo_changesets = RepoPage(c.rhodecode_repo, page=1,
 
                                     items_per_page=10, url=url_generator)
 
@@ -145,70 +147,81 @@ class SummaryController(BaseRepoControll
 

	
 
            c.trending_languages = json.dumps(OrderedDict(
 
                                       sorted(lang_stats, reverse=True,
 
                                            key=lambda k: k[1])[:10]
 
                                        )
 
                                    )
 
            last_rev = stats.stat_on_revision
 
            c.repo_last_rev = c.rhodecode_repo.count() - 1 \
 
                if c.rhodecode_repo.revisions else 0
 
            if last_rev == 0 or c.repo_last_rev == 0:
 
                pass
 
            else:
 
                c.stats_percentage = '%.2f' % ((float((last_rev)) /
 
                                                c.repo_last_rev) * 100)
 
        else:
 
            c.commit_data = json.dumps({})
 
            c.overview_data = json.dumps([[ts_min_y, 0], [ts_max_y, 10]])
 
            c.trending_languages = json.dumps({})
 
            c.no_data = True
 

	
 
        c.enable_downloads = dbrepo.enable_downloads
 
        if c.enable_downloads:
 
            c.download_options = self._get_download_links(c.rhodecode_repo)
 

	
 
        c.readme_data,c.readme_file = self.__get_readme_data()
 
        c.readme_data, c.readme_file = self.__get_readme_data(c.rhodecode_repo)
 
        return render('summary/summary.html')
 

	
 
    def __get_readme_data(self):
 
        readme_data = None
 
        readme_file = None
 
        
 
        try:
 
            cs = c.rhodecode_repo.get_changeset('tip')
 
            renderer = MarkupRenderer()
 
            for f in README_FILES:
 
                try:
 
                    readme = cs.get_node(f)
 
                    readme_file = f
 
                    readme_data = renderer.render(readme.content, f)
 
                    break
 
                except NodeDoesNotExistError:
 
                    continue
 
        except ChangesetError:
 
            pass
 
        except EmptyRepositoryError:
 
            pass
 
        except Exception:
 
            log.error(traceback.format_exc())        
 
    def __get_readme_data(self, repo):
 

	
 
        return readme_data, readme_file
 
        @cache_region('long_term')
 
        def _get_readme_from_cache(key):
 
            readme_data = None
 
            readme_file = None
 
            log.debug('Fetching readme file')
 
            try:
 
                cs = repo.get_changeset('tip')
 
                renderer = MarkupRenderer()
 
                for f in README_FILES:
 
                    try:
 
                        readme = cs.get_node(f)
 
                        readme_file = f
 
                        readme_data = renderer.render(readme.content, f)
 
                        log.debug('Found readme %s' % readme_file)
 
                        break
 
                    except NodeDoesNotExistError:
 
                        continue
 
            except ChangesetError:
 
                pass
 
            except EmptyRepositoryError:
 
                pass
 
            except Exception:
 
                log.error(traceback.format_exc())
 

	
 
            return readme_data, readme_file
 

	
 
        key = repo.name + '_README'
 
        inv = CacheInvalidation.invalidate(key)
 
        if inv is not None:
 
            region_invalidate(_get_readme_from_cache, None, key)
 
            CacheInvalidation.set_valid(inv.cache_key)
 
        return _get_readme_from_cache(key)
 

	
 
    def _get_download_links(self, repo):
 

	
 
        download_l = []
 

	
 
        branches_group = ([], _("Branches"))
 
        tags_group = ([], _("Tags"))
 

	
 
        for name, chs in c.rhodecode_repo.branches.items():
 
            #chs = chs.split(':')[-1]
 
            branches_group[0].append((chs, name),)
 
        download_l.append(branches_group)
 

	
 
        for name, chs in c.rhodecode_repo.tags.items():
 
            #chs = chs.split(':')[-1]
 
            tags_group[0].append((chs, name),)
 
        download_l.append(tags_group)
 

	
 
        return download_l
 

	
rhodecode/model/db.py
Show inline comments
 
@@ -642,116 +642,94 @@ class Repository(Base, BaseModel):
 
    # SCM PROPERTIES
 
    #==========================================================================
 

	
 
    def get_changeset(self, rev):
 
        return get_changeset_safe(self.scm_instance, rev)
 

	
 
    @property
 
    def tip(self):
 
        return self.get_changeset('tip')
 

	
 
    @property
 
    def author(self):
 
        return self.tip.author
 

	
 
    @property
 
    def last_change(self):
 
        return self.scm_instance.last_change
 

	
 
    #==========================================================================
 
    # SCM CACHE INSTANCE
 
    #==========================================================================
 

	
 
    @property
 
    def invalidate(self):
 
        """
 
        Returns Invalidation object if this repo should be invalidated
 
        None otherwise. `cache_active = False` means that this cache
 
        state is not valid and needs to be invalidated
 
        """
 
        return CacheInvalidation.query()\
 
            .filter(CacheInvalidation.cache_key == self.repo_name)\
 
            .filter(CacheInvalidation.cache_active == False)\
 
            .scalar()
 
        return CacheInvalidation.invalidate(self.repo_name)
 

	
 
    def set_invalidate(self):
 
        """
 
        set a cache for invalidation for this instance
 
        """
 
        inv = CacheInvalidation.query()\
 
            .filter(CacheInvalidation.cache_key == self.repo_name)\
 
            .scalar()
 

	
 
        if inv is None:
 
            inv = CacheInvalidation(self.repo_name)
 
        inv.cache_active = True
 
        Session.add(inv)
 
        Session.commit()
 
        CacheInvalidation.set_invalidate(self.repo_name)
 

	
 
    @LazyProperty
 
    def scm_instance(self):
 
        return self.__get_instance()
 

	
 
    @property
 
    def scm_instance_cached(self):
 
        @cache_region('long_term')
 
        def _c(repo_name):
 
            return self.__get_instance()
 

	
 
        # TODO: remove this trick when beaker 1.6 is released
 
        # and have fixed this issue with not supporting unicode keys
 
        rn = safe_str(self.repo_name)
 
        rn = self.repo_name
 

	
 
        inv = self.invalidate
 
        if inv is not None:
 
            region_invalidate(_c, None, rn)
 
            # update our cache
 
            inv.cache_active = True
 
            Session.add(inv)
 
            Session.commit()
 

	
 
            CacheInvalidation.set_valid(inv.cache_key)
 
        return _c(rn)
 

	
 
    def __get_instance(self):
 

	
 
        repo_full_path = self.repo_full_path
 

	
 
        try:
 
            alias = get_scm(repo_full_path)[0]
 
            log.debug('Creating instance of %s repository', alias)
 
            backend = get_backend(alias)
 
        except VCSError:
 
            log.error(traceback.format_exc())
 
            log.error('Perhaps this repository is in db and not in '
 
                      'filesystem run rescan repositories with '
 
                      '"destroy old data " option from admin panel')
 
            return
 

	
 
        if alias == 'hg':
 

	
 
            repo = backend(safe_str(repo_full_path), create=False,
 
                           baseui=self._ui)
 
            #skip hidden web repository
 
            # skip hidden web repository
 
            if repo._get_hidden():
 
                return
 
        else:
 
            repo = backend(repo_full_path, create=False)
 

	
 
        return repo
 

	
 

	
 
class Group(Base, BaseModel):
 
    __tablename__ = 'groups'
 
    __table_args__ = (UniqueConstraint('group_name', 'group_parent_id'),
 
                      CheckConstraint('group_id != group_parent_id'), {'extend_existing':True},)
 
    __mapper_args__ = {'order_by':'group_name'}
 

	
 
    group_id = Column("group_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
 
    group_name = Column("group_name", String(length=255, convert_unicode=False, assert_unicode=None), nullable=False, unique=True, default=None)
 
    group_parent_id = Column("group_parent_id", Integer(), ForeignKey('groups.group_id'), nullable=True, unique=None, default=None)
 
    group_description = Column("group_description", String(length=10000, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
 

	
 
    parent_group = relationship('Group', remote_side=group_id)
 

	
 

	
 
    def __init__(self, group_name='', parent_group=None):
 
        self.group_name = group_name
 
@@ -834,49 +812,49 @@ class Group(Base, BaseModel):
 
    @property
 
    def repositories(self):
 
        return Repository.query().filter(Repository.group == self)
 

	
 
    @property
 
    def repositories_recursive_count(self):
 
        cnt = self.repositories.count()
 

	
 
        def children_count(group):
 
            cnt = 0
 
            for child in group.children:
 
                cnt += child.repositories.count()
 
                cnt += children_count(child)
 
            return cnt
 

	
 
        return cnt + children_count(self)
 

	
 

	
 
    def get_new_name(self, group_name):
 
        """
 
        returns new full group name based on parent and new name
 

	
 
        :param group_name:
 
        """
 
        path_prefix = (self.parent_group.full_path_splitted if 
 
        path_prefix = (self.parent_group.full_path_splitted if
 
                       self.parent_group else [])
 
        return Group.url_sep().join(path_prefix + [group_name])
 

	
 

	
 
class Permission(Base, BaseModel):
 
    __tablename__ = 'permissions'
 
    __table_args__ = {'extend_existing':True}
 
    permission_id = Column("permission_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
 
    permission_name = Column("permission_name", String(length=255, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
 
    permission_longname = Column("permission_longname", String(length=255, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
 

	
 
    def __repr__(self):
 
        return "<%s('%s:%s')>" % (self.__class__.__name__,
 
                                  self.permission_id, self.permission_name)
 

	
 
    @classmethod
 
    def get_by_key(cls, key):
 
        return cls.query().filter(cls.permission_name == key).scalar()
 

	
 
class RepoToPerm(Base, BaseModel):
 
    __tablename__ = 'repo_to_perm'
 
    __table_args__ = (UniqueConstraint('user_id', 'repository_id'), {'extend_existing':True})
 
    repo_to_perm_id = Column("repo_to_perm_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
 
    user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=None, default=None)
 
@@ -1039,31 +1017,82 @@ class UserFollowing(Base, BaseModel):
 

	
 

	
 
    @classmethod
 
    def get_repo_followers(cls, repo_id):
 
        return cls.query().filter(cls.follows_repo_id == repo_id)
 

	
 
class CacheInvalidation(Base, BaseModel):
 
    __tablename__ = 'cache_invalidation'
 
    __table_args__ = (UniqueConstraint('cache_key'), {'extend_existing':True})
 
    cache_id = Column("cache_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
 
    cache_key = Column("cache_key", String(length=255, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
 
    cache_args = Column("cache_args", String(length=255, convert_unicode=False, assert_unicode=None), nullable=True, unique=None, default=None)
 
    cache_active = Column("cache_active", Boolean(), nullable=True, unique=None, default=False)
 

	
 

	
 
    def __init__(self, cache_key, cache_args=''):
 
        self.cache_key = cache_key
 
        self.cache_args = cache_args
 
        self.cache_active = False
 

	
 
    def __repr__(self):
 
        return "<%s('%s:%s')>" % (self.__class__.__name__,
 
                                  self.cache_id, self.cache_key)
 

	
 
    @classmethod
 
    def invalidate(cls, key):
 
        """
 
        Returns Invalidation object if this given key should be invalidated
 
        None otherwise. `cache_active = False` means that this cache
 
        state is not valid and needs to be invalidated
 
        
 
        :param key:
 
        """
 
        return cls.query()\
 
                .filter(CacheInvalidation.cache_key == key)\
 
                .filter(CacheInvalidation.cache_active == False)\
 
                .scalar()
 

	
 
    @classmethod
 
    def set_invalidate(cls, key):
 
        """
 
        Mark this Cache key for invalidation
 
        
 
        :param key:
 
        """
 

	
 
        log.debug('marking %s for invalidation' % key)
 
        inv_obj = Session().query(cls)\
 
            .filter(cls.cache_key == key).scalar()
 
        if inv_obj:
 
            inv_obj.cache_active = False
 
        else:
 
            log.debug('cache key not found in invalidation db -> creating one')
 
            inv_obj = CacheInvalidation(key)
 

	
 
        try:
 
            Session.add(inv_obj)
 
            Session.commit()
 
        except Exception:
 
            log.error(traceback.format_exc())
 
            Session.rollback()
 

	
 
    @classmethod
 
    def set_valid(cls, key):
 
        """
 
        Mark this cache key as active and currently cached
 
        
 
        :param key:
 
        """
 
        inv_obj = Session().query(CacheInvalidation)\
 
            .filter(CacheInvalidation.cache_key == key).scalar()
 
        inv_obj.cache_active = True
 
        Session.add(inv_obj)
 
        Session.commit()
 

	
 
class DbMigrateVersion(Base, BaseModel):
 
    __tablename__ = 'db_migrate_version'
 
    __table_args__ = {'extend_existing':True}
 
    repository_id = Column('repository_id', String(250), primary_key=True)
 
    repository_path = Column('repository_path', Text)
 
    version = Column('version', Integer)
 

	
rhodecode/model/scm.py
Show inline comments
 
@@ -176,66 +176,50 @@ class ScmModel(BaseModel):
 
    def get_repos(self, all_repos=None, sort_key=None):
 
        """
 
        Get all repos from db and for each repo create it's
 
        backend instance and fill that backed with information from database
 

	
 
        :param all_repos: list of repository names as strings
 
            give specific repositories list, good for filtering
 
        """
 
        if all_repos is None:
 
            all_repos = self.sa.query(Repository)\
 
                        .filter(Repository.group_id == None)\
 
                        .order_by(Repository.repo_name).all()
 

	
 
        repo_iter = CachedRepoList(all_repos, repos_path=self.repos_path,
 
                                   order_by=sort_key)
 

	
 
        return repo_iter
 

	
 
    def mark_for_invalidation(self, repo_name):
 
        """Puts cache invalidation task into db for
 
        further global cache invalidation
 

	
 
        :param repo_name: this repo that should invalidation take place
 
        """
 

	
 
        log.debug('marking %s for invalidation', repo_name)
 
        cache = self.sa.query(CacheInvalidation)\
 
            .filter(CacheInvalidation.cache_key == repo_name).scalar()
 

	
 
        if cache:
 
            # mark this cache as inactive
 
            cache.cache_active = False
 
        else:
 
            log.debug('cache key not found in invalidation db -> creating one')
 
            cache = CacheInvalidation(repo_name)
 

	
 
        try:
 
            self.sa.add(cache)
 
            self.sa.commit()
 
        except (DatabaseError,):
 
            log.error(traceback.format_exc())
 
            self.sa.rollback()
 
        CacheInvalidation.set_invalidate(repo_name)
 
        CacheInvalidation.set_invalidate(repo_name+"_README")
 

	
 
    def toggle_following_repo(self, follow_repo_id, user_id):
 

	
 
        f = self.sa.query(UserFollowing)\
 
            .filter(UserFollowing.follows_repo_id == follow_repo_id)\
 
            .filter(UserFollowing.user_id == user_id).scalar()
 

	
 
        if f is not None:
 

	
 
            try:
 
                self.sa.delete(f)
 
                self.sa.commit()
 
                action_logger(UserTemp(user_id),
 
                              'stopped_following_repo',
 
                              RepoTemp(follow_repo_id))
 
                return
 
            except:
 
                log.error(traceback.format_exc())
 
                self.sa.rollback()
 
                raise
 

	
 
        try:
 
            f = UserFollowing()
 
            f.user_id = user_id
 
@@ -374,41 +358,26 @@ class ScmModel(BaseModel):
 
            content = content.read()
 

	
 
        message = safe_str(message)
 
        path = safe_str(f_path)
 
        author = safe_str(author)
 
        m = IMC(repo)
 

	
 
        if isinstance(cs, EmptyChangeset):
 
            # Emptychangeset means we we're editing empty repository
 
            parents = None
 
        else:
 
            parents = [cs]
 

	
 
        m.add(FileNode(path, content=content))
 
        tip = m.commit(message=message,
 
                 author=author,
 
                 parents=parents, branch=cs.branch)
 
        new_cs = tip.short_id
 
        action = 'push_local:%s' % new_cs
 

	
 
        action_logger(user, action, repo_name)
 

	
 
        self.mark_for_invalidation(repo_name)
 

	
 

	
 
    def get_unread_journal(self):
 
        return self.sa.query(UserLog).count()
 

	
 
    def _should_invalidate(self, repo_name):
 
        """Looks up database for invalidation signals for this repo_name
 

	
 
        :param repo_name:
 
        """
 

	
 
        ret = self.sa.query(CacheInvalidation)\
 
            .filter(CacheInvalidation.cache_key == repo_name)\
 
            .filter(CacheInvalidation.cache_active == False)\
 
            .scalar()
 

	
 
        return ret
 

	
test.ini
Show inline comments
 
@@ -67,104 +67,110 @@ celery.imports = rhodecode.lib.celerylib
 
celery.result.backend = amqp
 
celery.result.dburi = amqp://
 
celery.result.serialier = json
 

	
 
#celery.send.task.error.emails = true
 
#celery.amqp.task.result.expires = 18000
 

	
 
celeryd.concurrency = 2
 
#celeryd.log.file = celeryd.log
 
celeryd.log.level = debug
 
celeryd.max.tasks.per.child = 1
 

	
 
#tasks will never be sent to the queue, but executed locally instead.
 
celery.always.eager = false
 

	
 
####################################
 
###         BEAKER CACHE        ####
 
####################################
 
beaker.cache.data_dir=/tmp/data/cache/data
 
beaker.cache.lock_dir=/tmp/data/cache/lock
 
beaker.cache.regions=super_short_term,short_term,long_term,sql_cache_short,sql_cache_med,sql_cache_long
 

	
 
beaker.cache.super_short_term.type=memory
 
beaker.cache.super_short_term.expire=10
 
beaker.cache.super_short_term.key_length = 256
 

	
 
beaker.cache.short_term.type=memory
 
beaker.cache.short_term.expire=60
 
beaker.cache.short_term.key_length = 256
 

	
 
beaker.cache.long_term.type=memory
 
beaker.cache.long_term.expire=36000
 
beaker.cache.long_term.key_length = 256
 

	
 
beaker.cache.sql_cache_short.type=memory
 
beaker.cache.sql_cache_short.expire=10
 
beaker.cache.sql_cache_short.key_length = 256
 

	
 
beaker.cache.sql_cache_med.type=memory
 
beaker.cache.sql_cache_med.expire=360
 
beaker.cache.sql_cache_med.key_length = 256
 

	
 
beaker.cache.sql_cache_long.type=file
 
beaker.cache.sql_cache_long.expire=3600
 
beaker.cache.sql_cache_long.key_length = 256
 

	
 
####################################
 
###       BEAKER SESSION        ####
 
####################################
 
## Type of storage used for the session, current types are 
 
## dbm, file, memcached, database, and memory. 
 
## The storage uses the Container API 
 
##that is also used by the cache system.
 
beaker.session.type = file
 

	
 
beaker.session.key = rhodecode
 
beaker.session.secret = g654dcno0-9873jhgfreyu
 
beaker.session.timeout = 36000
 

	
 
##auto save the session to not to use .save()
 
beaker.session.auto = False
 

	
 
##true exire at browser close
 
#beaker.session.cookie_expires = 3600
 

	
 
    
 
################################################################################
 
## WARNING: *THE LINE BELOW MUST BE UNCOMMENTED ON A PRODUCTION ENVIRONMENT*  ##
 
## Debug mode will enable the interactive debugging tool, allowing ANYONE to  ##
 
## execute malicious code after an exception is raised.                       ##
 
################################################################################
 
#set debug = false
 

	
 
##################################
 
###       LOGVIEW CONFIG       ###
 
##################################
 
logview.sqlalchemy = #faa
 
logview.pylons.templating = #bfb
 
logview.pylons.util = #eee
 

	
 
#########################################################
 
### DB CONFIGS - EACH DB WILL HAVE IT'S OWN CONFIG    ###
 
#########################################################
 
sqlalchemy.db1.url = sqlite:///%(here)s/test.db
 
#sqlalchemy.db1.url = postgresql://postgres:qwe@localhost/rhodecode_tests
 
#sqlalchemy.db1.echo = False
 
#sqlalchemy.db1.echo = false
 
#sqlalchemy.db1.pool_recycle = 3600
 
sqlalchemy.convert_unicode = true
 

	
 
################################
 
### LOGGING CONFIGURATION   ####
 
################################
 
[loggers]
 
keys = root, routes, rhodecode, sqlalchemy, beaker, templates
 

	
 
[handlers]
 
keys = console
 

	
 
[formatters]
 
keys = generic, color_formatter
 

	
 
#############
 
## LOGGERS ##
 
#############
 
[logger_root]
 
level = ERROR
 
handlers = console
 

	
 
[logger_routes]
 
level = ERROR
0 comments (0 inline, 0 general)