kallithea Changeset - e886f91fcb71

Changeset - e886f91fcb71

Parent rev.

Child rev.

[Not reviewed]

beta

0 7 0

Marcin Kuzminski - 14 years ago 2011-10-28 02:40:22
marcin@python-works.com

Cached readme generation
- cleaned up cache system

7 files changed with 123 insertions and 87 deletions:

development.ini

production.ini

rhodecode/config/deployment.ini_tmpl

rhodecode/controllers/summary.py

rhodecode/model/db.py

rhodecode/model/scm.py

test.ini

0 comments (0 inline, 0 general)

development.ini

➞

Show inline comments

@@ @@ -91,21 +91,27 @@ beaker.cache.regions=super_short_term,sh @@
 beaker.cache.super_short_term.type=memory
 beaker.cache.super_short_term.expire=10
 beaker.cache.super_short_term.key_length = 256
 beaker.cache.short_term.type=memory
 beaker.cache.short_term.expire=60
 beaker.cache.short_term.key_length = 256
 beaker.cache.long_term.type=memory
 beaker.cache.long_term.expire=36000
 beaker.cache.long_term.key_length = 256
 beaker.cache.sql_cache_short.type=memory
 beaker.cache.sql_cache_short.expire=10
 beaker.cache.sql_cache_short.key_length = 256
 beaker.cache.sql_cache_med.type=memory
 beaker.cache.sql_cache_med.expire=360
 beaker.cache.sql_cache_med.key_length = 256
 beaker.cache.sql_cache_long.type=file
 beaker.cache.sql_cache_long.expire=3600
 beaker.cache.sql_cache_long.key_length = 256
 ####################################
 ###       BEAKER SESSION        ####

production.ini

➞

Show inline comments

@@ @@ -91,21 +91,27 @@ beaker.cache.regions=super_short_term,sh @@
 beaker.cache.super_short_term.type=memory
 beaker.cache.super_short_term.expire=10
 beaker.cache.super_short_term.key_length = 256
 beaker.cache.short_term.type=memory
 beaker.cache.short_term.expire=60
 beaker.cache.short_term.key_length = 256
 beaker.cache.long_term.type=memory
 beaker.cache.long_term.expire=36000
 beaker.cache.long_term.key_length = 256
 beaker.cache.sql_cache_short.type=memory
 beaker.cache.sql_cache_short.expire=10
 beaker.cache.sql_cache_short.key_length = 256
 beaker.cache.sql_cache_med.type=memory
 beaker.cache.sql_cache_med.expire=360
 beaker.cache.sql_cache_med.key_length = 256
 beaker.cache.sql_cache_long.type=file
 beaker.cache.sql_cache_long.expire=3600
 beaker.cache.sql_cache_long.key_length = 256
 ####################################
 ###       BEAKER SESSION        ####

rhodecode/config/deployment.ini_tmpl

➞

Show inline comments

@@ @@ -91,21 +91,27 @@ beaker.cache.regions=super_short_term,sh @@
 beaker.cache.super_short_term.type=memory
 beaker.cache.super_short_term.expire=10
 beaker.cache.super_short_term.key_length = 256
 beaker.cache.short_term.type=memory
 beaker.cache.short_term.expire=60
 beaker.cache.short_term.key_length = 256
 beaker.cache.long_term.type=memory
 beaker.cache.long_term.expire=36000
 beaker.cache.long_term.key_length = 256
 beaker.cache.sql_cache_short.type=memory
 beaker.cache.sql_cache_short.expire=10
 beaker.cache.sql_cache_short.key_length = 256
 beaker.cache.sql_cache_med.type=memory
 beaker.cache.sql_cache_med.expire=360
 beaker.cache.sql_cache_med.key_length = 256
 beaker.cache.sql_cache_long.type=file
 beaker.cache.sql_cache_long.expire=3600
 beaker.cache.sql_cache_long.key_length = 256
 ####################################
 ###       BEAKER SESSION        ####
@@ @@ -154,6 +160,7 @@ sqlalchemy.db1.url = sqlite:///%(here)s/ @@
 # MySQL
 # sqlalchemy.db1.url = mysql://user:pass@localhost/rhodecode
 # see sqlalchemy docs for others
 sqlalchemy.db1.echo = false
 sqlalchemy.db1.pool_recycle = 3600

rhodecode/controllers/summary.py

➞

Show inline comments

@@ @@ -36,7 +36,9 @@ from vcs.exceptions import ChangesetErro @@
 from pylons import tmpl_context as c, request, url
 from pylons.i18n.translation import _
 from rhodecode.model.db import Statistics
 from beaker.cache import cache_region, region_invalidate
 from rhodecode.model.db import Statistics, CacheInvalidation
 from rhodecode.lib import ALL_READMES, ALL_EXTS
 from rhodecode.lib.auth import LoginRequired, HasRepoPermissionAnyDecorator
 from rhodecode.lib.base import BaseRepoController, render
@@ @@ -50,7 +52,7 @@ from rhodecode.lib.compat import json, O @@
 log = logging.getLogger(__name__)
-README_FILES = [''.join([x[0][0], x[1][0]]) for x in
 README_FILES = [''.join([x[0][0], x[1][0]]) for x in
                     sorted(list(product(ALL_READMES, ALL_EXTS)),
                            key=lambda y:y[0][1] + y[1][1])]
@@ @@ -166,32 +168,43 @@ class SummaryController(BaseRepoControll @@
         if c.enable_downloads:
             c.download_options = self._get_download_links(c.rhodecode_repo)
         c.readme_data,c.readme_file = self.__get_readme_data()
+        c.readme_data, c.readme_file = self.__get_readme_data(c.rhodecode_repo)
         return render('summary/summary.html')
     def __get_readme_data(self):
         readme_data = None
         readme_file = None
         try:
             cs = c.rhodecode_repo.get_changeset('tip')
             renderer = MarkupRenderer()
             for f in README_FILES:
                 try:
                     readme = cs.get_node(f)
                     readme_file = f
                     readme_data = renderer.render(readme.content, f)
                     break
                 except NodeDoesNotExistError:
                     continue
         except ChangesetError:
             pass
         except EmptyRepositoryError:
             pass
         except Exception:
             log.error(traceback.format_exc())
     def __get_readme_data(self, repo):
         return readme_data, readme_file
         @cache_region('long_term')
         def _get_readme_from_cache(key):
             readme_data = None
             readme_file = None
             log.debug('Fetching readme file')
             try:
                 cs = repo.get_changeset('tip')
                 renderer = MarkupRenderer()
                 for f in README_FILES:
                     try:
                         readme = cs.get_node(f)
                         readme_file = f
                         readme_data = renderer.render(readme.content, f)
                         log.debug('Found readme %s' % readme_file)
                         break
                     except NodeDoesNotExistError:
                         continue
             except ChangesetError:
                 pass
             except EmptyRepositoryError:
                 pass
             except Exception:
                 log.error(traceback.format_exc())
             return readme_data, readme_file
         key = repo.name + '_README'
         inv = CacheInvalidation.invalidate(key)
         if inv is not None:
             region_invalidate(_get_readme_from_cache, None, key)
             CacheInvalidation.set_valid(inv.cache_key)
         return _get_readme_from_cache(key)
     def _get_download_links(self, repo):

rhodecode/model/db.py

➞

Show inline comments

@@ @@ -663,29 +663,13 @@ class Repository(Base, BaseModel): @@
     @property
     def invalidate(self):
         """
         Returns Invalidation object if this repo should be invalidated
         None otherwise. `cache_active = False` means that this cache
         state is not valid and needs to be invalidated
         """
         return CacheInvalidation.query()\
             .filter(CacheInvalidation.cache_key == self.repo_name)\
             .filter(CacheInvalidation.cache_active == False)\
             .scalar()
         return CacheInvalidation.invalidate(self.repo_name)
     def set_invalidate(self):
         """
         set a cache for invalidation for this instance
         """
         inv = CacheInvalidation.query()\
             .filter(CacheInvalidation.cache_key == self.repo_name)\
             .scalar()
         if inv is None:
             inv = CacheInvalidation(self.repo_name)
         inv.cache_active = True
         Session.add(inv)
         Session.commit()
         CacheInvalidation.set_invalidate(self.repo_name)
     @LazyProperty
     def scm_instance(self):
@@ @@ -696,19 +680,13 @@ class Repository(Base, BaseModel): @@
         @cache_region('long_term')
         def _c(repo_name):
             return self.__get_instance()
         # TODO: remove this trick when beaker 1.6 is released
         # and have fixed this issue with not supporting unicode keys
         rn = safe_str(self.repo_name)
         rn = self.repo_name
         inv = self.invalidate
         if inv is not None:
             region_invalidate(_c, None, rn)
             # update our cache
             inv.cache_active = True
             Session.add(inv)
             Session.commit()
             CacheInvalidation.set_valid(inv.cache_key)
         return _c(rn)
     def __get_instance(self):
@@ @@ -730,7 +708,7 @@ class Repository(Base, BaseModel): @@
             repo = backend(safe_str(repo_full_path), create=False,
                            baseui=self._ui)
             #skip hidden web repository
+            # skip hidden web repository
             if repo._get_hidden():
                 return
         else:
@@ @@ -855,7 +833,7 @@ class Group(Base, BaseModel): @@
         :param group_name:
         """
-        path_prefix = (self.parent_group.full_path_splitted if
         path_prefix = (self.parent_group.full_path_splitted if
                        self.parent_group else [])
         return Group.url_sep().join(path_prefix + [group_name])
@@ @@ -1060,6 +1038,57 @@ class CacheInvalidation(Base, BaseModel) @@
         return "<%s('%s:%s')>" % (self.__class__.__name__,
                                   self.cache_id, self.cache_key)
     @classmethod
     def invalidate(cls, key):
         """
         Returns Invalidation object if this given key should be invalidated
         None otherwise. `cache_active = False` means that this cache
         state is not valid and needs to be invalidated
         :param key:
         """
         return cls.query()\
                 .filter(CacheInvalidation.cache_key == key)\
                 .filter(CacheInvalidation.cache_active == False)\
                 .scalar()
     @classmethod
     def set_invalidate(cls, key):
         """
         Mark this Cache key for invalidation
         :param key:
         """
         log.debug('marking %s for invalidation' % key)
         inv_obj = Session().query(cls)\
             .filter(cls.cache_key == key).scalar()
         if inv_obj:
             inv_obj.cache_active = False
         else:
             log.debug('cache key not found in invalidation db -> creating one')
             inv_obj = CacheInvalidation(key)
         try:
             Session.add(inv_obj)
             Session.commit()
         except Exception:
             log.error(traceback.format_exc())
             Session.rollback()
     @classmethod
     def set_valid(cls, key):
         """
         Mark this cache key as active and currently cached
         :param key:
         """
         inv_obj = Session().query(CacheInvalidation)\
             .filter(CacheInvalidation.cache_key == key).scalar()
         inv_obj.cache_active = True
         Session.add(inv_obj)
         Session.commit()
 class DbMigrateVersion(Base, BaseModel):
     __tablename__ = 'db_migrate_version'
     __table_args__ = {'extend_existing':True}

rhodecode/model/scm.py

➞

Show inline comments

@@ @@ -197,24 +197,8 @@ class ScmModel(BaseModel): @@
         :param repo_name: this repo that should invalidation take place
         """
         log.debug('marking %s for invalidation', repo_name)
         cache = self.sa.query(CacheInvalidation)\
             .filter(CacheInvalidation.cache_key == repo_name).scalar()
         if cache:
             # mark this cache as inactive
             cache.cache_active = False
         else:
             log.debug('cache key not found in invalidation db -> creating one')
             cache = CacheInvalidation(repo_name)
         try:
             self.sa.add(cache)
             self.sa.commit()
         except (DatabaseError,):
             log.error(traceback.format_exc())
             self.sa.rollback()
         CacheInvalidation.set_invalidate(repo_name)
         CacheInvalidation.set_invalidate(repo_name+"_README")
     def toggle_following_repo(self, follow_repo_id, user_id):
@@ @@ -395,20 +379,5 @@ class ScmModel(BaseModel): @@
         self.mark_for_invalidation(repo_name)
     def get_unread_journal(self):
         return self.sa.query(UserLog).count()
     def _should_invalidate(self, repo_name):
         """Looks up database for invalidation signals for this repo_name
         :param repo_name:
         """
         ret = self.sa.query(CacheInvalidation)\
             .filter(CacheInvalidation.cache_key == repo_name)\
             .filter(CacheInvalidation.cache_active == False)\
             .scalar()
         return ret

test.ini

➞

Show inline comments

@@ @@ -88,21 +88,27 @@ beaker.cache.regions=super_short_term,sh @@
 beaker.cache.super_short_term.type=memory
 beaker.cache.super_short_term.expire=10
 beaker.cache.super_short_term.key_length = 256
 beaker.cache.short_term.type=memory
 beaker.cache.short_term.expire=60
 beaker.cache.short_term.key_length = 256
 beaker.cache.long_term.type=memory
 beaker.cache.long_term.expire=36000
 beaker.cache.long_term.key_length = 256
 beaker.cache.sql_cache_short.type=memory
 beaker.cache.sql_cache_short.expire=10
 beaker.cache.sql_cache_short.key_length = 256
 beaker.cache.sql_cache_med.type=memory
 beaker.cache.sql_cache_med.expire=360
 beaker.cache.sql_cache_med.key_length = 256
 beaker.cache.sql_cache_long.type=file
 beaker.cache.sql_cache_long.expire=3600
 beaker.cache.sql_cache_long.key_length = 256
 ####################################
 ###       BEAKER SESSION        ####
@@ @@ -143,7 +149,7 @@ logview.pylons.util = #eee @@
 #########################################################
 sqlalchemy.db1.url = sqlite:///%(here)s/test.db
 #sqlalchemy.db1.url = postgresql://postgres:qwe@localhost/rhodecode_tests
-#sqlalchemy.db1.echo = False
+#sqlalchemy.db1.echo = false
 #sqlalchemy.db1.pool_recycle = 3600
 sqlalchemy.convert_unicode = true

0 comments (0 inline, 0 general)