Changeset - c0ac34b8eb07
[Not reviewed]
beta
0 5 0
Marcin Kuzminski - 13 years ago 2013-03-06 18:47:59
marcin@python-works.com
initial version of #788 tarball cache
5 files changed with 48 insertions and 9 deletions:
0 comments (0 inline, 0 general)
development.ini
Show inline comments
 
@@ -20,96 +20,98 @@ pdebug = false
 
#email_prefix = [RhodeCode]
 

	
 
#smtp_server = mail.server.com
 
#smtp_username = 
 
#smtp_password = 
 
#smtp_port = 
 
#smtp_use_tls = false
 
#smtp_use_ssl = true
 
# Specify available auth parameters here (e.g. LOGIN PLAIN CRAM-MD5, etc.)
 
#smtp_auth = 
 

	
 
[server:main]
 
## PASTE
 
##nr of threads to spawn
 
#threadpool_workers = 5
 

	
 
##max request before thread respawn
 
#threadpool_max_requests = 10
 

	
 
##option to use threads of process
 
#use_threadpool = true
 

	
 
#use = egg:Paste#http
 

	
 
#WAITRESS
 
threads = 5
 
#100GB
 
max_request_body_size = 107374182400
 
use = egg:waitress#main
 

	
 
host = 0.0.0.0
 
port = 5000
 

	
 
[filter:proxy-prefix]
 
# prefix middleware for rc
 
use = egg:PasteDeploy#prefix
 
prefix = /<your-prefix>
 

	
 
[app:main]
 
use = egg:rhodecode
 
#filter-with = proxy-prefix
 
full_stack = true
 
static_files = true
 
# Optional Languages
 
# en, fr, ja, pt_BR, zh_CN, zh_TW, pl
 
lang = en
 
cache_dir = %(here)s/data
 
index_dir = %(here)s/data/index
 
# set this path to use archive download cache
 
#archive_cache_dir = /tmp/rhodecode_tarballcache
 
app_instance_uuid = rc-develop
 
cut_off_limit = 256000
 
vcs_full_cache = True
 
# force https in RhodeCode, fixes https redirects, assumes it's always https
 
force_https = false
 
# use Strict-Transport-Security headers
 
use_htsts = false
 
commit_parse_limit = 25
 
# number of items displayed in lightweight dashboard before paginating
 
dashboard_items = 100
 
use_gravatar = true
 

	
 
# path to git executable
 
git_path = git
 

	
 
## RSS feed options
 

	
 
rss_cut_off_limit = 256000
 
rss_items_per_page = 10
 
rss_include_diff = false
 

	
 

	
 
## alternative_gravatar_url allows you to use your own avatar server application
 
## the following parts of the URL will be replaced
 
## {email}        user email
 
## {md5email}     md5 hash of the user email (like at gravatar.com)
 
## {size}         size of the image that is expected from the server application
 
## {scheme}       http/https from RhodeCode server
 
## {netloc}       network location from RhodeCode server
 
#alternative_gravatar_url = http://myavatarserver.com/getbyemail/{email}/{size}
 
#alternative_gravatar_url = http://myavatarserver.com/getbymd5/{md5email}?s={size}
 

	
 
container_auth_enabled = false
 
proxypass_auth_enabled = false
 
## default encoding used to convert from and to unicode
 
## can be also a comma seperated list of encoding in case of mixed encodings
 
default_encoding = utf8
 

	
 
## overwrite schema of clone url
 
## available vars:
 
## scheme - http/https
 
## user - current user
 
## pass - password 
 
## netloc - network location
 
## path - usually repo_name
 

	
 
#clone_uri = {scheme}://{user}{pass}{netloc}{path}
 

	
production.ini
Show inline comments
 
@@ -20,96 +20,98 @@ pdebug = false
 
#email_prefix = [RhodeCode]
 

	
 
#smtp_server = mail.server.com
 
#smtp_username = 
 
#smtp_password = 
 
#smtp_port = 
 
#smtp_use_tls = false
 
#smtp_use_ssl = true
 
# Specify available auth parameters here (e.g. LOGIN PLAIN CRAM-MD5, etc.)
 
#smtp_auth = 
 

	
 
[server:main]
 
## PASTE
 
##nr of threads to spawn
 
#threadpool_workers = 5
 

	
 
##max request before thread respawn
 
#threadpool_max_requests = 10
 

	
 
##option to use threads of process
 
#use_threadpool = true
 

	
 
#use = egg:Paste#http
 

	
 
#WAITRESS
 
threads = 5
 
#100GB
 
max_request_body_size = 107374182400
 
use = egg:waitress#main
 

	
 
host = 127.0.0.1
 
port = 8001
 

	
 
[filter:proxy-prefix]
 
# prefix middleware for rc
 
use = egg:PasteDeploy#prefix
 
prefix = /<your-prefix>
 

	
 
[app:main]
 
use = egg:rhodecode
 
#filter-with = proxy-prefix
 
full_stack = true
 
static_files = true
 
# Optional Languages
 
# en, fr, ja, pt_BR, zh_CN, zh_TW, pl
 
lang = en
 
cache_dir = %(here)s/data
 
index_dir = %(here)s/data/index
 
# set this path to use archive download cache
 
#archive_cache_dir = /tmp/rhodecode_tarballcache
 
app_instance_uuid = rc-production
 
cut_off_limit = 256000
 
vcs_full_cache = True
 
# force https in RhodeCode, fixes https redirects, assumes it's always https
 
force_https = false
 
# use Strict-Transport-Security headers
 
use_htsts = false
 
commit_parse_limit = 50
 
# number of items displayed in lightweight dashboard before paginating
 
dashboard_items = 100
 
use_gravatar = true
 

	
 
# path to git executable
 
git_path = git
 

	
 
## RSS feed options
 

	
 
rss_cut_off_limit = 256000
 
rss_items_per_page = 10
 
rss_include_diff = false
 

	
 

	
 
## alternative_gravatar_url allows you to use your own avatar server application
 
## the following parts of the URL will be replaced
 
## {email}        user email
 
## {md5email}     md5 hash of the user email (like at gravatar.com)
 
## {size}         size of the image that is expected from the server application
 
## {scheme}       http/https from RhodeCode server
 
## {netloc}       network location from RhodeCode server
 
#alternative_gravatar_url = http://myavatarserver.com/getbyemail/{email}/{size}
 
#alternative_gravatar_url = http://myavatarserver.com/getbymd5/{md5email}?s={size}
 

	
 
container_auth_enabled = false
 
proxypass_auth_enabled = false
 
## default encoding used to convert from and to unicode
 
## can be also a comma seperated list of encoding in case of mixed encodings
 
default_encoding = utf8
 

	
 
## overwrite schema of clone url
 
## available vars:
 
## scheme - http/https
 
## user - current user
 
## pass - password 
 
## netloc - network location
 
## path - usually repo_name
 

	
 
#clone_uri = {scheme}://{user}{pass}{netloc}{path}
 

	
rhodecode/config/deployment.ini_tmpl
Show inline comments
 
@@ -20,96 +20,98 @@ pdebug = false
 
#email_prefix = [RhodeCode]
 

	
 
#smtp_server = mail.server.com
 
#smtp_username = 
 
#smtp_password = 
 
#smtp_port = 
 
#smtp_use_tls = false
 
#smtp_use_ssl = true
 
# Specify available auth parameters here (e.g. LOGIN PLAIN CRAM-MD5, etc.)
 
#smtp_auth = 
 

	
 
[server:main]
 
## PASTE
 
##nr of threads to spawn
 
#threadpool_workers = 5
 

	
 
##max request before thread respawn
 
#threadpool_max_requests = 10
 

	
 
##option to use threads of process
 
#use_threadpool = true
 

	
 
#use = egg:Paste#http
 

	
 
#WAITRESS
 
threads = 5
 
#100GB
 
max_request_body_size = 107374182400
 
use = egg:waitress#main
 

	
 
host = 127.0.0.1
 
port = 5000
 

	
 
[filter:proxy-prefix]
 
# prefix middleware for rc
 
use = egg:PasteDeploy#prefix
 
prefix = /<your-prefix>
 

	
 
[app:main]
 
use = egg:rhodecode
 
#filter-with = proxy-prefix
 
full_stack = true
 
static_files = true
 
# Optional Languages
 
# en, fr, ja, pt_BR, zh_CN, zh_TW, pl
 
lang = en
 
cache_dir = %(here)s/data
 
index_dir = %(here)s/data/index
 
# set this path to use archive download cache
 
#archive_cache_dir = /tmp/rhodecode_tarballcache
 
app_instance_uuid = ${app_instance_uuid}
 
cut_off_limit = 256000
 
vcs_full_cache = True
 
# force https in RhodeCode, fixes https redirects, assumes it's always https
 
force_https = false
 
# use Strict-Transport-Security headers
 
use_htsts = false
 
commit_parse_limit = 50
 
# number of items displayed in lightweight dashboard before paginating
 
dashboard_items = 100
 
use_gravatar = true
 

	
 
# path to git executable
 
git_path = git
 

	
 
## RSS feed options
 

	
 
rss_cut_off_limit = 256000
 
rss_items_per_page = 10
 
rss_include_diff = false
 

	
 

	
 
## alternative_gravatar_url allows you to use your own avatar server application
 
## the following parts of the URL will be replaced
 
## {email}        user email
 
## {md5email}     md5 hash of the user email (like at gravatar.com)
 
## {size}         size of the image that is expected from the server application
 
## {scheme}       http/https from RhodeCode server
 
## {netloc}       network location from RhodeCode server
 
#alternative_gravatar_url = http://myavatarserver.com/getbyemail/{email}/{size}
 
#alternative_gravatar_url = http://myavatarserver.com/getbymd5/{md5email}?s={size}
 

	
 
container_auth_enabled = false
 
proxypass_auth_enabled = false
 
## default encoding used to convert from and to unicode
 
## can be also a comma seperated list of encoding in case of mixed encodings
 
default_encoding = utf8
 

	
 
## overwrite schema of clone url
 
## available vars:
 
## scheme - http/https
 
## user - current user
 
## pass - password 
 
## netloc - network location
 
## path - usually repo_name
 

	
 
#clone_uri = {scheme}://{user}{pass}{netloc}{path}
 

	
rhodecode/controllers/files.py
Show inline comments
 
# -*- coding: utf-8 -*-
 
"""
 
    rhodecode.controllers.files
 
    ~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

	
 
    Files controller for RhodeCode
 

	
 
    :created_on: Apr 21, 2010
 
    :author: marcink
 
    :copyright: (C) 2010-2012 Marcin Kuzminski <marcin@python-works.com>
 
    :license: GPLv3, see COPYING for more details.
 
"""
 
# This program is free software: you can redistribute it and/or modify
 
# it under the terms of the GNU General Public License as published by
 
# the Free Software Foundation, either version 3 of the License, or
 
# (at your option) any later version.
 
#
 
# This program is distributed in the hope that it will be useful,
 
# but WITHOUT ANY WARRANTY; without even the implied warranty of
 
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 
# GNU General Public License for more details.
 
#
 
# You should have received a copy of the GNU General Public License
 
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
from __future__ import with_statement
 
import os
 
import logging
 
import traceback
 
import tempfile
 
import shutil
 

	
 
from pylons import request, response, tmpl_context as c, url
 
from pylons.i18n.translation import _
 
from pylons.controllers.util import redirect
 
from rhodecode.lib.utils import jsonify
 

	
 
from rhodecode.lib import diffs
 
from rhodecode.lib import helpers as h
 

	
 
from rhodecode.lib.compat import OrderedDict
 
from rhodecode.lib.utils2 import convert_line_endings, detect_mode, safe_str,\
 
    str2bool
 
from rhodecode.lib.auth import LoginRequired, HasRepoPermissionAnyDecorator
 
from rhodecode.lib.base import BaseRepoController, render
 
from rhodecode.lib.vcs.backends.base import EmptyChangeset
 
from rhodecode.lib.vcs.conf import settings
 
from rhodecode.lib.vcs.exceptions import RepositoryError, \
 
    ChangesetDoesNotExistError, EmptyRepositoryError, \
 
    ImproperArchiveTypeError, VCSError, NodeAlreadyExistsError,\
 
    NodeDoesNotExistError, ChangesetError, NodeError
 
from rhodecode.lib.vcs.nodes import FileNode
 

	
 
from rhodecode.model.repo import RepoModel
 
from rhodecode.model.scm import ScmModel
 
from rhodecode.model.db import Repository
 

	
 
from rhodecode.controllers.changeset import anchor_url, _ignorews_url,\
 
    _context_url, get_line_ctx, get_ignore_ws
 

	
 

	
 
log = logging.getLogger(__name__)
 

	
 

	
 
class FilesController(BaseRepoController):
 

	
 
    def __before__(self):
 
        super(FilesController, self).__before__()
 
        c.cut_off_limit = self.cut_off_limit
 

	
 
    def __get_cs_or_redirect(self, rev, repo_name, redirect_after=True):
 
        """
 
        Safe way to get changeset if error occur it redirects to tip with
 
        proper message
 

	
 
        :param rev: revision to fetch
 
        :param repo_name: repo name to redirect after
 
        """
 

	
 
@@ -384,116 +385,146 @@ class FilesController(BaseRepoController
 
                h.flash(_('Successfully committed to %s') % node_path,
 
                        category='success')
 
            except NodeAlreadyExistsError, e:
 
                h.flash(_(e), category='error')
 
            except Exception:
 
                log.error(traceback.format_exc())
 
                h.flash(_('Error occurred during commit'), category='error')
 
            return redirect(url('changeset_home',
 
                                repo_name=c.repo_name, revision='tip'))
 

	
 
        return render('files/files_add.html')
 

	
 
    @LoginRequired()
 
    @HasRepoPermissionAnyDecorator('repository.read', 'repository.write',
 
                                   'repository.admin')
 
    def archivefile(self, repo_name, fname):
 

	
 
        fileformat = None
 
        revision = None
 
        ext = None
 
        subrepos = request.GET.get('subrepos') == 'true'
 

	
 
        for a_type, ext_data in settings.ARCHIVE_SPECS.items():
 
            archive_spec = fname.split(ext_data[1])
 
            if len(archive_spec) == 2 and archive_spec[1] == '':
 
                fileformat = a_type or ext_data[1]
 
                revision = archive_spec[0]
 
                ext = ext_data[1]
 

	
 
        try:
 
            dbrepo = RepoModel().get_by_repo_name(repo_name)
 
            if dbrepo.enable_downloads is False:
 
                return _('downloads disabled')
 

	
 
            if c.rhodecode_repo.alias == 'hg':
 
                # patch and reset hooks section of UI config to not run any
 
                # hooks on fetching archives with subrepos
 
                for k, v in c.rhodecode_repo._repo.ui.configitems('hooks'):
 
                    c.rhodecode_repo._repo.ui.setconfig('hooks', k, None)
 

	
 
            cs = c.rhodecode_repo.get_changeset(revision)
 
            content_type = settings.ARCHIVE_SPECS[fileformat][0]
 
        except ChangesetDoesNotExistError:
 
            return _('Unknown revision %s') % revision
 
        except EmptyRepositoryError:
 
            return _('Empty repository')
 
        except (ImproperArchiveTypeError, KeyError):
 
            return _('Unknown archive type')
 
        # archive cache
 
        from rhodecode import CONFIG
 
        rev_name = cs.raw_id[:12]
 
        archive_name = '%s-%s%s' % (safe_str(repo_name.replace('/', '_')),
 
                                    safe_str(rev_name), ext)
 

	
 
        fd, archive = tempfile.mkstemp()
 
        t = open(archive, 'wb')
 
        cs.fill_archive(stream=t, kind=fileformat, subrepos=subrepos)
 
        t.close()
 
        use_cached_archive = False  # defines if we use cached version of archive
 
        archive_cache_enabled = CONFIG.get('archive_cache_dir')
 
        if not subrepos and archive_cache_enabled:
 
            #check if we it's ok to write
 
            if not os.path.isdir(CONFIG['archive_cache_dir']):
 
                os.makedirs(CONFIG['archive_cache_dir'])
 
            cached_archive_path = os.path.join(CONFIG['archive_cache_dir'], archive_name)
 
            if os.path.isfile(cached_archive_path):
 
                log.debug('Found cached archive in %s' % cached_archive_path)
 
                fd, archive = None, cached_archive_path
 
                use_cached_archive = True
 
            else:
 
                log.debug('Archive %s is not yet cached' % (archive_name))
 

	
 
        if not use_cached_archive:
 
            #generate new archive
 
            try:
 
                fd, archive = tempfile.mkstemp()
 
                t = open(archive, 'wb')
 
                log.debug('Creating new temp archive in %s' % archive)
 
                cs.fill_archive(stream=t, kind=fileformat, subrepos=subrepos)
 
                if archive_cache_enabled:
 
                    #if we generated the archive and use cache rename that
 
                    log.debug('Storing new archive in %s' % cached_archive_path)
 
                    shutil.move(archive, cached_archive_path)
 
                    archive = cached_archive_path
 
            finally:
 
                t.close()
 

	
 
        def get_chunked_archive(archive):
 
            stream = open(archive, 'rb')
 
            while True:
 
                data = stream.read(16 * 1024)
 
                if not data:
 
                    stream.close()
 
                    os.close(fd)
 
                    os.remove(archive)
 
                    if fd:  # fd means we used temporary file
 
                        os.close(fd)
 
                    if not archive_cache_enabled:
 
                        log.debug('Destroing temp archive %s' % archive)
 
                        os.remove(archive)
 
                    break
 
                yield data
 

	
 
        response.content_disposition = str('attachment; filename=%s-%s%s' \
 
                                           % (safe_str(repo_name),
 
                                              safe_str(revision), ext))
 
        response.content_disposition = str('attachment; filename=%s' % (archive_name))
 
        response.content_type = str(content_type)
 
        return get_chunked_archive(archive)
 

	
 
    @LoginRequired()
 
    @HasRepoPermissionAnyDecorator('repository.read', 'repository.write',
 
                                   'repository.admin')
 
    def diff(self, repo_name, f_path):
 
        ignore_whitespace = request.GET.get('ignorews') == '1'
 
        line_context = request.GET.get('context', 3)
 
        diff1 = request.GET.get('diff1', '')
 
        diff2 = request.GET.get('diff2', '')
 
        c.action = request.GET.get('diff')
 
        c.no_changes = diff1 == diff2
 
        c.f_path = f_path
 
        c.big_diff = False
 
        c.anchor_url = anchor_url
 
        c.ignorews_url = _ignorews_url
 
        c.context_url = _context_url
 
        c.changes = OrderedDict()
 
        c.changes[diff2] = []
 

	
 
        #special case if we want a show rev only, it's impl here
 
        #to reduce JS and callbacks
 

	
 
        if request.GET.get('show_rev'):
 
            if str2bool(request.GET.get('annotate', 'False')):
 
                _url = url('files_annotate_home', repo_name=c.repo_name,
 
                           revision=diff1, f_path=c.f_path)
 
            else:
 
                _url = url('files_home', repo_name=c.repo_name,
 
                           revision=diff1, f_path=c.f_path)
 

	
 
            return redirect(_url)
 
        try:
 
            if diff1 not in ['', None, 'None', '0' * 12, '0' * 40]:
 
                c.changeset_1 = c.rhodecode_repo.get_changeset(diff1)
 
                try:
 
                    node1 = c.changeset_1.get_node(f_path)
 
                    if node1.is_dir():
 
                        raise NodeError('%s path is a %s not a file' % (node1, type(node1)))
 
                except NodeDoesNotExistError:
 
                    c.changeset_1 = EmptyChangeset(cs=diff1,
 
                                                   revision=c.changeset_1.revision,
 
                                                   repo=c.rhodecode_repo)
 
                    node1 = FileNode(f_path, '', changeset=c.changeset_1)
 
            else:
 
                c.changeset_1 = EmptyChangeset(repo=c.rhodecode_repo)
 
                node1 = FileNode(f_path, '', changeset=c.changeset_1)
test.ini
Show inline comments
 
@@ -20,96 +20,98 @@ pdebug = false
 
#email_prefix = [RhodeCode]
 

	
 
#smtp_server = mail.server.com
 
#smtp_username = 
 
#smtp_password = 
 
#smtp_port = 
 
#smtp_use_tls = false
 
#smtp_use_ssl = true
 
# Specify available auth parameters here (e.g. LOGIN PLAIN CRAM-MD5, etc.)
 
#smtp_auth = 
 

	
 
[server:main]
 
## PASTE
 
##nr of threads to spawn
 
#threadpool_workers = 5
 

	
 
##max request before thread respawn
 
#threadpool_max_requests = 10
 

	
 
##option to use threads of process
 
#use_threadpool = true
 

	
 
#use = egg:Paste#http
 

	
 
#WAITRESS
 
threads = 5
 
#100GB
 
max_request_body_size = 107374182400
 
use = egg:waitress#main
 

	
 
host = 127.0.0.1
 
port = 5000
 

	
 
[filter:proxy-prefix]
 
# prefix middleware for rc
 
use = egg:PasteDeploy#prefix
 
prefix = /<your-prefix>
 

	
 
[app:main]
 
use = egg:rhodecode
 
#filter-with = proxy-prefix
 
full_stack = true
 
static_files = true
 
# Optional Languages
 
# en, fr, ja, pt_BR, zh_CN, zh_TW, pl
 
lang = en
 
cache_dir = /tmp/rc/data
 
index_dir = /tmp/rc/index
 
# set this path to use archive download cache
 
#archive_cache_dir = /tmp/rhodecode_tarballcache
 
app_instance_uuid = develop-test
 
cut_off_limit = 256000
 
vcs_full_cache = False
 
# force https in RhodeCode, fixes https redirects, assumes it's always https
 
force_https = false
 
# use Strict-Transport-Security headers
 
use_htsts = false
 
commit_parse_limit = 25
 
# number of items displayed in lightweight dashboard before paginating
 
dashboard_items = 100
 
use_gravatar = true
 

	
 
# path to git executable
 
git_path = git
 

	
 
## RSS feed options
 

	
 
rss_cut_off_limit = 256000
 
rss_items_per_page = 10
 
rss_include_diff = false
 

	
 

	
 
## alternative_gravatar_url allows you to use your own avatar server application
 
## the following parts of the URL will be replaced
 
## {email}        user email
 
## {md5email}     md5 hash of the user email (like at gravatar.com)
 
## {size}         size of the image that is expected from the server application
 
## {scheme}       http/https from RhodeCode server
 
## {netloc}       network location from RhodeCode server
 
#alternative_gravatar_url = http://myavatarserver.com/getbyemail/{email}/{size}
 
#alternative_gravatar_url = http://myavatarserver.com/getbymd5/{md5email}?s={size}
 

	
 
container_auth_enabled = false
 
proxypass_auth_enabled = false
 
## default encoding used to convert from and to unicode
 
## can be also a comma seperated list of encoding in case of mixed encodings
 
default_encoding = utf8
 

	
 
## overwrite schema of clone url
 
## available vars:
 
## scheme - http/https
 
## user - current user
 
## pass - password 
 
## netloc - network location
 
## path - usually repo_name
 

	
 
#clone_uri = {scheme}://{user}{pass}{netloc}{path}
 

	
0 comments (0 inline, 0 general)