Changeset - 742d1b8ca263
[Not reviewed]
beta
0 2 0
Marcin Kuzminski - 13 years ago 2012-10-26 00:26:20
marcin@python-works.com
use str() on os.walk passing unicode can lead to UnicodeDecode errors when iterating
2 files changed with 2 insertions and 2 deletions:
0 comments (0 inline, 0 general)
rhodecode/lib/cleanup.py
Show inline comments
 
@@ -43,97 +43,97 @@ from rhodecode.lib.utils import BasePast
 

	
 
log = logging.getLogger(__name__)
 

	
 

	
 
class CleanupCommand(BasePasterCommand):
 

	
 
    max_args = 1
 
    min_args = 1
 

	
 
    usage = "CONFIG_FILE"
 
    summary = "Cleanup deleted repos"
 
    group_name = "RhodeCode"
 
    takes_config_file = -1
 
    parser = Command.standard_parser(verbose=True)
 

	
 
    def _parse_older_than(self, val):
 
        regex = re.compile(r'((?P<days>\d+?)d)?((?P<hours>\d+?)h)?((?P<minutes>\d+?)m)?((?P<seconds>\d+?)s)?')
 
        parts = regex.match(val)
 
        if not parts:
 
            return
 
        parts = parts.groupdict()
 
        time_params = {}
 
        for (name, param) in parts.iteritems():
 
            if param:
 
                time_params[name] = int(param)
 
        return datetime.timedelta(**time_params)
 

	
 
    def _extract_date(self, name):
 
        """
 
        Extract the date part from rm__<date> pattern of removed repos,
 
        and convert it to datetime object
 

	
 
        :param name:
 
        """
 
        date_part = name[4:19]  # 4:19 since we don't parse milisecods
 
        return datetime.datetime.strptime(date_part, '%Y%m%d_%H%M%S')
 

	
 
    def command(self):
 
        logging.config.fileConfig(self.path_to_ini_file)
 
        from pylons import config
 

	
 
        #get to remove repos !!
 
        add_cache(config)
 
        engine = engine_from_config(config, 'sqlalchemy.db1.')
 
        init_model(engine)
 

	
 
        repos_location = RhodeCodeUi.get_repos_location()
 
        to_remove = []
 
        for dn, dirs, f in os.walk(str(repos_location)):
 
        for dn, dirs, f in os.walk(safe_str(repos_location)):
 
            for loc in dirs:
 
                if REMOVED_REPO_PAT.match(loc):
 
                    to_remove.append([loc, self._extract_date(loc)])
 

	
 
        #filter older than (if present)!
 
        now = datetime.datetime.now()
 
        older_than = self.options.older_than
 
        if older_than:
 
            to_remove_filtered = []
 
            older_than_date = self._parse_older_than(older_than)
 
            for name, date_ in to_remove:
 
                repo_age = now - date_
 
                if repo_age > older_than_date:
 
                    to_remove_filtered.append([name, date_])
 

	
 
            to_remove = to_remove_filtered
 
            print >> sys.stdout, 'removing [%s] deleted repos older than %s[%s]' \
 
                % (len(to_remove), older_than, older_than_date)
 
        else:
 
            print >> sys.stdout, 'removing all [%s] deleted repos' \
 
                % len(to_remove)
 
        if self.options.dont_ask or not to_remove:
 
            # don't ask just remove !
 
            remove = True
 
        else:
 
            remove = ask_ok('are you sure to remove listed repos \n%s [y/n]?'
 
                            % ', \n'.join(['%s removed on %s'
 
                    % (safe_str(x[0]), safe_str(x[1])) for x in to_remove]))
 

	
 
        if remove:
 
            for name, date_ in to_remove:
 
                print >> sys.stdout, 'removing repository %s' % name
 
                shutil.rmtree(os.path.join(repos_location, name))
 
        else:
 
            print 'nothing done exiting...'
 
            sys.exit(0)
 

	
 
    def update_parser(self):
 
        self.parser.add_option('--older-than',
 
                          action='store',
 
                          dest='older_than',
 
                          help=(
 
                            "only remove repos that have been removed "
 
                            "at least given time ago "
 
                            "ex. --older-than=30d deletes repositores "
 
                            "removed more than 30days ago. Possible options "
 
                            "d[ays]/h[ours]/m[inutes]/s[seconds]. OPTIONAL"),
 
                          )
rhodecode/lib/hooks.py
Show inline comments
 
# -*- coding: utf-8 -*-
 
"""
 
    rhodecode.lib.hooks
 
    ~~~~~~~~~~~~~~~~~~~
 

	
 
    Hooks runned by rhodecode
 

	
 
    :created_on: Aug 6, 2010
 
    :author: marcink
 
    :copyright: (C) 2010-2012 Marcin Kuzminski <marcin@python-works.com>
 
    :license: GPLv3, see COPYING for more details.
 
"""
 
# This program is free software: you can redistribute it and/or modify
 
# it under the terms of the GNU General Public License as published by
 
# the Free Software Foundation, either version 3 of the License, or
 
# (at your option) any later version.
 
#
 
# This program is distributed in the hope that it will be useful,
 
# but WITHOUT ANY WARRANTY; without even the implied warranty of
 
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 
# GNU General Public License for more details.
 
#
 
# You should have received a copy of the GNU General Public License
 
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
import os
 
import sys
 
import time
 
import binascii
 
from inspect import isfunction
 

	
 
from mercurial.scmutil import revrange
 
from mercurial.node import nullrev
 

	
 
from rhodecode.lib import helpers as h
 
from rhodecode.lib.utils import action_logger
 
from rhodecode.lib.vcs.backends.base import EmptyChangeset
 
from rhodecode.lib.compat import json
 
from rhodecode.lib.exceptions import HTTPLockedRC
 
from rhodecode.lib.utils2 import safe_str, datetime_to_time
 
from rhodecode.model.db import Repository, User
 

	
 

	
 
def _get_scm_size(alias, root_path):
 

	
 
    if not alias.startswith('.'):
 
        alias += '.'
 

	
 
    size_scm, size_root = 0, 0
 
    for path, dirs, files in os.walk(root_path):
 
    for path, dirs, files in os.walk(safe_str(root_path)):
 
        if path.find(alias) != -1:
 
            for f in files:
 
                try:
 
                    size_scm += os.path.getsize(os.path.join(path, f))
 
                except OSError:
 
                    pass
 
        else:
 
            for f in files:
 
                try:
 
                    size_root += os.path.getsize(os.path.join(path, f))
 
                except OSError:
 
                    pass
 

	
 
    size_scm_f = h.format_byte_size(size_scm)
 
    size_root_f = h.format_byte_size(size_root)
 
    size_total_f = h.format_byte_size(size_root + size_scm)
 

	
 
    return size_scm_f, size_root_f, size_total_f
 

	
 

	
 
def repo_size(ui, repo, hooktype=None, **kwargs):
 
    """
 
    Presents size of repository after push
 

	
 
    :param ui:
 
    :param repo:
 
    :param hooktype:
 
    """
 

	
 
    size_hg_f, size_root_f, size_total_f = _get_scm_size('.hg', repo.root)
 

	
 
    last_cs = repo[len(repo) - 1]
 

	
 
    msg = ('Repository size .hg:%s repo:%s total:%s\n'
 
           'Last revision is now r%s:%s\n') % (
 
        size_hg_f, size_root_f, size_total_f, last_cs.rev(), last_cs.hex()[:12]
 
    )
 

	
 
    sys.stdout.write(msg)
 

	
 

	
 
def pre_push(ui, repo, **kwargs):
 
    # pre push function, currently used to ban pushing when
 
    # repository is locked
 
    try:
 
        rc_extras = json.loads(os.environ.get('RC_SCM_DATA', "{}"))
 
    except:
 
        rc_extras = {}
0 comments (0 inline, 0 general)