# -*- coding: utf-8 -*-

"""

    celery libs for RhodeCode

    :created_on: Nov 27, 2010

    :author: marcink

    :copyright: (C) 2009-2011 Marcin Kuzminski <marcin@python-works.com>    

    :license: GPLv3, see COPYING for more details.

"""

# This program is free software; you can redistribute it and/or

# modify it under the terms of the GNU General Public License

# as published by the Free Software Foundation; version 2

# of the License or (at your opinion) any later version of the license.

# 

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU General Public License for more details.

# 

# You should have received a copy of the GNU General Public License

# along with this program; if not, write to the Free Software

# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,

# MA  02110-1301, USA.

import os

import sys

import socket

import traceback

import logging

from hashlib import md5

from decorator import decorator

from vcs.utils.lazy import LazyProperty

from rhodecode.lib.pidlock import DaemonLock, LockHeld

from pylons import  config

log = logging.getLogger(__name__)

def str2bool(v):

    return v.lower() in ["yes", "true", "t", "1"] if v else None

try:

    CELERY_ON = str2bool(config['app_conf'].get('use_celery'))

except KeyError:

    CELERY_ON = False

class ResultWrapper(object):

    def __init__(self, task):

        self.task = task

    @LazyProperty

    def result(self):

        return self.task

def run_task(task, *args, **kwargs):

    if CELERY_ON:

        try:

            t = task.delay(*args, **kwargs)

            log.info('running task %s:%s', t.task_id, task)

            return t

        except socket.error, e:

            if  e.errno == 111:

                log.debug('Unable to connect to celeryd. Sync execution')

            else:

                log.error(traceback.format_exc())

        except KeyError, e:

                log.debug('Unable to connect to celeryd. Sync execution')

        except Exception, e:

            log.error(traceback.format_exc())

    log.debug('executing task %s in sync mode', task)

    return ResultWrapper(task(*args, **kwargs))

def locked_task(func):

    def __wrapper(func, *fargs, **fkwargs):

        params = list(fargs)

        params.extend(['%s-%s' % ar for ar in fkwargs.items()])

        lockkey = 'task_%s' % \

            md5(str(func.__name__) + '-' + \

                '-'.join(map(str, params))).hexdigest()

        log.info('running task with lockkey %s', lockkey)

        try:

            l = DaemonLock(lockkey)

            ret = func(*fargs, **fkwargs)

            l.release()

            return ret

        except LockHeld:

            log.info('LockHeld')

            return 'Task with key %s already running' % lockkey

    return decorator(__wrapper, func)

from celery.decorators import task

import os

import traceback

from time import mktime

from operator import itemgetter

from pylons import config

from pylons.i18n.translation import _

from rhodecode.lib.celerylib import run_task, locked_task, str2bool

from rhodecode.lib.helpers import person

from rhodecode.lib.smtp_mailer import SmtpMailer

from rhodecode.lib.utils import OrderedDict, add_cache

from rhodecode.model import init_model

from rhodecode.model import meta

from rhodecode.model.db import RhodeCodeUi

from vcs.backends import get_repo

from sqlalchemy import engine_from_config

add_cache(config)

try:

    import json

except ImportError:

    #python 2.5 compatibility

    import simplejson as json

__all__ = ['whoosh_index', 'get_commits_stats',

           'reset_user_password', 'send_email']

CELERY_ON = str2bool(config['app_conf'].get('use_celery'))

def get_session():

    if CELERY_ON:

        engine = engine_from_config(config, 'sqlalchemy.db1.')

        init_model(engine)

    sa = meta.Session()

    return sa

def get_repos_path():

    sa = get_session()

    q = sa.query(RhodeCodeUi).filter(RhodeCodeUi.ui_key == '/').one()

    return q.ui_value

@task

@locked_task

def whoosh_index(repo_location, full_index):

    log = whoosh_index.get_logger()

    from rhodecode.lib.indexers.daemon import WhooshIndexingDaemon

    index_location = config['index_dir']

    WhooshIndexingDaemon(index_location=index_location,

                         repo_location=repo_location, sa=get_session())\

                         .run(full_index=full_index)

@task

@locked_task

def get_commits_stats(repo_name, ts_min_y, ts_max_y):

    from rhodecode.model.db import Statistics, Repository

    log = get_commits_stats.get_logger()

    #for js data compatibilty

    author_key_cleaner = lambda k: person(k).replace('"', "")

    commits_by_day_author_aggregate = {}

    commits_by_day_aggregate = {}

    repos_path = get_repos_path()

    p = os.path.join(repos_path, repo_name)

    repo = get_repo(p)

    skip_date_limit = True

    parse_limit = 250 #limit for single task changeset parsing optimal for

    last_rev = 0

    last_cs = None

    timegetter = itemgetter('time')

    sa = get_session()

    dbrepo = sa.query(Repository)\

        .filter(Repository.repo_name == repo_name).scalar()

    cur_stats = sa.query(Statistics)\

        .filter(Statistics.repository == dbrepo).scalar()

    if cur_stats:

        last_rev = cur_stats.stat_on_revision

    if not repo.revisions:

        return True

    if last_rev == repo.revisions[-1] and len(repo.revisions) > 1:

        #pass silently without any work if we're not on first revision or 

        #current state of parsing revision(from db marker) is the last revision

        return True

    if cur_stats:

        commits_by_day_aggregate = OrderedDict(

import os

import sys

import traceback

from os.path import dirname as dn, join as jn

#to get the rhodecode import

sys.path.append(dn(dn(dn(os.path.realpath(__file__)))))

from string import strip

from rhodecode.model import init_model

from rhodecode.model.scm import ScmModel

from rhodecode.config.environment import load_environment

from rhodecode.lib.utils import BasePasterCommand, Command, add_cache

from shutil import rmtree

from webhelpers.html.builder import escape

from vcs.utils.lazy import LazyProperty

from sqlalchemy import engine_from_config

from whoosh.analysis import RegexTokenizer, LowercaseFilter, StopFilter

from whoosh.fields import TEXT, ID, STORED, Schema, FieldType

from whoosh.index import create_in, open_dir

from whoosh.formats import Characters

from whoosh.highlight import highlight, SimpleFragmenter, HtmlFormatter

#EXTENSIONS WE WANT TO INDEX CONTENT OFF

INDEX_EXTENSIONS = ['action', 'adp', 'ashx', 'asmx', 'aspx', 'asx', 'axd', 'c',

                    'cfg', 'cfm', 'cpp', 'cs', 'css', 'diff', 'do', 'el', 'erl',

                    'h', 'htm', 'html', 'ini', 'java', 'js', 'jsp', 'jspx', 'lisp',

                    'lua', 'm', 'mako', 'ml', 'pas', 'patch', 'php', 'php3',

                    'php4', 'phtml', 'pm', 'py', 'rb', 'rst', 's', 'sh', 'sql',

                    'tpl', 'txt', 'vim', 'wss', 'xhtml', 'xml', 'xsl', 'xslt',

                    'yaws']

#CUSTOM ANALYZER wordsplit + lowercase filter

ANALYZER = RegexTokenizer(expression=r"\w+") | LowercaseFilter()

#INDEX SCHEMA DEFINITION

SCHEMA = Schema(owner=TEXT(),

                repository=TEXT(stored=True),

                path=TEXT(stored=True),

                content=FieldType(format=Characters(ANALYZER),

                             scorable=True, stored=True),

                modtime=STORED(), extension=TEXT(stored=True))

IDX_NAME = 'HG_INDEX'

FORMATTER = HtmlFormatter('span', between='\n<span class="break">...</span>\n')

FRAGMENTER = SimpleFragmenter(200)

class MakeIndex(BasePasterCommand):

    max_args = 1

    min_args = 1

    usage = "CONFIG_FILE"

    summary = "Creates index for full text search given configuration file"

    group_name = "RhodeCode"

    takes_config_file = -1

    parser = Command.standard_parser(verbose=True)

    def command(self):

        from pylons import config

        add_cache(config)

        engine = engine_from_config(config, 'sqlalchemy.db1.')

        init_model(engine)

        index_location = config['index_dir']

        repo_location = self.options.repo_location

        repo_list = map(strip, self.options.repo_list.split(','))

        #======================================================================

        # WHOOSH DAEMON

        #======================================================================

        from rhodecode.lib.pidlock import LockHeld, DaemonLock

        from rhodecode.lib.indexers.daemon import WhooshIndexingDaemon

        try:

            l = DaemonLock()

            WhooshIndexingDaemon(index_location=index_location,

                                 repo_location=repo_location,

                                 repo_list=repo_list)\

                .run(full_index=self.options.full_index)

            l.release()

        except LockHeld:

            sys.exit(1)

    def update_parser(self):

        self.parser.add_option('--repo-location',

                          action='store',

                          dest='repo_location',

# This program is free software; you can redistribute it and/or

# modify it under the terms of the GNU General Public License

# as published by the Free Software Foundation; version 2

# of the License or (at your opinion) any later version of the license.

# 

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU General Public License for more details.

# 

# You should have received a copy of the GNU General Public License

# along with this program; if not, write to the Free Software

# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,

# MA  02110-1301, USA.

class HttpsFixup(object):

    def __init__(self, app):

        self.application = app

    def __call__(self, environ, start_response):

        self.__fixup(environ)

        return self.application(environ, start_response)

    def __fixup(self, environ):

        """Function to fixup the environ as needed. In order to use this

        middleware you should set this header inside your 

        proxy ie. nginx, apache etc.

        """

        proto = environ.get('HTTP_X_URL_SCHEME')

        if proto == 'https':

            environ['wsgi.url_scheme'] = proto

        else:

            environ['wsgi.url_scheme'] = 'http'

        return None

# This program is free software; you can redistribute it and/or

# modify it under the terms of the GNU General Public License

# as published by the Free Software Foundation; version 2

# of the License or (at your opinion) any later version of the license.

# 

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU General Public License for more details.

# 

# You should have received a copy of the GNU General Public License

# along with this program; if not, write to the Free Software

# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,

# MA  02110-1301, USA.

from dulwich import server as dulserver

class SimpleGitUploadPackHandler(dulserver.UploadPackHandler):

    def handle(self):

        write = lambda x: self.proto.write_sideband(1, x)

        graph_walker = dulserver.ProtocolGraphWalker(self, self.repo.object_store,

            self.repo.get_peeled)

        objects_iter = self.repo.fetch_objects(

          graph_walker.determine_wants, graph_walker, self.progress,

          get_tagged=self.get_tagged)

        # Do they want any objects?

        if len(objects_iter) == 0:

            return

        self.progress("counting objects: %d, done.\n" % len(objects_iter))

        dulserver.write_pack_data(dulserver.ProtocolFile(None, write), objects_iter,

                        len(objects_iter))

        messages = []

        messages.append('thank you for using rhodecode')

        for msg in messages:

            self.progress(msg + "\n")

        # we are done

        self.proto.write("0000")

dulserver.DEFAULT_HANDLERS = {

  'git-upload-pack': SimpleGitUploadPackHandler,

  'git-receive-pack': dulserver.ReceivePackHandler,

}

from dulwich.repo import Repo

from dulwich.web import HTTPGitApplication

from paste.auth.basic import AuthBasicAuthenticator

from paste.httpheaders import REMOTE_USER, AUTH_TYPE

from rhodecode.lib.auth import authfunc, HasPermissionAnyMiddleware

from rhodecode.lib.utils import invalidate_cache, check_repo_fast

from rhodecode.model.user import UserModel

from webob.exc import HTTPNotFound, HTTPForbidden, HTTPInternalServerError

log = logging.getLogger(__name__)

def is_git(environ):

    then have git client version given.

    :param environ:

    """

    http_user_agent = environ.get('HTTP_USER_AGENT')

    if http_user_agent and http_user_agent.startswith('git'):

        return True

    return False

class SimpleGit(object):

    def __init__(self, application, config):

        self.application = application

        self.config = config

        #authenticate this git request using 

        self.authenticate = AuthBasicAuthenticator('', authfunc)

        self.ipaddr = '0.0.0.0'

        self.repository = None

        self.username = None

        self.action = None

    def __call__(self, environ, start_response):

        if not is_git(environ):

            return self.application(environ, start_response)

        proxy_key = 'HTTP_X_REAL_IP'

        def_key = 'REMOTE_ADDR'

        self.ipaddr = environ.get(proxy_key, environ.get(def_key, '0.0.0.0'))

        # skip passing error to error controller

        environ['pylons.status_code_redirect'] = True

        #===================================================================

        # AUTHENTICATE THIS GIT REQUEST

        #===================================================================

        username = REMOTE_USER(environ)

        if not username:

            self.authenticate.realm = self.config['rhodecode_realm']

            result = self.authenticate(environ)

            if isinstance(result, str):

                AUTH_TYPE.update(environ, 'basic')

                REMOTE_USER.update(environ, result)

            else:

                return result.wsgi_application(environ, start_response)

        #=======================================================================

        # GET REPOSITORY

        #=======================================================================

        try:

            repo_name = '/'.join(environ['PATH_INFO'].split('/')[1:])

            if repo_name.endswith('/'):

                repo_name = repo_name.rstrip('/')

            self.repository = repo_name

        except:

            log.error(traceback.format_exc())

            return HTTPInternalServerError()(environ, start_response)

        #===================================================================

        # CHECK PERMISSIONS FOR THIS REQUEST

        #===================================================================

        self.action = self.__get_action(environ)

        if self.action:

            username = self.__get_environ_user(environ)

            try:

                user = self.__get_user(username)

                self.username = user.username

            except:

                log.error(traceback.format_exc())

                return HTTPInternalServerError()(environ, start_response)

            #check permissions for this repository

            if self.action == 'push':

                if not HasPermissionAnyMiddleware('repository.write',

                                                  'repository.admin')\

                                                    (user, repo_name):

                    return HTTPForbidden()(environ, start_response)

            else:

                #any other action need at least read permission

                if not HasPermissionAnyMiddleware('repository.read',

                                                  'repository.write',

                                                  'repository.admin')\

                                                    (user, repo_name):

                    return HTTPForbidden()(environ, start_response)

        self.extras = {'ip':self.ipaddr,

                       'username':self.username,

                       'action':self.action,

                       'repository':self.repository}

        #===================================================================

        # GIT REQUEST HANDLING

        #===================================================================

        self.basepath = self.config['base_path']

        self.repo_path = os.path.join(self.basepath, self.repo_name)

        #quick check if that dir exists...

        if check_repo_fast(self.repo_name, self.basepath):

            return HTTPNotFound()(environ, start_response)

        try:

            app = self.__make_app()

        except:

            log.error(traceback.format_exc())

            return HTTPInternalServerError()(environ, start_response)

        #invalidate cache on push

        if self.action == 'push':

            self.__invalidate_cache(self.repo_name)

            messages = []

            messages.append('thank you for using rhodecode')

            return app(environ, start_response)

        else:

            return app(environ, start_response)

    def __make_app(self):

        backend = dulserver.DictBackend({'/' + self.repo_name: Repo(self.repo_path)})

        gitserve = HTTPGitApplication(backend)

        return gitserve

    def __get_environ_user(self, environ):

        return environ.get('REMOTE_USER')

    def __get_user(self, username):

        return UserModel().get_by_username(username, cache=True)

    def __get_action(self, environ):

        :param environ:

        """

        service = environ['QUERY_STRING'].split('=')

        if len(service) > 1:

            service_cmd = service[1]

            mapping = {'git-receive-pack': 'push',

                       'git-upload-pack': 'pull',

                       }

            return mapping.get(service_cmd, service_cmd if service_cmd else 'other')

        else:

            return 'other'

    def __invalidate_cache(self, repo_name):

        """we know that some change was made to repositories and we should

        invalidate the cache to see the changes right away but only for

        push requests"""

        invalidate_cache('get_repo_cached_%s' % repo_name)

# This program is free software; you can redistribute it and/or

# modify it under the terms of the GNU General Public License

# as published by the Free Software Foundation; version 2

# of the License or (at your opinion) any later version of the license.

# 

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU General Public License for more details.

# 

# You should have received a copy of the GNU General Public License

# along with this program; if not, write to the Free Software

# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,

# MA  02110-1301, USA.

from mercurial.error import RepoError

from mercurial.hgweb import hgweb

from mercurial.hgweb.request import wsgiapplication

from paste.auth.basic import AuthBasicAuthenticator

from paste.httpheaders import REMOTE_USER, AUTH_TYPE

from rhodecode.lib.auth import authfunc, HasPermissionAnyMiddleware

from rhodecode.lib.utils import make_ui, invalidate_cache, \

    check_repo_fast, ui_sections

from rhodecode.model.user import UserModel

from webob.exc import HTTPNotFound, HTTPForbidden, HTTPInternalServerError

import logging

import os

import traceback

log = logging.getLogger(__name__)

def is_mercurial(environ):

    ``HTTP_ACCEPT`` of such request would start with ``application/mercurial``.

    """

    http_accept = environ.get('HTTP_ACCEPT')

    if http_accept and http_accept.startswith('application/mercurial'):

        return True

    return False

class SimpleHg(object):

    def __init__(self, application, config):

        self.application = application

        self.config = config

        #authenticate this mercurial request using authfunc

        self.authenticate = AuthBasicAuthenticator('', authfunc)

        self.ipaddr = '0.0.0.0'

        self.repository = None

        self.username = None

        self.action = None

    def __call__(self, environ, start_response):

        if not is_mercurial(environ):

            return self.application(environ, start_response)

        proxy_key = 'HTTP_X_REAL_IP'

        def_key = 'REMOTE_ADDR'

        self.ipaddr = environ.get(proxy_key, environ.get(def_key, '0.0.0.0'))

        # skip passing error to error controller

        environ['pylons.status_code_redirect'] = True

        #===================================================================

        # AUTHENTICATE THIS MERCURIAL REQUEST

        #===================================================================

        username = REMOTE_USER(environ)

        if not username:

            self.authenticate.realm = self.config['rhodecode_realm']

            result = self.authenticate(environ)

            if isinstance(result, str):

                AUTH_TYPE.update(environ, 'basic')

                REMOTE_USER.update(environ, result)

            else:

                return result.wsgi_application(environ, start_response)

        #=======================================================================

        # GET REPOSITORY

        #=======================================================================

        try:

            repo_name = '/'.join(environ['PATH_INFO'].split('/')[1:])

            if repo_name.endswith('/'):

                repo_name = repo_name.rstrip('/')

            self.repository = repo_name

        except:

            log.error(traceback.format_exc())

            return HTTPInternalServerError()(environ, start_response)

        #===================================================================

        # CHECK PERMISSIONS FOR THIS REQUEST

        #===================================================================

        self.action = self.__get_action(environ)

        if self.action:

            username = self.__get_environ_user(environ)

            try:

                user = self.__get_user(username)

                self.username = user.username

            except:

                log.error(traceback.format_exc())

                return HTTPInternalServerError()(environ, start_response)

            #check permissions for this repository

            if self.action == 'push':

                if not HasPermissionAnyMiddleware('repository.write',

                                                  'repository.admin')\

                                                    (user, repo_name):

                    return HTTPForbidden()(environ, start_response)

            else:

                #any other action need at least read permission

                if not HasPermissionAnyMiddleware('repository.read',

                                                  'repository.write',

                                                  'repository.admin')\

                                                    (user, repo_name):

                    return HTTPForbidden()(environ, start_response)

        self.extras = {'ip':self.ipaddr,

                       'username':self.username,

                       'action':self.action,

                       'repository':self.repository}

        #===================================================================

        # MERCURIAL REQUEST HANDLING

        #===================================================================

        environ['PATH_INFO'] = '/'#since we wrap into hgweb, reset the path

        self.baseui = make_ui('db')

        self.basepath = self.config['base_path']

        self.repo_path = os.path.join(self.basepath, repo_name)

        #quick check if that dir exists...

        if check_repo_fast(repo_name, self.basepath):

            return HTTPNotFound()(environ, start_response)

        try:

            app = wsgiapplication(self.__make_app)

        except RepoError, e:

            if str(e).find('not found') != -1:

                return HTTPNotFound()(environ, start_response)

        except Exception:

            log.error(traceback.format_exc())

            return HTTPInternalServerError()(environ, start_response)

        #invalidate cache on push

        if self.action == 'push':

            self.__invalidate_cache(repo_name)

        return app(environ, start_response)

    def __make_app(self):

        hgserve = hgweb(str(self.repo_path), baseui=self.baseui)

        return  self.__load_web_settings(hgserve, self.extras)

    def __get_environ_user(self, environ):

        return environ.get('REMOTE_USER')

    def __get_user(self, username):

        return UserModel().get_by_username(username, cache=True)

    def __get_action(self, environ):

        This should always return a valid command string

        :param environ:

        """

        mapping = {'changegroup': 'pull',

                   'changegroupsubset': 'pull',

                   'stream_out': 'pull',

                   'listkeys': 'pull',

                   'unbundle': 'push',

                   'pushkey': 'push', }

        for qry in environ['QUERY_STRING'].split('&'):

            if qry.startswith('cmd'):

                cmd = qry.split('=')[-1]

                if mapping.has_key(cmd):

                    return mapping[cmd]

                else:

                    return cmd

    def __invalidate_cache(self, repo_name):