Sindbad~EG File Manager

# -*- coding: utf-8 -*-

# Copyright © Cloud Linux GmbH & Cloud Linux Software, Inc 2010-2019 All Rights Reserved
#
# Licensed under CLOUD LINUX LICENSE AGREEMENT
# http://cloudlinux.com/docs/LICENSE.TXT

from __future__ import print_function
from __future__ import absolute_import
from __future__ import division
import os
import sys
import pwd
import subprocess
import shutil
import filecmp

from datetime import datetime
from clselect.clselectctl import get_directory
from clselect.utils import check_call, check_output, list_dirs, run_command
from .extensions import EXTENSION_PATTERN, ExtensionInfo
from .interpreters import Interpreter, interpreters
import simplejson as json
from simplejson import JSONDecodeError
from clselect.clselectexcept import ClSelectExcept
DEFAULT_PREFIX = 'virtualenv'
BACKUP_PREFIX = '.virtualenv.backup'
# only this binary it brought by alt-python-virtualenv package
VIRTUALENV_BIN = '/opt/alt/python37/bin/virtualenv'
VERSION_DELIMITER = '#'
WRAPPERS_PATH = '/usr/share/l.v.e-manager/utils'
PYTHON_WRAPPER = 'python_wrapper'
SET_ENV_VARS_SCRIPT = 'set_env_vars.py'


class Environment(object):

    def __init__(self, name, user=None, prefix=None):
        self.name = name
        if user:
            self.user = user
        else:
            self.user = pwd.getpwuid(os.getuid()).pw_name
        self.homepath = pwd.getpwnam(self.user).pw_dir
        self.pip_logfile = os.path.join(self.homepath, '.pip/pip.log')
        if prefix is None:
            self.prefix = DEFAULT_PREFIX
        else:
            self.prefix = prefix
        self.path = os.path.join(_abs_prefix(self.user, self.prefix), name)
        self.backup_path = os.path.join(
            _abs_prefix(self.user, BACKUP_PREFIX), self.name)
        self._requirements = None
        self._interpreter = None
        self._pip = None
        self.interpreter_name = 'python' + name
        # Create extenstion remap table
        self._extension_remap = {'MySQLdb': 'MySQL-python'}

    def __repr__(self):
        return ("%s.%s(name='%s', user='%s', prefix='%s')" % (
            self.__class__.__module__, self.__class__.__name__,
            self.name, self.user, self.prefix))

    def _demote(self):
        user_pwd = pwd.getpwnam(self.user)

        def func():
            os.setgid(user_pwd.pw_gid)
            os.setuid(user_pwd.pw_uid)
            os.environ['USER'] = self.user
            os.environ['HOME'] = user_pwd.pw_dir

        return func

    def as_dict(self, key=None):
        e = {
            'name': self.name,
            'interpreter': self.interpreter(),
            'extensions': self.extensions(),
            }
        if key:
            del e[key]
            return {getattr(self, key): e}
        return e

    def as_deepdict(self, key=None, with_extensions=True):
        e = {
            'name': self.name,
            'interpreter': self.interpreter().as_dict(),
        }
        if with_extensions:
            e.update({
                'extensions': self.extensions(),
            })
        if key:
            del e[key]
            return {getattr(self, key): e}
        return e

    def create(self, interpreter=None, version=None, wait=None):
        if not interpreter:
            interpreter = Interpreter(target_user=self.user)
        path = self.path
        if version:
            path = os.path.join(path, version)
        prompt = "({}:{})".format(
            get_directory(os.path.basename(self.prefix)), self.name,
        )
        args = [
            VIRTUALENV_BIN,
            '--prompt', prompt,
            '--python', interpreter.binary,
            path,
        ]
        kwargs = {"preexec_fn": self._demote(), "cwd": self.homepath, "wait": wait}
        try:
            check_call(*args, **kwargs)
        except ClSelectExcept.ExternalProgramFailed as err:
            err = str(err)
            err_trace = None
            # Change error text and add help if disk quota exceeded
            if "Disk quota exceeded" in err:
                err_text = "Disk quota exceeded.\n " \
                           "Contact system administrator to increase disk quota."
            elif "Traceback" in err:
                # Find second ":" character. First is "Traceback :"
                err_char = err.find(":", err.find(":")+1)
                # Find last row of trace
                err_trace_end = err[:err_char].rfind('\n')  # pylint: disable=indexing-exception
                if err_trace_end == -1 or err_char == -1:
                    err_text = err
                else:
                    # Trace row without error
                    err_trace = err[:err_trace_end]  # pylint: disable=indexing-exception
                    # Only error without first trace
                    err_text = err[err_trace_end+1:]  # pylint: disable=indexing-exception
            else:
                err_text = err
            raise ClSelectExcept.ExternalProgramFailed(
                message=err_text,
                details=err_trace,
            )

        self.configure_environment()

    def detect_python_binary(self, bin_path):
        files_to_check = [
            'python',
            self.interpreter_name.split('.')[0],
            self.interpreter_name,
        ]
        for file in files_to_check:
            path = os.path.join(bin_path, file)
            if not os.path.islink(path) or os.readlink(path).startswith('/opt/alt/python'):
                return path
        return None

    def configure_environment(self, auto_restore=False):
        """
        Configures environment:
        1. Rename binary to pythonX.Y_bin
        2. Makes symlink from python binary to python_wrapper
        """
        bin_path = os.path.join(self.path, 'bin')
        new_interpreter_path = os.path.join(bin_path, self.interpreter_name) + '_bin'
        interpreter_path = self.detect_python_binary(bin_path)
        if interpreter_path is None:
            return

        if os.path.exists(new_interpreter_path):
            os.remove(new_interpreter_path)

        os.rename(interpreter_path, new_interpreter_path)
        try:
            if not os.path.exists(interpreter_path):
                os.symlink(os.path.join(WRAPPERS_PATH, PYTHON_WRAPPER), interpreter_path)
        except (IOError, OSError):
            if auto_restore:
                os.rename(new_interpreter_path, interpreter_path)
            raise

        if not os.path.exists(os.path.join(bin_path, SET_ENV_VARS_SCRIPT)):
            os.symlink(os.path.join(WRAPPERS_PATH, SET_ENV_VARS_SCRIPT),
                       os.path.join(bin_path, SET_ENV_VARS_SCRIPT))

    def destroy(self, version=None):
        path = self.path
        if version:
            path = os.path.join(path, version)
        if os.path.exists(path):
            check_call('/bin/rm', '-r', '--interactive=never', path,
                       preexec_fn=self._demote())

    def _get_extension_name(self, extension_name):
        """
        Returns extensions name considering extension remap table
        :param extension_name: Input extension name
        :return: Result extension name
        """
        if extension_name in self._extension_remap:
            return self._extension_remap[extension_name]
        else:
            return extension_name

    def _recreate(self, version):
        """
        Recreate python virtual environment with requirements
        :return:
        """
        # if virtual environment does not exists, just create it
        # unfortunately, we don't have requirements
        env_path = os.path.join(self.path, version)
        if not os.path.exists(self.pip(version=version)):
            return
        print('Re-create python virtualenv:', env_path)
        # pip freeze, save last requirements into the file
        self._pip_freeze(version)
        # remember the requirements in the memory
        requirements_path = self.pip_requirements(version)
        requirements = []
        if os.path.exists(requirements_path):
            reqs_file = open(requirements_path, 'r')
            requirements = reqs_file.readlines()
            reqs_file.close()
        # destroy python virtual environment
        self.destroy(version=version)
        # create python virtual environment
        self.create(version=version, wait=True)
        # put remembered requirements into the file
        reqs_file = open(requirements_path, 'w')
        reqs_file.writelines(requirements)
        reqs_file.close()
        # pip install -r requirements, install requirements
        check_call(
            self.pip(version=version), 'install', '-r',
            self.pip_requirements(version))

    def recreate(self):
        for version in interpreters(key='version').keys():
            self._recreate(version)

    def exists(self):
        return os.path.exists(self.path)

    def interpreter(self):
        if not self._interpreter:
            self._interpreter = Interpreter(prefix=self.path, target_user=self.user)
        return self._interpreter

    def extension_install(self, extension_name):
        extension_name = self._get_extension_name(extension_name)
        locked_extensions = ExtensionInfo.get_locked_extensions(self.interpreter_name)
        t = extension_name.split(VERSION_DELIMITER)
        extension, version = t[0], t[1:] or ''
        command = ('/bin/bash', '-l', '-c', self.pip() + ' --log-file=' + self.pip_logfile +
                   ' install ' + extension_name)
        if version:
            version = version[0]
            command = ('/bin/bash', '-l', '-c', self.pip() + ' --log-file=' + self.pip_logfile +
                       ' install ' + extension + '==' + version)
        if ExtensionInfo.is_extensions_locked(locked_extensions, extension_name, version):
            raise ValueError("Extension '%s' install is prohibited. System extension" % extension_name)
        check_call(args=command, preexec_fn=self._demote(), cwd=self.homepath)
        self._pip_freeze()

    def extension_install_requirements(self, requirements_path):
        command = ('/bin/bash', '-l', '-c', self.pip() + ' --log-file=' + self.pip_logfile +
                   ' install -r {}'.format(requirements_path))
        check_call(args=command, preexec_fn=self._demote(), cwd=self.homepath)
        self._pip_freeze()

    def extension_update(self, extension):
        check_call(self.pip(), '--log-file='+self.pip_logfile,
                   'install', '--upgrade', extension,
                   preexec_fn=self._demote(), cwd=self.homepath)
        self._pip_freeze()

    def extension_uninstall(self, extension):
        locked_extensions = ExtensionInfo.get_locked_extensions(self.interpreter_name)
        t = extension.split(VERSION_DELIMITER)
        extension, version = t[0], t[1:] or ''
        if version:
            version = version[0]
        if ExtensionInfo.is_extensions_locked(locked_extensions, extension, version):
            raise ValueError("Extension '%s' removal is prohibited" % extension)
        p = subprocess.Popen(
            (self.pip(), '--log-file='+self.pip_logfile, 'uninstall', extension), preexec_fn=self._demote(),
            stdin=subprocess.PIPE, stderr=subprocess.PIPE,
            stdout=subprocess.PIPE, cwd=self.homepath, text=True)
        stdout, stderr = p.communicate('y')
        if p.returncode:
            raise Exception(stderr or stdout)
        self._pip_freeze()

    def extensions(self):
        result = {}
        locked_extensions = ExtensionInfo.get_locked_extensions(self.interpreter_name)
        try:
            output = check_output(self.pip(), 'list', '--log-file='+self.pip_logfile, '--format=json', preexec_fn=self._demote(), cwd=self.homepath)
            extensions = [(x['name'], x['version']) for x in json.loads(output)]
        except (JSONDecodeError, KeyError, ValueError, ClSelectExcept.FileProcessError, ClSelectExcept.ExternalProgramFailed):
            output = check_output(self.pip(), 'list', '--log-file='+self.pip_logfile, preexec_fn=self._demote(), cwd=self.homepath)
            extensions = EXTENSION_PATTERN.findall(output)
        docs = (ExtensionInfo().extension_doc(extension)
                for extension, _ in extensions)
        for (name, version), doc in zip(extensions, docs):
            if ExtensionInfo.is_extensions_locked(locked_extensions, name, version):
                version_diff = list(set([v.strip() for v in version.split(',')])
                                    - set(locked_extensions.get(name)))
                if version_diff and len(locked_extensions.get(name)) != 0:
                    result[name] = {'doc': doc, 'version': ', '.join(version_diff)}
            else:
                result[name] = {'doc': doc, 'version': version}

        return result

    def pip(self, version=None):
        if version is not None:
            return os.path.join(self.path, version, 'bin', 'pip')
        if not self._pip:
            self._pip = os.path.join(self.path, 'bin', 'pip')
        return self._pip

    def pip_requirements(self, version=None):
        if version is not None:
            return os.path.join(self.path, version, 'requirement.pip')
        return os.path.join(self.path, 'requirement.pip')

    def update_python_interpreter(self, backup=False, force=False, verbose=True, _alt_interpreters_dict=None):
        """
        copy binary python from /opt/alt/pythonXY/bin/pythonX.Y to virtualenvdir/bin/pythonX.Y
        :param backup: make backup old python interpreter
        :param force: force rewrite python interpreter without check
        :param verbose: print actions details to stdout
        :return: True - updating success; False - updating fail
        """
        update_result = False
        interpreter = self.interpreter()
        if _alt_interpreters_dict:
            main_interpreter = _alt_interpreters_dict[interpreter.version]
        else:
            main_interpreter = interpreters(key='version')[interpreter.version]  # path to original /opt/alt/pythonXY/bin/pythonX.Y

        updated_list = list()  # list updated interpreters

        if os.path.islink(interpreter.python_bin) and os.readlink(interpreter.python_bin).startswith('/opt/alt/python'):
            if verbose:
                print('Nothing to do, binary in your virtual environment is already symlink to global python!')
            return False

        # make backup and delete old python binary
        python_backup = interpreter.python_bin + '.orig_%s' % datetime.now().strftime("%Y-%m-%d_%H-%M")
        stat_ = os.stat(interpreter.python_bin)
        shutil.copy(interpreter.python_bin, python_backup)
        os.chown(python_backup, stat_.st_uid, stat_.st_gid)  # preserving owner

        try:
            for virtualenv_python_bin in interpreter.binary_list:
                if filecmp.cmp(main_interpreter.binary, interpreter.python_bin) and not force:
                    update_result = False
                    if verbose:
                        print("    not need updating; skip '%s'" % virtualenv_python_bin)
                    continue

                if verbose:
                    sys.stdout.write("    copy '%s' -> '%s'..." % (main_interpreter.binary, virtualenv_python_bin))
                run_command(cmd=('/bin/cp', '--force', main_interpreter.binary, virtualenv_python_bin))
                updated_list.append(virtualenv_python_bin)
                print("Done")
            update_result = True
        except (OSError, IOError) as e:
            # rollback binaries python if something is wrong
            print("Fail %s" % str(e))
            for updated_python in updated_list:
                shutil.copyfile(python_backup, updated_python)  # safe copy with preserve owner and mode
            os.unlink(python_backup)
        if not backup:  # delete backup if not need
            os.unlink(python_backup)
        return update_result

    def _pip_freeze(self, version=None):
        """
        Output installed packages in requirements format
        :return: None
        """
        if not os.path.exists(self.pip(version)):
            return
        command = (self.pip(version), 'freeze', '-l')
        f = open(self.pip_requirements(version), 'w')
        check_call(args=command, preexec_fn=self._demote(),
                   cwd=self.homepath, output=f)

    def pip_freeze(self):
        """
        Output installed packages in requirements format
        :return: None
        """
        for version in interpreters(key='version').keys():
            self._pip_freeze(version=version)


def _abs_prefix(user=None, prefix=None):
    if not prefix:
        prefix = DEFAULT_PREFIX
    if user:
        return os.path.join(pwd.getpwnam(user).pw_dir, prefix)
    else:
        return os.path.join(pwd.getpwuid(os.getuid()).pw_dir, prefix)


def environments(user=None, prefix=None):
    venv_path = _abs_prefix(user, prefix)
    try:
        env_list = list_dirs(venv_path)
    except OSError:
        return []
    envs = []
    for env_name in env_list:
        envs.append(Environment(env_name, user, prefix))
    return envs


def environments_dict(key, user=None, prefix=None):
    return dict(list(e.as_dict(key=key).items()) for e in environments(user, prefix))


def environments_deepdict(key, user=None, prefix=None):
    return dict(list(e.as_deepdict(key=key).items())
                for e in environments(user, prefix))