Sindbad~EG File Manager

# coding=utf-8
#
# Copyright © Cloud Linux GmbH & Cloud Linux Software, Inc 2010-2019 All Rights Reserved
#
# Licensed under CLOUD LINUX LICENSE AGREEMENT
# http://cloudlinux.com/docs/LICENSE.TXT

from __future__ import absolute_import
from builtins import range, map
import csv
import os
import pwd
import uuid
import time
import fcntl
import errno
from _socket import gethostname, error as SocketError
from urllib.parse import urlparse
from xml.dom import minidom, DOMException
from xml.parsers.expat import ExpatError

from clcommon import FormattedException
from clcommon import cpapi
from clcommon.clproc import LIMIT_LVP_ID
from clcommon.cpapi.cpapiexceptions import CPAPIExternalProgramFailed, NotSupported, ParsingError
from clcommon.lib import MySQLGovernor, MySQLGovException
from sqlalchemy import engine as sqlalchemy_engine
from clcommon.clcaptain import mkdir as mkdir_p
from clcommon.clfunc import uid_max
from lvestats.lib.config import get_max_lve_id, USER_NOTIFICATIONS_OFF_MARKER

MAX_LVE_ID = None
LVE_FILE = '/proc/lve/list'
GOVERNOR_CONFIG = '/var/run/mysql-governor-config.xml'  # can read all in system; present in governor-mysql >= 1.1-14
GOVERNOR_CONFIG_OLD = '/etc/container/mysql-governor.xml'  # can read root only


def get_current_max_lve_id():
    """
    Get current maximum of LVE ID
    """
    global MAX_LVE_ID  # pylint: disable=global-statement
    if MAX_LVE_ID is None:
        MAX_LVE_ID = get_max_lve_id()
    return MAX_LVE_ID


def gcd(a, b):
    # returns greater common divisor
    while b:
        a, b = b, a % b
    return a


def merge_dicts(*dicts):
    """
    >>> merge_dicts({1: 2}, {3: 4})
    {1: 2, 3: 4}
    >>> merge_dicts({1: '1', 2: '2'}, {2: 'two', 3: 'three'})
    {1: '1', 2: 'two', 3: 'three'}
    """""
    dic_merged = dict()
    for dic_ in dicts:
        dic_merged.update(dic_)
    return dic_merged


def get_chunks(_list, chunk_length=500):
    """
    :type chunk_length: int
    :type _list: list
    :rtype: Generator[list]
    """
    for i in range(0, len(_list), chunk_length):
        yield _list[i:i + chunk_length]


class LVEVersionError(Exception):
    def __init__(self, message=''):
        Exception.__init__(self, "Can't detect LVE version; {}".format(message))


def get_lve_version(_lve_file=LVE_FILE):
    """
    :param str _lve_file:
    :return int:
    """
    try:
        with open(_lve_file, 'r') as lve_file_stream:
            lve_ver_str = lve_file_stream.read(5)  # read only first 5 symbols
            try:
                return int(lve_ver_str.split(':')[0])
            except (TypeError, IndexError, ValueError):
                raise LVEVersionError("error parsing line '{}' from file {}".format(lve_ver_str, _lve_file))
    except (OSError, IOError) as e:
        raise LVEVersionError(e)


def get_governor_mode(governor=None):
    """
    Parse governor config file and return governor mode
    :type governor: MySQLGovernor
    :rtype: str
    :return:
        if file exists and correct: "on", "single", "off", "abusers" or "all"
        if governor is not installed or error occurs: "none"
    """
    governor = governor or MySQLGovernor()
    try:
        governor_mode = governor.get_governor_mode()
    except MySQLGovException:
        governor_mode = None
    return governor_mode or "none"


def get_governor_config_path():
    # check governor installed, file GOVERNOR_CONFIG can be present after mysql-governor uninstalled
    """
    :rtype: str|None
    """
    if not os.path.exists('/usr/sbin/db_governor'):  # this need for compatibility with mysql-governor 1.1-15
        return None
    if os.path.exists(GOVERNOR_CONFIG):
        governor_config_path = GOVERNOR_CONFIG
    else:
        governor_config_path = GOVERNOR_CONFIG_OLD  # config for backward compatible with governor-mysql < 1.1-14
    return governor_config_path


# TODO: replace with MysqlGovernor().get_governor_status_by_username
def get_governor_ignore_for_user(username):
    """
    :rtype: bool|None
    :type username: str
    """
    governor_config_path = get_governor_config_path()
    if governor_config_path is None:
        return None
    try:
        governor_config = minidom.parse(governor_config_path)
        for user in governor_config.getElementsByTagName("user"):
            try:
                if user.attributes["name"].value == username:
                    # mode = ”restrict|norestrict|ignore”
                    return user.attributes["mode"].value.lower() == "ignore"
            except KeyError:
                continue
    except (IOError, ExpatError, DOMException):
        return None
    return False


class DomainException(FormattedException):
    pass


def get_governor_status():
    """
    :rtype: (str, MySQLGovException|None)
    """
    mysql_governor = MySQLGovernor()
    return mysql_governor.get_governor_status()


def get_reseller_domains(reseller_name):
    """
    Get dict[user, domain], empty if cannot obtain.
    :param reseller_name: reseller's name
    :rtype: dict[str, str|None]
    """
    try:
        return cpapi.reseller_domains(reseller_name)
    except (CPAPIExternalProgramFailed, NotSupported, AttributeError):
        return dict()


def get_aliases(username, domain, raise_exception=True):
    """
    Get list of aliases for specified user's domain, or empty list.
    :type username: str
    :type domain: str
    :type raise_exception: bool
    :raises DomainException: if cannot obtain domains
    :rtype: list[str]|None
    """
    try:
        user_aliases = cpapi.useraliases(username, domain)
    except (KeyError, IOError, TypeError, cpapi.cpapiexceptions.NotSupported,
            cpapi.cpapiexceptions.NoPanelUser, ParsingError) as e:
        if raise_exception:
            raise DomainException(str(e))
        return []
    else:
        return user_aliases


def get_domains(username, raise_exception=True):
    """
    Get list of domains for specified user, or None.
    :type username: str
    :type raise_exception: bool
    :raises DomainException: if cannot obtain domains
    :rtype: list[str]|None
    """
    try:
        user_domains = cpapi.userdomains(username)
    except (KeyError, IOError, TypeError, cpapi.cpapiexceptions.NotSupported,
            cpapi.cpapiexceptions.NoPanelUser, ParsingError) as e:
        if raise_exception:
            raise DomainException(str(e))
        return []
    else:
        return [domain[0] for domain in user_domains]


def get_domain(username):
    """
    :param username: user name string
    :return:
        if not supported in CP, or domain name is not set: None
        if we can find it: domain name string
    :rtype: str|None
    :type username: str
    """
    user_domains = get_domains(username)
    if len(user_domains) == 0:
        return None
    return user_domains[0]


def init_database(db_engine, db_name):
    """
    Create database
    :param db_engine:
    :param db_name:
    :return:
    """
    conn = db_engine.connect()
    conn.execute('COMMIT')  # we can't run CREATE DATABASE in one transaction
    conn.execute("CREATE DATABASE %s" % db_name)
    conn.close()


def init_database_rnd(db_engine, prefix='tmp_unittest_'):
    """
    Create database with random prefix in name
    Using for unittests
    :param db_engine:
    :param str prefix:
    :return str:
    """
    db_name = prefix + str(uuid.uuid4())[0:8]
    init_database(db_engine, db_name)
    return db_name


def drop_database(uri, force=True):
    """
    Drop database (support MySQL and PostgresSQL)
    :param uri:
    :param force: kill all connection and locked session before drop database
    :return:
    """
    uri_ = urlparse(uri)
    db_name = uri_.path[1:]  # cut '/' on strings start
    if db_name:
        if uri_.scheme == 'sqlite':
            os.unlink(db_name)
            return True

        db_engine = sqlalchemy_engine.create_engine(uri_.scheme + '://' + uri_.netloc)
        conn = db_engine.connect()

        # force kill all session for dropped database
        if force and 'postgresql' in db_engine.name:
            # http://dba.stackexchange.com/questions/11893/force-drop-db-while-others-may-be-connected
            conn.execute("UPDATE pg_database SET datallowconn = 'false' where datname = '%s'" % db_name)
            pg_ver = conn.execute("SHOW server_version").fetchall()[0][0]
            # pg_ver here: CL - '9.6.22', Ubuntu - '12.9 (Ubuntu 12.9-0ubuntu0.20.04.1)'
            pg_ver = pg_ver.strip().split(' ')[0]
            if list(map(int, pg_ver.split('.'))) < [9, 2]:  # compare versions
                conn.execute("SELECT pg_terminate_backend(procpid) FROM pg_stat_activity WHERE datname = '%s'" % db_name)
            else:
                conn.execute("SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '%s'" % db_name)
        elif force and 'mysql' in db_engine.name:
            cursor = conn.execute("SHOW PROCESSLIST")
            for fetched in cursor.fetchall():
                if fetched[3] == db_name:
                    conn.execute("KILL %s" % fetched[0])

        conn.execute("COMMIT")
        conn.execute("DROP DATABASE %s" % db_name)
        conn.close()
        return True


def _release_line():
    path = '/etc/redhat-release'
    if not os.path.exists(path):
        return ''
    with open(path) as release_file:
        return release_file.readline()


def cl_ver(release_line=None):
    """
    Return CloudLinux version as list [major, minor]
    or [] if can't obtain version
    :param str release_line:
    :return list: splited version
    >>> cl_ver('CloudLinux Server release 5.11 (Vladislav Volkov)')
    [5, 11]
    >>> cl_ver('CloudLinux Server release 6.8 (Oleg Makarov)')
    [6, 8]
    >>> cl_ver('CloudLinux release 7.2 (Valeri Kubasov)')
    [7, 2]
    >>> cl_ver('Incorrect release line')
    []
    """
    if release_line is None:
        release_line = _release_line()
    release_line_splited = release_line.split()
    if 'release' in release_line_splited:
        ver_indx = release_line_splited.index('release') + 1
    else:
        return list()
    ver_str = release_line_splited[ver_indx]
    ver = list()
    for s in ver_str.split('.'):
        try:
            ver.append(int(s))
        except ValueError:
            pass
    return ver


def _reseller_users(reseller=None):
    cplogin_reseller_ = cpapi.cpinfo(keyls=('cplogin', 'reseller'))
    return [cplogin for cplogin, reseller_ in cplogin_reseller_ if reseller == reseller_]


def get_users_for_reseller(reseller=None):
    reseller_users = getattr(cpapi, 'reseller_users', _reseller_users)
    try:
        return reseller_users(reseller)
    except cpapi.cpapiexceptions.CPAPIException:
        return list()


def touch(fname, times=None):
    """
    The equivalent console touch command
    :param str fname: path to file
    :param None| tuple times: (atime, mtime) if None atime and mtime are default
    """
    with open(fname, 'a'):
        os.utime(fname, times)
    os.chmod(fname, 0o644)


def atomic_write_csv(filename, data):
    """
    :type data: list
    :type filename: str
    """
    def csv_write(csv_file):
        """:type csv_file: file"""
        writer = csv.writer(csv_file)
        writer.writerows(data)
    _atomic_write(filename, csv_write)


def atomic_write_str(filename, data):
    """
    :type data: str
    :type filename: str
    """
    def str_write(str_file):
        """:type str_file: file"""
        str_file.write(data)
    _atomic_write(filename, str_write)


def _atomic_write(filename, write_function):
    """
    :type write_function: (file) -> None
    :type filename: str
    """
    temporary_filename = filename + ".tmp"
    f = open(temporary_filename, 'w')
    write_function(f)
    f.flush()
    os.fsync(f.fileno())
    f.close()

    os.rename(temporary_filename, filename)


class reboot_lock(object):
    def __init__(self, fname=None, timeout=None):
        self.timeout = timeout
        if fname is None:
            self.fname = reboot_lock._build_lock_filename()
        else:
            self.fname = fname
        self.lock = os.open(self.fname, os.O_WRONLY | os.O_CREAT, 0o600)

    @staticmethod
    def _build_lock_filename():
        lock_file_dir = '/var/lve'
        # Create /var/lve directory if it is absent
        if not os.path.exists(lock_file_dir):
            mkdir_p(lock_file_dir)
        return os.path.join(lock_file_dir, 'lvestats.lock')

    def __enter__(self):
        start_time = time.time()
        while True:
            try:
                fcntl.flock(self.lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
                return
            except IOError as e:
                # raise on unrelated IOErrors
                if e.errno != errno.EAGAIN:
                    raise
                else:
                    if self.timeout is not None and time.time() - start_time > self.timeout:
                        return
                    time.sleep(0.1)

    def __exit__(self, type, value, traceback):
        fcntl.flock(self.lock, fcntl.LOCK_UN)
        os.close(self.lock)
        try:
            os.unlink(self.fname)
        except:
            pass


def deserialize_lve_id(serialized_value):
    """
    Extract container_id and bool(is_reseller) from serialized value;
    :type serialized_value: int 
    :rtype: tuple[int, bool]
    """

    if serialized_value > get_current_max_lve_id() and serialized_value != LIMIT_LVP_ID:
        return serialized_value - get_current_max_lve_id(), True
    return serialized_value, False


def serialize_lve_id(lve_id, lvp_id):
    """
    Get serialized int by lvp_id and lve_id
    :type lvp_id: int 
    :type lve_id: int 
    :rtype: int
    """

    if lve_id == LIMIT_LVP_ID and lvp_id != 0:
        return get_current_max_lve_id() + lvp_id
    return lve_id


def get_hostname():
    """
    Get hostname using socket.gethostname;
    Return 'N/A' if some error happens;
    :return: 
    """
    try:
        hostname = gethostname()
    except SocketError:
        hostname = 'N/A'
    return hostname


def skip_user_by_maxuid(user_id):
    """
    Returns is user should be skipped due to set max uid
    :param user_id: user id
    """
    return user_id > uid_max()


def get_all_user_domains(username, include_aliases=True):
    """
    Returns ALL user domains: main/sub/aliases
    """
    domains = get_domains(username, raise_exception=False)
    aliases = []
    if include_aliases:
        for domain in domains:
            alias = get_aliases(username, domain, raise_exception=False)
            if alias:
                aliases += alias
    return list(set(domains + aliases))


def normalize_domain(domain):
    """
    Cuts extra prefix www from domain
    """
    if domain.startswith('www.'):
        domain = domain[4:]
    return domain


def user_should_be_notified(user: str) -> bool:
    """
    Checks if notifications are turned off by user: marker file exists
    """
    return not os.path.exists(os.path.join(pwd.getpwnam(user).pw_dir, '.lvestats',
                                           USER_NOTIFICATIONS_OFF_MARKER))