Sindbad~EG File Manager

# coding=utf-8
#
# Copyright © Cloud Linux GmbH & Cloud Linux Software, Inc 2010-2019 All Rights Reserved
#
# Licensed under CLOUD LINUX LICENSE AGREEMENT
# http://cloudlinux.com/docs/LICENSE.TXT
from __future__ import print_function
from __future__ import absolute_import
from __future__ import division
from builtins import range
import sys
import logging
import time

from lvestats.core.plugin import LveStatsPlugin, LveStatsPluginTerminated
from sqlalchemy import insert
from lvestats.orm.history import history, history_x60
from sqlalchemy.sql.expression import func
from sqlalchemy.sql import select
from sqlalchemy.exc import OperationalError

from lvestats.lib.commons.func import get_chunks, reboot_lock
from lvestats.lib.lveinfolib import HistoryShow, FIELD_TO_TABLE_COLUMN
from lvestats.lib.commons.progress import ProgressCallback
from lvestats.lib import dbengine


AGGREGATE_PERIOD = 60*60  # one hour


class DBSaverX60(LveStatsPlugin):
    FIELDS = [
        'ID',
        'aCPU',   'lCPU',   'CPUf',
        'aEP',    'lEP',    'EPf',
        'aVMem',  'lVMem',  'VMemF',
        'aPMem',  'lPMem',  'PMemF',
        'aNproc', 'lNproc', 'NprocF',
        'aIO',    'lIO',    'IOf',
        'IOPSf',  'lIOPS',  'aIOPS'
    ]

    def __init__(self):
        self.log = logging.getLogger('plugin.DBSaverX60')
        self.now = 0  # This changes in MainLoop
        self.config = None
        self.period = AGGREGATE_PERIOD  # once an hour
        self.server_id = 'localhost'
        self.verbose_ = False
        self.execute_timeout = 20
        self.progress = ProgressCallback(time_change_step=0.1)

    def set_config(self, _config):
        self.config = _config

    def aggregate_db_data_by_hours(self, utc_from, utc_to):
        if self.verbose_:
            self.progress(0)
        utc_from_ = int(utc_from) // int(AGGREGATE_PERIOD)*int(AGGREGATE_PERIOD)
        if utc_from_ != utc_from:
            utc_from_ += AGGREGATE_PERIOD

        if utc_to - utc_from_ > AGGREGATE_PERIOD:
            self.log.debug("Data aggregation from %s to %s started.", utc_from_, utc_to)
            for aggr_cycle, hour_from in enumerate(range(utc_from_, int(utc_to), AGGREGATE_PERIOD), 1):
                hour_to = hour_from + AGGREGATE_PERIOD
                if hour_to > utc_to:
                    break
                history_show = HistoryShow(
                    dbengine=self.engine,
                    period_from=hour_from + 1,  # "+1" for exclude "hour_from" timestamp
                    period_to=hour_to,
                    show_columns=self.FIELDS,
                    server_id=self.server_id,
                    log=self.log)
                rows = history_show.proceed()
                mass_insert_values = list()
                for row in rows:
                    one_insert_values = {
                        'server_id': self.server_id,
                        'time': history_show.time_count,
                        'created': hour_to}
                    one_insert_values.update(
                        {FIELD_TO_TABLE_COLUMN[field_]: row[indx_] for indx_, field_ in enumerate(self.FIELDS)})
                    # correct cpu
                    one_insert_values['cpu'] *= 100
                    one_insert_values['cpu_limit'] *= 100
                    mass_insert_values.append(one_insert_values)

                with reboot_lock():
                    conn_ = self.engine.connect()
                    trans = conn_.begin()
                    try:
                        if mass_insert_values:
                            for chunk in get_chunks(mass_insert_values):
                                conn_.execute(insert(history_x60), chunk)
                        else:
                            conn_.execute(insert(history_x60), {'server_id': self.server_id, 'created': hour_to, 'id': 0})
                        if self.verbose_:  # show progress
                            progress_ = 100 - (utc_to-hour_from)*100 // (utc_to-utc_from_)
                            self.progress(progress_)
                    except OperationalError as oe:
                        trans.rollback()
                        self.log.error('Can not write aggregated data to database: %s', str(oe))
                        if self.verbose_:
                            print(' Warning: {}'.format(str(oe)[:255] + '...'))
                    except LveStatsPluginTerminated:
                        trans.rollback()
                        conn_.close()
                        self.log.debug("Plugin is terminated.")
                        raise LveStatsPluginTerminated()
                    except TypeError as oe:
                        trans.rollback()
                        conn_.close()
                        msg = str(oe)
                        # try fix incorrect database records after migration
                        if "*=: 'NoneType' and 'int'" not in msg:
                            raise
                        self.log.warning(msg)
                        from lvestats.lib.dbengine import fix_db
                        fix_db(self.engine, self.config,
                               from_timestmp=hour_from, to_timestamp=hour_to,
                               log_=self.log)
                        return
                    else:
                        trans.commit()
                        conn_.close()

                work_time = time.time() - self.now  # calculate plugin working time
                if (self.execute_timeout > 0 and
                        self.execute_timeout - work_time < work_time/aggr_cycle + 1):  # continue if can't finish in time
                    self.log.debug('Data was aggregated from %s to %s in %s seconds', utc_from_, hour_to, work_time)
                    return
            self.log.debug("Data aggregation done.")
        else:
            self.log.debug("Nothing to aggregate.")
        if self.verbose_:
            self.progress(100, force=True)
            self.progress.stop()

    def execute(self, lve_data):
        self.log.debug("Plugin executed")
        self.server_id = self.config.get('server_id', 'localhost')
        dt_x1_max, dt_x1_min, dt_x60_max, dt_x60_min = self.get_max_min_ts()

        self.aggregate_new(dt_x1_max, dt_x1_min, dt_x60_max)
        self.aggregate_migrated(dt_x1_min, dt_x60_min)

    def aggregate_migrated(self, dt_x1_min, dt_x60_min):
        self.log.debug("Aggregating migrated")
        if dt_x1_min is not None and dt_x60_min is not None and dt_x1_min + AGGREGATE_PERIOD < dt_x60_min:
            self.aggregate_db_data_by_hours(dt_x60_min - AGGREGATE_PERIOD, dt_x60_min)

    def aggregate_new(self, dt_x1_max, dt_x1_min, dt_x60_max):
        if dt_x1_max:
            if not dt_x60_max:
                dt_x60_max = dt_x1_min  # from minimum timestamp
            if self.verbose_:
                print('[lve-stats]: Start aggregating data from "{}" to "{}"; to skip press Ctrl+C ...'.format(
                    dt_x60_max, dt_x1_max))
            # correct execute period
            if dt_x1_max - dt_x60_max > AGGREGATE_PERIOD * 2:  # run plugin often if need aggregate more than one hour
                self.period = 60  # FIXME: This looks like mistake?
            else:
                self.period = AGGREGATE_PERIOD
            self.aggregate_db_data_by_hours(dt_x60_max, dt_x1_max)

    def get_max_min_ts(self):
        conn_ = self.engine.connect()
        try:
            # get last created
            query = select(
                [func.max(history.created), func.min(history.created)]).where(
                self.server_id == history.server_id)
            time_start = time.time()
            row = conn_.execute(query)
            dt_x1_max, dt_x1_min = row.fetchone()
            self.log.debug(
                '%s; query time: %s'
                % (str(query.compile(compile_kwargs={"literal_binds": True})).replace('\n', ' '),
                   time.time() - time_start))

            # get last created_max
            query = select(
                [func.max(history_x60.created), func.min(history_x60.created)]).where(
                self.server_id == history_x60.server_id)
            row = conn_.execute(query)
            dt_x60_max, dt_x60_min = row.fetchone()
            self.log.debug(
                '%s; query time: %s'
                % (str(query.compile(compile_kwargs={"literal_binds": True})).replace('\n', ' '),
                   time.time() - time_start))
        except LveStatsPluginTerminated:
            conn_.close()
            self.log.debug("Plugin is terminated.")
            raise LveStatsPluginTerminated()
        else:
            conn_.close()

        return dt_x1_max, dt_x1_min, dt_x60_max, dt_x60_min


def main(argv_, config_, db_engine=None):
    lve_data = dict()
    verbose_ = '--verbose' in argv_
    if db_engine is None:
        db_engine = dbengine.make_db_engine(config_, debug=False)
    if 'init' in argv_:
        if verbose_:
            print('[lve-stats]: creating table "{}"...'.format(history_x60.__tablename__))
        with db_engine.begin() as conn_:
            history_x60.__table__.create(bind=conn_, checkfirst=True)  # create table if exist
    if 'aggregate' in argv_:
        # configure plugin
        plugin_instance = DBSaverX60()
        plugin_instance.config = config_  # use specific config
        plugin_instance.set_db_engine(db_engine)
        plugin_instance.execute_timeout = 2*60*60  # two hours
        plugin_instance.verbose_ = verbose_
        plugin_instance.now = time.time()

        plugin_instance.execute(lve_data)  # generate aggregate data


# for initial from command line
# create table if exist and generate aggregate data
if __name__ == '__main__' and len(sys.argv) >= 2:
    import lvestats.lib.config as config
    try:
        cnf = config.read_plugin_config(config.read_config(), plugin=DBSaverX60.__name__)
        main(argv_=sys.argv[1:], config_=cnf)
    except KeyboardInterrupt:
        print('Stopped')
    except dbengine.MakeDbException as e:
        print(str(e))
    except config.ConfigError as ce:
        ce.log_and_exit()