Sindbad~EG File Manager
# coding=utf-8
#
# Copyright © Cloud Linux GmbH & Cloud Linux Software, Inc 2010-2019 All Rights Reserved
#
# Licensed under CLOUD LINUX LICENSE AGREEMENT
# http://cloudlinux.com/docs/LICENSE.TXT
from __future__ import absolute_import
from __future__ import division
from builtins import range
from clcommon.cpapi.pluginlib import getuser
from lvestats.lib.uidconverter import uid_to_username
from lvestats.lib.lveinfolib import HistoryShow, FIELD_LIMIT
from lvestats.lib.chart import ChartMain
from lvestats.lib.commons import dateutil, sizeutil
from lvestats.lib.chart.util import X_LEGEND_POINTS
from lvestats.lib import lveinfolib_gov
from lvestats.lib.commons import func
def _4k_page_to_bytes(v):
return (v or 0) * 4 * 1024
SHOW_COLUMNS = ('From', 'aCPU', 'lVMem', 'aPMem', 'lPMem', 'NprocF', 'PMemF', 'VMemF',
'aIO', 'lIO', 'lIOPS', 'IOPSf',
'IOf', 'lCPU', 'CPUf', 'aVMem', 'aEP', 'lEP', 'aNproc', 'lNproc',
'EPf', 'aIOPS', 'mIOPS',)
class LveChart(ChartMain):
def __init__(self, config):
super(LveChart, self).__init__('lvechart',
'Creates a chart representing usage pattern for LVE/user',
config)
self.MINIMAL_TIME_STEP = int(config.get('aggregation_period', 60))
self.BYTES_IN_MB = 1024 * 1024
self.governor_mode = func.get_governor_mode()
self.user_ignore = False
def customize_parser(self, parser):
is_admin = getuser() == 'root'
id_user_group = parser.add_mutually_exclusive_group(required=is_admin)
id_user_group.add_argument('--id',
help='LVE id -- will display record only for that LVE id',
dest='user_id',
type=int)
id_user_group.add_argument('--user',
help='Use username instead of LVE id, and show only record for that user',
dest='user_name',
default=None)
return parser
def append_dbgov_data_to_dict(self, dictionary, governor_mode_is_all, num, dbcpu_sum, dbio_sum, _lve_time_index):
"""
:param dictionary: dict to wich we append dbgov data
:param governor_mode_is_all:
:param num: number of dbgov data points near this LVEStatsHistory time
:param dbcpu_sum: sum value of dbgov cpu near this LVEStatsHistory time
:param dbio_sum: sum value of dbgov io near this LVEStatsHistory time
:param _lve_time_index: index of lve point
:return:
"""
if num > 0:
if governor_mode_is_all:
# append mean value
dictionary["dbcpu"].append(min(dbcpu_sum / num, dictionary['acpu'][_lve_time_index]))
try:
dictionary["dbio"].append(
min(dbio_sum / num * self.BYTES_IN_MB, dictionary['aio'][_lve_time_index]))
except (KeyError, IndexError):
# if lve_version <= 4 there is no by_key['aio']
pass
else:
dictionary["dbcpu"].append(dbcpu_sum / num)
dictionary["dbio"].append(dbio_sum / num * self.BYTES_IN_MB)
# when there was no dbgov data near this LVEStatsHistory time
else:
dictionary["dbcpu"].append(0)
dictionary["dbio"].append(0)
@staticmethod
def make_lvedata_with_idle(lvedata, show_columns, period_from, period_to, time_step):
def get_time_from(index):
try:
return int(lvedata[index][show_columns.index('From')])
except IndexError:
return 0
lvedata_with_idle = list() # prepare data for correct show with server idle
indx_ = 0
for period_from_ in range(period_from, period_to, time_step):
if lvedata and period_from_ - time_step / 2 < get_time_from(indx_) <= period_from_ + time_step / 2:
lvedata_with_idle.append(lvedata[indx_])
indx_ += 1
else:
line_ = list()
for item in show_columns:
if item == 'From':
line_.append(period_from_)
elif item in FIELD_LIMIT:
# calculate limit for limit fields
if lvedata:
# use previous limit
line_.append(lvedata[max(0, indx_ - 1)][show_columns.index(item)])
else:
line_.append(0)
else:
line_.append(0)
lvedata_with_idle.append(line_)
return lvedata_with_idle
def get_chart_data(self,
engine,
from_ts,
to_ts,
server,
user_id,
show_all=False):
username = uid_to_username(
uid=user_id,
local_server_id=self.cfg.get("server_id", "localhost"),
server_id=server,
db_engine=engine
)
self.user_ignore = func.get_governor_ignore_for_user(username)
utc_from = dateutil.local_to_gm(from_ts)
utc_to = dateutil.local_to_gm(to_ts)
period_sec, time_step = self.get_time_step(utc_from, utc_to)
lvedata_with_idle = self.load_data(engine, server, time_step, user_id, utc_from, utc_to)
by_key_ = {col.lower(): [row[col_index] for row in lvedata_with_idle] for col_index, col in
enumerate(SHOW_COLUMNS)}
by_key = LveChart.convert_lvedata_to_dict(by_key_)
lve_times = by_key['from']
del by_key['from']
show_columns = ('ts', 'cpu', 'lcpu', 'read', 'lread', 'write', 'lwrite')
if self.governor_mode != "none" and not self.user_ignore:
dbdata = lveinfolib_gov.HistoryShowDBGov(
engine,
utc_from,
utc_to,
uid=user_id,
server_id=server,
show_columns=show_columns,
cfg=self.cfg,
).history_dbgov_show()
data_collected = LveChart.convert_dbdata_to_dict(dbdata, show_columns)
dbtimes = data_collected['ts']
del data_collected['ts']
time_step2 = time_step / 2.0
# append dbgov stats to the nearest HistoryShow points
for lve_time_index, lve_time in enumerate(lve_times):
dbcpu, dbio = 0, 0
tmp_dbtimes_list = dbtimes[:] # temporary list for iteration
for dbtime_index, dbtime in enumerate(tmp_dbtimes_list):
# check if dbgov data is near this LVEStatsHistory time
if dbtime < lve_time + time_step2:
dbcpu += data_collected['cpu'].pop(0)
dbio += data_collected["read"].pop(0) + data_collected["write"].pop(0)
dbtimes.pop(0)
else:
self.append_dbgov_data_to_dict(
by_key,
self.governor_mode == "all",
dbtime_index,
dbcpu,
dbio,
lve_time_index)
break
else:
self.append_dbgov_data_to_dict(
by_key,
self.governor_mode == "all",
len(tmp_dbtimes_list),
dbcpu,
dbio,
lve_time_index)
return by_key, lve_times, period_sec
def load_data(self, engine, server, time_step, user_id, utc_from, utc_to):
history_show = HistoryShow(dbengine=engine, period_from=utc_from, period_to=utc_to, uid=user_id,
show_columns=SHOW_COLUMNS, server_id=server, time_unit=time_step)
lvedata = list(history_show.proceed())
lvedata_with_idle = self.make_lvedata_with_idle(lvedata, SHOW_COLUMNS, history_show.period_from,
history_show.period_to, time_step)
return lvedata_with_idle
def get_time_step(self, utc_from, utc_to):
dt = utc_to - utc_from
period_sec = dt.total_seconds()
return period_sec, self.MINIMAL_TIME_STEP
@staticmethod
def _add_faults_graph(renderer, data_collected, times, lve_version):
legend = {
'CPUf': ('CPUf', '#8DB600'),
'EPf': ('EPf', 'green',),
'VMemF': ('VMemF', '#00DDFF'),
'NprocF': ('NprocF', 'red'),
'PMemF': ('PMemF', 'blue'),
'IOf': ('IOf', '#9966CC')}
if lve_version >= 8:
legend.update({
'IOPSf': ('IOPSf', '#007FFF'),
})
legend_keys = tuple(key.lower() for key in list(legend.keys()))
faults_present = any(tuple(any(data_collected[key]) for key in legend_keys))
message = 'No Faults' if not faults_present else None
renderer.add_graph(
data_collected, 'Faults',
legend=legend, x_values=times, min_y=0,
y_legend_converter=sizeutil.convert_powers_of_1000_for_graph,
message=message)
def add_graphs(self, renderer, data_collected, times, lve_version, show_all, is_user=False):
title_suffix = self._get_title_suffix(data_collected)
is_lcpu_limited = any(data_collected['lcpu'])
if is_lcpu_limited or show_all:
if is_user and self.is_normalized_user_cpu and is_lcpu_limited:
self._normalize_cpu(data_collected)
cpu_legend = {'aCPU': ('average', 'green'), 'lCPU': ('limit', 'red')}
if show_all and not is_lcpu_limited:
del cpu_legend['lCPU']
if len(data_collected['dbcpu']) > 0:
cpu_legend['dbcpu'] = ('database', 'blue')
renderer.add_graph(data_collected, 'CPU Usage' + title_suffix,
legend=cpu_legend,
x_values=times, min_y=0, unit='%',
y_legend_converter=int,
faults=('CPUf', 'aCPU', 'lCPU'))
is_lvmem_limited = any(data_collected['lvmem'])
if is_lvmem_limited or show_all:
vmem_legend = {
'lVmem': ('limit', 'red', _4k_page_to_bytes),
'aVmem': ('average', 'green', _4k_page_to_bytes)}
if show_all and not is_lvmem_limited:
del vmem_legend['lVmem']
renderer.add_graph(data_collected, 'Virtual Memory Usage',
legend=vmem_legend, min_y=0, x_values=times,
y_legend_converter=sizeutil.convert_bytes_for_graph)
is_lpmem_limited = any(data_collected['lpmem'])
if is_lpmem_limited or show_all:
pmem_legend = {
'lPMem': ('limit', 'red', _4k_page_to_bytes),
'aPMem': ('average', 'green', _4k_page_to_bytes)}
if show_all and not is_lpmem_limited:
del pmem_legend['lPMem']
renderer.add_graph(data_collected, 'Physical Memory Usage',
legend=pmem_legend, x_values=times, min_y=0,
y_legend_converter=sizeutil.convert_bytes_for_graph)
is_lio_limited = any(data_collected['lio'])
if is_lio_limited or show_all:
io_legend = {
'lIO': ('limit', 'red'),
'aIO': ('average', 'green')}
if show_all and not is_lio_limited:
del io_legend['lIO']
if len(data_collected['dbio']) > 0:
io_legend['dbio'] = ('database', 'blue')
renderer.add_graph(data_collected, 'Input/Output Usage' + title_suffix,
legend=io_legend, x_values=times, min_y=0,
y_legend_converter=sizeutil.convert_bytes_for_graph,
unit='/s',
faults=('IOf', 'aIO', 'lIO'))
if lve_version > 6:
is_liops_limited = any(data_collected['liops'])
if is_liops_limited or show_all:
iops_legend = {
'lIOPS': ('limit', 'red'),
'aIOPS': ('average', 'green')}
if show_all and not is_liops_limited:
del iops_legend['lIOPS']
renderer.add_graph(data_collected, 'Io operations',
legend=iops_legend, x_values=times, min_y=0,
y_legend_converter=sizeutil.convert_powers_of_1000_for_graph,
unit='/s',
faults=('IOPSf', 'aIOPS', 'lIOPS'))
is_lep_limited = any(data_collected['lep'])
if is_lep_limited or show_all:
ep_legend = {
'lEp': ('limit', 'red'),
'aEp': ('average', 'green')}
if show_all and not is_lep_limited:
del ep_legend['lEp']
renderer.add_graph(data_collected, 'Entry Processes',
legend=ep_legend, x_values=times, min_y=0,
y_legend_converter=sizeutil.convert_powers_of_1000_for_graph)
is_lnproc_limited = any(data_collected['lnproc'])
if is_lnproc_limited or show_all:
nproc_legend = {
'lNproc': ('limit', 'red'),
'aNproc': ('average', 'green')}
if show_all and not is_lnproc_limited:
del nproc_legend['lNproc']
renderer.add_graph(data_collected, 'Processes',
legend=nproc_legend, x_values=times, min_y=0,
y_legend_converter=sizeutil.convert_powers_of_1000_for_graph)
self._add_faults_graph(renderer, data_collected, times, lve_version)
renderer.add_common_x_legend(times, X_LEGEND_POINTS)
@staticmethod
def _normalize_cpu(data_collected):
# Magnify aCPU, dbCPU, lCPU to 100%
# 1. Find max limit
max_limit = max(data_collected['lcpu'])
# 2. Calculate New user's data and limits
a_cpu_new = []
l_cpu_new = []
db_cpu_new = []
for idx in range(0, len(data_collected['lcpu'])):
# New limit
l_cpu_new.append(data_collected['lcpu'][idx] * 100.0 / max_limit)
# New lve cpu average
a_cpu_new.append(data_collected['acpu'][idx] * 100.0 / max_limit)
if len(data_collected['dbcpu']) > 0:
# New database cpu average
db_cpu_new.append(data_collected['dbcpu'][idx] * 100.0 / max_limit)
# Store new data
data_collected['lcpu'] = l_cpu_new
data_collected['acpu'] = a_cpu_new
if db_cpu_new:
data_collected['dbcpu'] = db_cpu_new
def _get_title_suffix(self, data_collected):
# <lve use="on|single|off|abusers|all"/>
if len(data_collected['dbcpu']) == 0 or self.governor_mode == "none" or self.user_ignore:
title_suffix = ""
elif self.governor_mode == "all":
title_suffix = ". DB usage included"
elif self.governor_mode == "abusers":
title_suffix = ". DB usage included, only if restricted"
elif self.governor_mode == "off":
title_suffix = ". DB usage is not limited, nor accounted for"
else:
# single or on
title_suffix = ". DB usage is not accounted as part of LVE"
return title_suffix
Sindbad File Manager Version 1.0, Coded By Sindbad EG ~ The Terrorists