Sindbad~EG File Manager
# coding=utf-8
#
# Copyright © Cloud Linux GmbH & Cloud Linux Software, Inc 2010-2019 All Rights Reserved
#
# Licensed under CLOUD LINUX LICENSE AGREEMENT
# http://cloudlinux.com/docs/LICENSE.TXT
from __future__ import print_function
from __future__ import division
from __future__ import absolute_import
from builtins import range, zip
import argparse
import logging
import datetime
import sys
import os
import subprocess
from collections import defaultdict
from bisect import bisect_left
import lvestats.lib.commons.decorators
from lvestats.lib.cloudlinux_statistics import _get_uid_for_select
from lvestats.lib import config, dbengine, lveinfolib, uidconverter
from lvestats.lib.chart.svggraph import SvgChart
from lvestats.lib.chart.rdp import ramerdouglas
from lvestats.lib.commons import dateutil
from clcommon.cpapi.pluginlib import getuser
from lvestats.lib.commons.argparse_utils import period_type2, ParseDatetime
from lvestats.lib.commons.logsetup import setup_logging
from sqlalchemy.exc import SQLAlchemyError
import lvestats
from lvestats.lib.chart.polysimplify import VWSimplifier
import numpy as np
__author__ = 'shaman'
DEFAULT_SERVER_ID = 'localhost'
IMAGEMAGICK_CONVERT = "/opt/alt/alt-ImageMagick/usr/bin/convert"
if not os.path.exists(IMAGEMAGICK_CONVERT):
IMAGEMAGICK_CONVERT = "/usr/bin/convert"
format_index = 1
class Renderer(object):
"""
Renders data to file
"""
MAX_POINTS_ON_GRAPH = 800
FAULT_COLOR = 'aquamarine'
def __init__(self):
self.log = logging.getLogger('Chart Renderer')
self.svg_chart = SvgChart()
@staticmethod
def _nop(arg):
return arg
@staticmethod
def _ts_to_str(ts):
# Example: [0]="5 Feb", [1]="Aug 15, 2:35pm"
formats = ['%b-%d', '%b-%d %I:%M%p']
gm = dateutil.unixtimestamp_to_gm_datetime(ts)
lo = dateutil.gm_to_local(gm)
return lo.strftime(formats[format_index])
def set_period_sec(self, period_sec):
global format_index
# 8 days = 8 * 24 * 3600 = 691200 sec
if period_sec >= 691200:
format_index = 0
def _optimisation_ramerdouglas(self, points, epsilon_max=2.0, epsilon_min=0):
epsilon_default = epsilon_max
points_output = points
points_len = 0
for i in range(10):
points_optimised = ramerdouglas(points, epsilon_default)
points_len_previous = points_len
points_len = len(points_optimised)
if points_len == points_len_previous:
# return result if changing epsilon not change points number
return points_output
if points_len >= self.MAX_POINTS_ON_GRAPH:
if i == 0:
# return result if in first iteration points more than max
return points_optimised
epsilon_min = epsilon_default
else:
points_output = points_optimised
epsilon_max = epsilon_default
# correcting epsilon
epsilon_default = (epsilon_max + epsilon_min) / 2.
return points_output
def _optimisation_polysimplify(self, points):
pts = np.array(points, dtype=np.float32)
simplifier = VWSimplifier(pts)
return simplifier.from_number(self.MAX_POINTS_ON_GRAPH)
def _optimisation_raw(self, points):
return points
def optimise_points(self, line):
line_len = len(line)
if line_len <= self.MAX_POINTS_ON_GRAPH:
line = self._optimisation_raw(line)
else:
line = self._optimisation_ramerdouglas(line, epsilon_max=1.4)
if len(line) > self.MAX_POINTS_ON_GRAPH:
line = self._optimisation_polysimplify(line)
return line
def get_two_closest(self, line, x_value):
"""
:type x_value: float
:type line: list
"""
pos = bisect_left(line, x_value)
before = line[pos - 1]
after = line[pos]
return pos - 1, before, pos, after
def add_graph(
self,
data,
title,
legend,
x_values,
min_y=None,
max_y=None,
x_labels=None,
generate_n_xlabels=None,
y_labels=None,
y_legend_converter=lambda v: v,
unit=None,
message=None,
faults=None):
"""
:type message: None|str
:type unit: None|str
:type y_legend_converter: (float) -> float
:type max_y: None|float
:type min_y: None|float
:type title: str
:type legend: dict
:type x_values: list
:type faults: (str, str, str)
:type data: defaultdict
"""
colors, datasets, fault_lines, names = self._add_graph(data, faults, legend, x_values)
self.svg_chart.add_graph(
datasets,
colors,
title=title,
minimum_y=min_y,
maximum_y=max_y,
x_legend=x_labels,
x_legend_generate=generate_n_xlabels,
y_legend=y_labels,
y_legend_converter=y_legend_converter,
x_legend_converter=self._ts_to_str,
names=names,
unit=unit,
message=message,
fault_lines=fault_lines,
fault_color=self.FAULT_COLOR)
def _add_graph(self, data, faults, legend, x_values):
"""
:type legend: dict
:type x_values: list
:type faults: (str, str, str)
:type data: defaultdict
:rtype (list, list, list, list)
"""
datasets = []
names = []
colors = []
datasets_dictionary = {}
for (key, metainfo) in legend.items():
try:
legend_title, color, modifier = metainfo
except ValueError:
legend_title, color = metainfo
modifier = self._nop
y_values = [modifier(y) for y in data[key.lower()]]
# x_values = range(0, len(y_values))
line = list(zip(x_values, y_values))
line_optimised = self.optimise_points(line)
datasets.append(line_optimised)
datasets_dictionary[key.lower()] = line_optimised
names.append(legend_title)
colors.append(color)
fault_lines = self.get_faults_lines(data, datasets_dictionary, faults, x_values)
# add legend
if fault_lines:
names.append('faults')
colors.append(self.FAULT_COLOR)
del datasets_dictionary
return colors, datasets, fault_lines, names
def get_faults_lines(self, data, datasets_dictionary, faults, x_values):
"""
:type x_values: list
:type faults: (str, str, str)
:type datasets_dictionary: dict
:type data: defaultdict
:rtype list
"""
fault_lines = []
if faults is not None:
fault_name, data_name, limit_name = [x.lower() for x in faults]
if limit_name in datasets_dictionary and data_name in datasets_dictionary:
faults_x = [x for x, y in zip(x_values, data[fault_name]) if y > 0]
average_times, average = list(zip(*datasets_dictionary[data_name]))
limit_times, limit = list(zip(*datasets_dictionary[limit_name]))
for fault in faults_x:
try:
average_dot = self.get_dot(fault, average, average_times)
limit_dot = self.get_dot(fault, limit, limit_times)
if average_dot[1] < limit_dot[1] and (average_dot, limit_dot) not in fault_lines:
fault_lines.append((average_dot, limit_dot))
except IndexError:
self.log.error("Can't get fault line: %s", str(fault))
return fault_lines
def get_dot(self, fault_time, line, line_times):
"""
:type line_times: list
:type line: list
:type fault_time: float
:rtype (float, float)
"""
if fault_time >= line_times[-1]:
return fault_time, line[-1]
if fault_time <= line_times[0]:
return fault_time, line[0]
try:
dot = (fault_time, line[line_times.index(fault_time)])
except ValueError:
before_index, before, after_index, after = self.get_two_closest(line_times, fault_time)
dot_y = (fault_time - before) / (after - before) * (line[after_index] - line[before_index]) + line[before_index]
dot = (fault_time, dot_y)
return dot
def add_common_x_legend(self, x_values, n):
self.svg_chart._add_x_legend(x_values=x_values, number=n, x_legend_converter=self._ts_to_str)
def add_text_box(self, text, font_size=None):
"""
add empty rectangle with text
"""
self.svg_chart.add_text_box(text, font_size)
def render(self):
return self.svg_chart.dump()
class UserNotFound(Exception):
def __init__(self, user_name, *args, **kwargs):
super(UserNotFound, self).__init__('User %s not found' % user_name, *args, **kwargs)
class ChartMain(object):
def __init__(self, prog_name, prog_desc, cnf):
self.prog_name = prog_name
self.prog_desc = prog_desc
self.log = setup_logging(cnf, prog_name, console_level=logging.ERROR)
self.cfg = cnf
# analog of previous parameter but get from config file
self.is_normalized_user_cpu = config.is_normalized_user_cpu()
@staticmethod
def convert_dbdata_to_dict(data, show_columns):
data_collected = defaultdict(list)
for row in data:
row_dict = dict(zip(show_columns, row))
for (k, v) in row_dict.items():
data_collected[k].append(float(v or 0))
return data_collected
@staticmethod
def convert_lvedata_to_dict(data):
# key -- field name, like 'cpu', 'id', 'created', etc.. value -- list of values
by_key = defaultdict(list)
for (k, values) in data.items():
by_key[k] = [float(v or 0) for v in values]
return by_key
def make_parser(self):
current_server_id = self.cfg.get('server_id', DEFAULT_SERVER_ID)
datetime_now = datetime.datetime.now()
parser = argparse.ArgumentParser(prog=self.prog_name,
add_help=True,
description=self.prog_desc)
parser.add_argument('--version', version=lvestats.__version__,
help='Version number',
dest='version',
action='version')
parser.add_argument('--period',
help='Time period\n'
'specify minutes with m, h - hours, days with d, and values: today, yesterday\n'
'5m - last 5 minutes, 4h - last four hours, 2d - last 2 days, as well as today',
type=lambda value: period_type2(value, datetime_now),
default=None
)
parser.add_argument('--from',
help='Run report from date and time in YYYY-MM-DD HH:MM format\n'
'if not present last 10 minutes are assumed',
action=ParseDatetime,
nargs='+',
dest='ffrom')
parser.add_argument('--to',
help='Run report up to date and time in YYYY-MM-DD HH:MM format\n'
'if not present, reports results up to now',
action=ParseDatetime,
nargs='+',
dest='to')
parser.add_argument('--server',
help='With username or LVE id show only record for that user at given server',
dest='server',
default=current_server_id)
parser.add_argument('--output',
help='Filename to save chart as, if not present, output will be sent to STDOUT',
default='-', dest='output')
parser.add_argument('--show-all',
help='Show all graphs (by default shows graphs for which limits are set)',
dest='show_all',
action='store_true',
default=False)
parser.add_argument('--style',
help='Set graph style',
choices=['admin', 'user'],
dest='style',
default=None)
parser.add_argument('--format',
help='Set graph output format',
choices=['svg', 'png'],
dest='format',
default='png')
parser.add_argument('--for-reseller',
help='Show graph for specific reseller and user (used only with --id key)',
type=str,
dest='for_reseller',
default=None)
# add opt --dpi, --width, --height for backward compatible with old lvechart
parser.add_argument('--dpi', help=argparse.SUPPRESS)
parser.add_argument('--width', help=argparse.SUPPRESS)
parser.add_argument('--height', help=argparse.SUPPRESS)
return parser
def customize_parser(self, parser):
"""
:type parser: argparse.ArgumentParser
:rtype : argparse.ArgumentParser
"""
raise NotImplementedError()
def get_chart_data(self,
engine,
from_ts,
to_ts,
server,
user_id,
show_all=False):
"""
Extracts data from database , in form of a tuple:
(dict, list)
where dict is
{
'cpu': [0, 5, 10 , 95.2, ...] -- values
'io': [0, 5, 10 , 95.2, ...] -- values
'foo': ,
}
and a list of timestamps for all that collected values:
2) [1, 2 ,3 ...]
:param show_all:
:rtype : tuple
:type engine: sqlalchemy.engine
:type from_ts: datetime.datetime
:type to_ts: datetime.datetime
:type server: str
:type user_id: int
"""
raise NotImplementedError()
def add_graphs(self, renderer, data_collected, times, lve_version, show_all, is_user=False):
"""
:rtype : None
:type renderer: Renderer
:type data_collected: dict
:type times: list
:type lve_version: str
:type show_all: bool
:type is_user: bool. True for user, False - admin
"""
raise NotImplementedError()
@lvestats.lib.commons.decorators.no_sigpipe
def main(self, args, debug_engine=None):
parser = self.make_parser()
parser = self.customize_parser(parser)
if debug_engine is not None:
engine = debug_engine
else:
try:
engine = dbengine.make_db_engine(self.cfg)
except dbengine.MakeDbException as e:
sys.stderr.write(str(e) + '\n')
return 1
opts = parser.parse_args(list(args))
if opts.period and any((opts.ffrom, opts.to)):
print("--period and [--from, --to] are mutually exclusive")
return 0
if opts.period:
f, t = opts.period
else:
datetime_now = datetime.datetime.now()
f = opts.ffrom or datetime_now - datetime.timedelta(minutes=10)
t = opts.to or datetime_now
user_id = self._obtain_user_id(engine, opts) if self._has_user_arguments(opts) else os.getuid()
if getuser() != 'root':
if user_id and user_id != os.getuid():
sys.stderr.write('Permission denied\n')
return 1
# LVESTATS-97
# If both params (for_reseller and id or username) are used
# and user_id not in resellers_users, we return `Permission denied` message
if opts.for_reseller:
reseller_name = opts.for_reseller
reseller_users = _get_uid_for_select(reseller_name, user_id)
# if reseller_users is just int, than everything is ok
if isinstance(reseller_users, int):
pass
# if not user's id is in reseller's ids
# neither user is reseller and user's id is reseller's id itself
elif user_id not in reseller_users:
error_msg = 'Permission denied. User with id ' \
'{} does not belong reseller `{}`\n'.format(
user_id, reseller_name)
sys.stderr.write(error_msg)
return 1
try:
data_collected, times, period_sec = self.get_chart_data(engine,
f,
t,
opts.server,
user_id=user_id)
except (UserNotFound, SQLAlchemyError) as ex:
sys.stderr.write(str(ex) + '\n')
return 1
rendered_graph = self._render(data_collected, engine, opts, period_sec, times)
self._output(opts, rendered_graph)
return 0
def _render(self, data_collected, engine, opts, period_sec, times):
show_all = opts.show_all
try:
style = opts.style
except AttributeError:
style = None
if style is None:
style = 'admin'
renderer = Renderer()
renderer.set_period_sec(period_sec)
lve_version = lveinfolib.get_lve_version(engine, opts.server)
self.add_graphs(renderer, data_collected, times, lve_version, show_all,
is_user=(style == 'user'))
rendered_graph = renderer.render()
return rendered_graph
def _output(self, opts, rendered_graph):
if opts.format == 'svg' and opts.output == '-':
# output as SVG
try:
from lxml import etree
root_node = etree.fromstring(rendered_graph.encode('utf8'))
rendered_graph = etree.tostring(root_node, pretty_print=True)
except (ImportError, ValueError, UnicodeEncodeError) as e:
self.log.debug('Can not use pretty print for svg xml formatting; %s', str(e))
if opts.format == 'png':
# output as PNG
cwd = '/var/lve/tmp' if os.getuid() == 0 else None
proc = subprocess.Popen(
[IMAGEMAGICK_CONVERT, "-", "png:-"],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
cwd=cwd)
rendered_graph, _ = proc.communicate(input=rendered_graph.encode())
if opts.output == '-':
if isinstance(rendered_graph, bytes):
sys.stdout.buffer.flush() # needed to write directly to buffer
sys.stdout.buffer.write(rendered_graph)
else:
sys.stdout.write(rendered_graph)
else:
try:
if isinstance(rendered_graph, bytes):
write_mode = 'wb'
else:
write_mode = 'w'
with open(opts.output, write_mode) as output:
output.write(rendered_graph)
except IOError:
self.log.error("Unable to create file: %s", opts.output)
def _obtain_user_id(self, engine, opts):
# obtain user id
try:
user_id = opts.user_id
except AttributeError:
user_id = None
try:
user_name = opts.user_name
except AttributeError:
user_name = None
if user_id is None:
user_id = uidconverter.username_to_uid(
username=user_name, local_server_id=self.cfg['server_id'],
server_id=opts.server, db_engine=engine) or -1
return user_id
def _has_user_arguments(self, opts):
return (hasattr(opts, 'user_id') and opts.user_id) or (hasattr(opts, 'user_name') and opts.user_name)
Sindbad File Manager Version 1.0, Coded By Sindbad EG ~ The Terrorists