HOME

Mini Shell 1.0
DIR:/proc/self/root/opt/cloudlinux/venv/lib/python3.11/site-packages/lvestats/lib/chart/
Current File : //proc/self/root/opt/cloudlinux/venv/lib/python3.11/site-packages/lvestats/lib/chart/__init__.py
# coding=utf-8
#
# Copyright © Cloud Linux GmbH & Cloud Linux Software, Inc 2010-2019 All Rights Reserved
#
# Licensed under CLOUD LINUX LICENSE AGREEMENT
# http://cloudlinux.com/docs/LICENSE.TXT

import argparse
import logging
import datetime
import sys
import os
import subprocess
from bisect import bisect_left
from collections import defaultdict
from typing import Iterable, List

from lxml import etree
from sqlalchemy.exc import SQLAlchemyError

import lvestats
import lvestats.lib.commons.decorators
from clcommon.cpapi.pluginlib import getuser
from lvestats.lib.cloudlinux_statistics import _get_uid_for_select
from lvestats.lib import config, dbengine, lveinfolib, uidconverter
from lvestats.lib.chart.svggraph import SvgChart
from lvestats.lib.chart.rdp import ramerdouglas
from lvestats.lib.commons import dateutil
from lvestats.lib.commons.argparse_utils import period_type2, ParseDatetime
from lvestats.lib.commons.logsetup import setup_logging
from lvestats.lib.chart.polysimplify import VWSimplifier

__author__ = 'shaman'

DEFAULT_SERVER_ID = 'localhost'
IMAGEMAGICK_BINARY = "/opt/alt/alt-ImageMagick/usr/bin/magick"
if not os.path.exists(IMAGEMAGICK_BINARY):
    IMAGEMAGICK_BINARY = "/usr/bin/convert"

format_index = 1


class Renderer(object):
    """
    Renders data to file
    """

    def __init__(self, max_points_on_graph: int = 800, fault_color: str = 'aquamarine'):
        self.log = logging.getLogger('Chart Renderer')
        self.svg_chart = SvgChart()
        self._max_points_on_graph = max_points_on_graph
        self._fault_color = fault_color

    @staticmethod
    def _nop(arg):
        return arg

    @staticmethod
    def _ts_to_str(ts):
        # Example: [0]="5 Feb", [1]="Aug 15, 2:35pm"
        formats = ['%b-%d', '%b-%d %I:%M%p']
        gm = dateutil.unixtimestamp_to_gm_datetime(ts)
        lo = dateutil.gm_to_local(gm)
        return lo.strftime(formats[format_index])

    def set_period_sec(self, period_sec):
        global format_index
        # 8 days = 8 * 24 * 3600 = 691200 sec
        if period_sec >= 691200:
            format_index = 0

    def _optimisation_ramerdouglas(self, points, epsilon_max=2.0, epsilon_min=0):
        epsilon_default = epsilon_max
        points_output = points
        points_len = 0
        for i in range(10):
            points_optimised = ramerdouglas(points, epsilon_default)
            points_len_previous = points_len
            points_len = len(points_optimised)
            if points_len == points_len_previous:
                # return result if changing epsilon not change points number
                return points_output
            if points_len >= self._max_points_on_graph:
                if i == 0:
                    # return result if in first iteration points more than max
                    return points_optimised
                epsilon_min = epsilon_default
            else:
                points_output = points_optimised
                epsilon_max = epsilon_default
            # correcting epsilon
            epsilon_default = (epsilon_max + epsilon_min) / 2.
        return points_output

    def _optimisation_polysimplify(self, points: Iterable[Iterable[float]]) -> List[List[float]]:
        simplifier = VWSimplifier(points)
        result = simplifier.from_number(self._max_points_on_graph)
        return result

    def optimise_points(self, line: Iterable[Iterable[float]]) -> List[List[float]]:
        if len(line) > self._max_points_on_graph:
            line = self._optimisation_ramerdouglas(line, epsilon_max=1.4)
            line = self._optimisation_polysimplify(line)
        return line

    def get_two_closest(self, line, x_value):
        """
        :type x_value: float
        :type line: list
        """
        pos = bisect_left(line, x_value)
        before = line[pos - 1]
        after = line[pos]
        return pos - 1, before, pos, after

    def add_graph(
            self,
            data,
            title,
            legend,
            x_values,
            min_y=None,
            max_y=None,
            x_labels=None,
            generate_n_xlabels=None,
            y_labels=None,
            y_legend_converter=lambda v: v,
            unit=None,
            message=None,
            faults=None):
        """
        :type message: None|str
        :type unit: None|str
        :type y_legend_converter: (float) -> float
        :type max_y: None|float
        :type min_y: None|float
        :type title: str
        :type legend: dict
        :type x_values: list
        :type faults: (str, str, str)
        :type data: defaultdict
        """
        colors, datasets, fault_lines, names = self._add_graph(data, faults, legend, x_values)

        self.svg_chart.add_graph(
            datasets,
            colors,
            title=title,
            minimum_y=min_y,
            maximum_y=max_y,
            x_legend=x_labels,
            x_legend_generate=generate_n_xlabels,
            y_legend=y_labels,
            y_legend_converter=y_legend_converter,
            x_legend_converter=self._ts_to_str,
            names=names,
            unit=unit,
            message=message,
            fault_lines=fault_lines,
            fault_color=self._fault_color)

    def _add_graph(self, data, faults, legend, x_values):
        """
        :type legend: dict
        :type x_values: list
        :type faults: (str, str, str)
        :type data: defaultdict
        :rtype (list, list, list, list)
        """
        datasets = []
        names = []
        colors = []
        datasets_dictionary = {}
        for (key, metainfo) in legend.items():
            try:
                legend_title, color, modifier = metainfo
            except ValueError:
                legend_title, color = metainfo
                modifier = self._nop
            y_values = [modifier(y) for y in data[key.lower()]]
            # x_values = range(0, len(y_values))
            line = list(zip(x_values, y_values))
            line_optimised = self.optimise_points(line)
            datasets.append(line_optimised)
            datasets_dictionary[key.lower()] = line_optimised
            names.append(legend_title)
            colors.append(color)
        fault_lines = self.get_faults_lines(data, datasets_dictionary, faults, x_values)

        # add legend
        if fault_lines:
            names.append('faults')
            colors.append(self._fault_color)
        del datasets_dictionary
        return colors, datasets, fault_lines, names

    def get_faults_lines(self, data, datasets_dictionary, faults, x_values):
        """
        :type x_values: list
        :type faults: (str, str, str)
        :type datasets_dictionary: dict
        :type data: defaultdict
        :rtype list
        """
        fault_lines = []
        if faults is not None:
            fault_name, data_name, limit_name = [x.lower() for x in faults]
            if limit_name in datasets_dictionary and data_name in datasets_dictionary:
                faults_x = [x for x, y in zip(x_values, data[fault_name]) if y > 0]
                average_times, average = list(zip(*datasets_dictionary[data_name]))
                limit_times, limit = list(zip(*datasets_dictionary[limit_name]))
                for fault in faults_x:
                    try:
                        average_dot = self.get_dot(fault, average, average_times)
                        limit_dot = self.get_dot(fault, limit, limit_times)
                        if average_dot[1] < limit_dot[1] and (average_dot, limit_dot) not in fault_lines:
                            fault_lines.append((average_dot, limit_dot))
                    except IndexError:
                        self.log.error("Can't get fault line: %s", str(fault))
        return fault_lines

    def get_dot(self, fault_time, line, line_times):
        """
        :type line_times: list
        :type line: list
        :type fault_time: float
        :rtype (float, float)
        """
        if fault_time >= line_times[-1]:
            return fault_time, line[-1]
        if fault_time <= line_times[0]:
            return fault_time, line[0]
        try:
            dot = (fault_time, line[line_times.index(fault_time)])
        except ValueError:
            before_index, before, after_index, after = self.get_two_closest(line_times, fault_time)
            dot_y = (
                (fault_time - before)
                / (after - before)
                * (line[after_index] - line[before_index])
                + line[before_index]
            )
            dot = (fault_time, dot_y)
        return dot

    def add_common_x_legend(self, x_values, n):
        self.svg_chart._add_x_legend(  # pylint: disable=protected-access
            x_values=x_values,
            number=n,
            x_legend_converter=self._ts_to_str,
        )

    def add_text_box(self, text, font_size=None):
        """
        add empty rectangle with text
        """
        self.svg_chart.add_text_box(text, font_size)

    def render(self):
        return self.svg_chart.dump()


class UserNotFound(Exception):
    def __init__(self, user_name, *args, **kwargs):
        super().__init__(f'User {user_name} not found', *args, **kwargs)


class ChartMain(object):
    def __init__(self, prog_name, prog_desc, cnf):
        self.prog_name = prog_name
        self.prog_desc = prog_desc
        self.log = setup_logging(cnf, prog_name, console_level=logging.ERROR)
        self.cfg = cnf
        # analog of previous parameter but get from config file
        self.is_normalized_user_cpu = config.is_normalized_user_cpu()

    @staticmethod
    def convert_dbdata_to_dict(data, show_columns):
        data_collected = defaultdict(list)
        for row in data:
            row_dict = dict(zip(show_columns, row))
            for (k, v) in row_dict.items():
                data_collected[k].append(float(v or 0))
        return data_collected

    @staticmethod
    def convert_lvedata_to_dict(data):
        # key -- field name, like 'cpu', 'id', 'created', etc.. value -- list of values
        by_key = defaultdict(list)
        for (k, values) in data.items():
            by_key[k] = [float(v or 0) for v in values]
        return by_key

    def make_parser(self):
        current_server_id = self.cfg.get('server_id', DEFAULT_SERVER_ID)
        datetime_now = datetime.datetime.now()
        parser = argparse.ArgumentParser(prog=self.prog_name,
                                         add_help=True,
                                         description=self.prog_desc)

        parser.add_argument('--version', version=lvestats.__version__,
                            help='Version number',
                            dest='version',
                            action='version')
        parser.add_argument('--period',
                            help='Time period\n'
                                 'specify minutes with m,  h - hours, days with d, and values: today, yesterday\n'
                                 '5m - last 5 minutes, 4h - last four hours, 2d - last 2 days, as well as today',
                            type=lambda value: period_type2(value, datetime_now),
                            default=None
                            )

        parser.add_argument('--from',
                            help='Run report from date and time in YYYY-MM-DD HH:MM format\n'
                                 'if not present last 10 minutes are assumed',
                            action=ParseDatetime,
                            nargs='+',
                            dest='ffrom')
        parser.add_argument('--to',
                            help='Run report up to date and time in YYYY-MM-DD HH:MM format\n'
                                 'if not present, reports results up to now',
                            action=ParseDatetime,
                            nargs='+',
                            dest='to')

        parser.add_argument('--server',
                            help='With username or LVE id show only record for that user at given server',
                            dest='server',
                            default=current_server_id)
        parser.add_argument('--output',
                            help='Filename to save chart as, if not present, output will be sent to STDOUT',
                            default='-', dest='output')
        parser.add_argument('--show-all',
                            help='Show all graphs (by default shows graphs for which limits are set)',
                            dest='show_all',
                            action='store_true',
                            default=False)
        parser.add_argument('--style',
                            help='Set graph style',
                            choices=['admin', 'user'],
                            dest='style',
                            default=None)
        parser.add_argument('--format',
                            help='Set graph output format',
                            choices=['svg', 'png'],
                            dest='format',
                            default='png')
        parser.add_argument('--for-reseller',
                            help='Show graph for specific reseller and user (used only with --id key)',
                            type=str,
                            dest='for_reseller',
                            default=None)

        # add opt --dpi, --width, --height for backward compatible with old lvechart
        parser.add_argument('--dpi', help=argparse.SUPPRESS)
        parser.add_argument('--width', help=argparse.SUPPRESS)
        parser.add_argument('--height', help=argparse.SUPPRESS)

        return parser

    def customize_parser(self, parser):
        """
        :type parser: argparse.ArgumentParser
        :rtype : argparse.ArgumentParser
        """
        raise NotImplementedError()

    def get_chart_data(self,
                       engine,
                       from_ts,
                       to_ts,
                       server,
                       user_id,
                       show_all=False):
        """

        Extracts data from database , in form of a tuple:
        (dict, list)
        where dict is

        {
            'cpu': [0, 5, 10 , 95.2, ...] -- values
            'io': [0, 5, 10 , 95.2, ...] -- values
            'foo': ,
        }

        and a list of timestamps for all that collected values:
        2) [1, 2 ,3 ...]



        :param show_all:
        :rtype : tuple
        :type engine: sqlalchemy.engine
        :type from_ts: datetime.datetime
        :type to_ts: datetime.datetime
        :type server: str
        :type user_id: int
        """
        raise NotImplementedError()

    def add_graphs(self, renderer, data_collected, times, lve_version, show_all, is_user=False):
        """
        :rtype : None
        :type renderer: Renderer
        :type data_collected: dict
        :type times: list
        :type lve_version: str
        :type show_all: bool
        :type is_user: bool. True for user, False - admin
        """
        raise NotImplementedError()

    @lvestats.lib.commons.decorators.no_sigpipe
    def main(self, args, debug_engine=None):
        parser = self.make_parser()
        parser = self.customize_parser(parser)
        if debug_engine is not None:
            engine = debug_engine
        else:
            try:
                engine = dbengine.make_db_engine(self.cfg)
            except dbengine.MakeDbException as e:
                sys.stderr.write(str(e) + '\n')
                return 1

        opts = parser.parse_args(list(args))

        if opts.period and any((opts.ffrom, opts.to)):
            print("--period and [--from, --to] are mutually exclusive")
            return 0

        if opts.period:
            f, t = opts.period
        else:
            datetime_now = datetime.datetime.now()
            f = opts.ffrom or datetime_now - datetime.timedelta(minutes=10)
            t = opts.to or datetime_now

        user_id = self._obtain_user_id(engine, opts) if self._has_user_arguments(opts) else os.getuid()
        if getuser() != 'root':
            if user_id and user_id != os.getuid():
                sys.stderr.write('Permission denied\n')
                return 1

        # LVESTATS-97
        # If both params (for_reseller and id or username) are used
        # and user_id not in resellers_users, we return `Permission denied` message
        if opts.for_reseller:
            reseller_name = opts.for_reseller
            reseller_users = _get_uid_for_select(reseller_name, user_id)
            # if reseller_users is just int, than everything is ok
            if isinstance(reseller_users, int):
                pass
            # if not user's id is in reseller's ids
            # neither user is reseller and user's id is reseller's id itself
            elif user_id not in reseller_users:
                error_msg = (
                    f'Permission denied. User with id {user_id} does not belong reseller `{reseller_name}`\n'
                )
                sys.stderr.write(error_msg)
                return 1
        try:
            data_collected, times, period_sec = self.get_chart_data(engine,
                                                                    f,
                                                                    t,
                                                                    opts.server,
                                                                    user_id=user_id)
        except (UserNotFound, SQLAlchemyError) as ex:
            sys.stderr.write(str(ex) + '\n')
            return 1

        rendered_graph = self._render(data_collected, engine, opts, period_sec, times)
        self._output(opts, rendered_graph)
        return 0

    def _render(self, data_collected, engine, opts, period_sec, times):
        show_all = opts.show_all
        try:
            style = opts.style
        except AttributeError:
            style = None
        if style is None:
            style = 'admin'
        renderer = Renderer()
        renderer.set_period_sec(period_sec)
        lve_version = lveinfolib.get_lve_version(engine, opts.server)
        self.add_graphs(renderer, data_collected, times, lve_version, show_all,
                        is_user=style == 'user')
        rendered_graph = renderer.render()
        return rendered_graph

    def _output(self, opts, rendered_graph):
        if opts.format == 'svg' and opts.output == '-':
            # output as SVG
            try:
                root_node = etree.fromstring(rendered_graph.encode('utf8'))
                rendered_graph = etree.tostring(root_node, pretty_print=True)
            except (ImportError, ValueError, UnicodeEncodeError) as e:
                self.log.debug('Can not use pretty print for svg xml formatting; %s', str(e))
        if opts.format == 'png':
            # output as PNG
            cwd = '/var/lve/tmp' if os.getuid() == 0 else None
            with subprocess.Popen(
                [IMAGEMAGICK_BINARY, "-", "png:-"],
                stdin=subprocess.PIPE,
                stdout=subprocess.PIPE,
                cwd=cwd
            ) as proc:
                rendered_graph, _ = proc.communicate(input=rendered_graph.encode())
        if opts.output == '-':
            if isinstance(rendered_graph, bytes):
                sys.stdout.buffer.flush()  # needed to write directly to buffer
                sys.stdout.buffer.write(rendered_graph)
            else:
                sys.stdout.write(rendered_graph)
        else:
            try:
                if isinstance(rendered_graph, bytes):
                    write_mode = 'wb'
                else:
                    write_mode = 'w'
                with open(opts.output, write_mode) as output:
                    output.write(rendered_graph)
            except IOError:
                self.log.error("Unable to create file: %s", opts.output)

    def _obtain_user_id(self, engine, opts):
        # obtain user id
        try:
            user_id = opts.user_id
        except AttributeError:
            user_id = None
        try:
            user_name = opts.user_name
        except AttributeError:
            user_name = None
        if user_id is None:
            user_id = uidconverter.username_to_uid(
                username=user_name, local_server_id=self.cfg['server_id'],
                server_id=opts.server, db_engine=engine) or -1
        return user_id

    def _has_user_arguments(self, opts):
        return (hasattr(opts, 'user_id') and opts.user_id) or (hasattr(opts, 'user_name') and opts.user_name)