Na}{

File Content: `cloudlinux_statistics.py`

# coding=utf-8
#
# Copyright © Cloud Linux GmbH & Cloud Linux Software, Inc 2010-2019 All Rights Reserved
#
# Licensed under CLOUD LINUX LICENSE AGREEMENT
# http://cloudlinux.com/docs/LICENSE.TXT
from __future__ import print_function
from __future__ import absolute_import
from __future__ import division
from builtins import range, map, filter
import logging
import os
from collections import defaultdict
import json
import math
from datetime import datetime
import sqlalchemy  # pylint: disable=unused-import
from typing import List, Optional, Union, Tuple, Dict  # pylint: disable=unused-import
from clcommon import cpapi
from clcommon.lib import MySQLGovernor, MySQLGovException, GovernorStatus

from lvestats.lib.commons.logsetup import setup_logging
import lvestats.lib.commons.decorators
from lvestats.lib.commons import dateutil, func
from lvestats.lib.commons.users_manager import g_usersManager
from lvestats.lib.commons.func import get_users_for_reseller, deserialize_lve_id, getuser
from lvestats.lib.lveinfolib import HistoryShowUnion
from lvestats.lib.lveinfolib_gov import HistoryShowDBGov
from lvestats.lib.uidconverter import uid_to_username, _username_to_uid_local, reseller_to_uid
from lvestats.lib.lve_list import LVEList
from lvestats.lib.parsers.cloudlinux_statistics_parser import setup_statistics_parser
from lvestats.lib.dbengine import make_db_engine
import lvestats.lib.config as config
from lvestats.lib.info.lveinfomain import main_
NOT_AVAILABLE = 'N/A'


class CloudLinuxStatistics(object):
    def __init__(
            self,
            engine,  # type: sqlalchemy.engine.base.Engine
            show_columns,  # type: List[str]
            order_by='',  # type: str
            server_id=None,  # type: Optional[str]
            by_usage=None,  # type: Optional[str]
            by_usage_percentage=0.9,  # type: float
            by_fault=None,  # type: Optional[str]
            threshold=1,  # type: int
            limit=0,  # type: int
            uid=None,  # type: Optional[Union[int, List[int], Tuple[int]]]
            time_unit=60  # type: int
    ):
        """
        Initializes statistics object used by SPA UI. Note, that if we set order_by to mysql_*, and set
        by_usage or by_fault, end result will will not be ordered by mysql_ column, using uid for ordering instead
        """
        self.order_by_lve = None
        self.order_by_gov = None
        self.server_id = server_id
        self.engine = engine
        self.limit_gov = 0
        self.limit_lve = 0
        self.uid = uid
        self.time_unit = time_unit
        self.is_many_uids = self.uid is None or isinstance(self.uid, (list, tuple))
        self._lve_results = []
        self._gov_results = []
        self._resellers_results = []

        if 'all' in show_columns:
            show_columns = ['cpu', 'ep', 'vmem', 'pmem', 'nproc', 'io', 'iops', 'nproc', 'mysql']
        self.show_columns = [s.lower() for s in show_columns]
        self.by_fault = self.normalize_by_fault(by_fault)
        self.order_by = order_by
        self.init_order_by(order_by, self.by_fault)
        self._show_columns_lve = []
        self._show_columns_gov = []
        self._gov_key_index = None
        self._lve_key_index = None
        self.init_show_columns()
        if self.order_by_gov:
            self.limit_gov = limit
        else:
            self.limit_lve = limit
        self.limit = limit
        self.by_usage_lve = None
        self.by_usage_gov = None
        self.init_by_usage(by_usage)
        self.by_usage_percentage = by_usage_percentage
        self.threshold = threshold

        self.mysql_governor = MySQLGovernor()
        self.log = logging.getLogger(self.__class__.__name__)
        self.show_mysql = self._show_mysql()
        self._admins_uids = self._get_admin_uids() if getuser() == 'root' else set()

    @staticmethod
    def _get_admin_uids():
        try:
            return set(_username_to_uid_local(admin) for admin in cpapi.admins()) - {None}
        except (cpapi.NotSupported, AttributeError):
            return set()

    def _show_mysql(self):
        # type: () -> bool
        return self.mysql_governor.is_governor_present() and 'mysql' in self.show_columns

    def _index_of(self, list_, key):
        # type: (List, Union[str,int]) -> Optional[int]
        if key in list_:
            return list_.index(key)

    def init_show_columns(self):
        show_columns_lve = []
        show_columns_gov = []
        show_columns_gov_singleuser = ['from', 'to']
        show_columns_lve_singleuser = ['id', 'from', 'to']
        columns = []
        for column in self.show_columns:
            if column != 'mysql':
                columns += ['a' + column, 'l' + column, column + 'f']
        if len(columns):
            show_columns_lve = ['id'] + columns
            show_columns_lve_singleuser += columns

        if 'mysql' in self.show_columns:
            show_columns_gov = ['id', 'cpu', 'lcpu', 'read', 'lread', 'write', 'lwrite']
            show_columns_gov_singleuser = show_columns_gov + ['from', 'to']

        self._show_columns_lve = show_columns_lve if self.is_many_uids else show_columns_lve_singleuser
        self._show_columns_gov = show_columns_gov if self.is_many_uids else show_columns_gov_singleuser
        if self.is_many_uids:
            self._gov_key_index = self._index_of(self._show_columns_gov, 'id')
            self._lve_key_index = self._index_of(self._show_columns_lve, 'id')
        else:
            self._gov_key_index = self._index_of(self._show_columns_gov, 'from')
            self._lve_key_index = self._index_of(self._show_columns_lve, 'from')

    def init_order_by(self, order_by, by_fault):
        # type: (str) -> None
        if order_by:
            order_by = order_by.lower()
            if order_by.endswith('faults'):
                self.order_by_lve = order_by.replace('_faults', '') + 'f'
            elif order_by.startswith('mysql'):
                self.order_by_gov = order_by[len('mysql_'):]
            else:
                self.order_by_lve = 'a' + order_by
        elif by_fault and by_fault[0] == 'any':
            self.order_by_lve = 'cpuf'
        else:
            self.order_by_lve = by_fault and by_fault[0]

    def get_username(self, uid):
        # type: (int) -> Tuple[str]
        username = uid_to_username(uid, self.server_id, self.server_id, self.engine)
        return username or NOT_AVAILABLE

    def _get_default_user_usage(self):
        """
        Return default info about user usage (mysql)
        :rtype: dict
        """
        usage_field = defaultdict(dict)
        for column in self.show_columns:
            if column != 'mysql':
                usage_field[column]['lve'] = 0.0
        if self.show_mysql:
            usage_field['cpu']['mysql'] = 0.0
            usage_field['io']['mysql'] = 0.0
        return usage_field

    def _get_default_user_limits(self, username):
        """
        Return default info about user limits (mysql)
        :rtype: dict
        """
        limit_field = defaultdict(dict)
        for column in self.show_columns:
            if column != 'mysql':
                limit_field[column]['lve'] = 0.0
        if self.show_mysql:
            cpu_limit, io_limit = self.get_user_limits_safe(username)
            limit_field['cpu']['mysql'] = cpu_limit
            limit_field['io']['mysql'] = io_limit
        return limit_field

    def _get_default_user_faults(self):
        fault_field = {}
        for column in self.show_columns:
            if column != 'mysql':
                fault_field[column] = {'lve': 0}
        return fault_field

    def get_item(self, ikey):
        if self.is_many_uids:
            _, is_reseller = deserialize_lve_id(ikey)
            username = self.get_username(ikey)
        else:
            _, is_reseller = deserialize_lve_id(self.uid)
            username = self.get_username(self.uid)
        item = dict(
            usage=self._get_default_user_usage(),
            limits=self._get_default_user_limits(username))
        if self.show_columns != ['mysql']:
            item['faults'] = self._get_default_user_faults()
        if self.is_many_uids:
            if is_reseller:
                item.update(dict(id=ikey, name=username))
            else:
                item.update(
                    dict(id=ikey, username=username,
                         domain=g_usersManager.get_domain(username, raise_exc=False) or NOT_AVAILABLE,
                         reseller=g_usersManager.get_reseller(username, raise_exc=False) or NOT_AVAILABLE))
        else:
            item.update({
                'from': int(ikey),
                'to': int(ikey) + self.time_unit})
        return item

    def fill_lve_item(self, item, row):
        if row is None:
            return item
        count = 1 if self.is_many_uids else 1 + 2
        for column in self.show_columns:
            if column == 'mysql':
                continue
            item['usage'][column]['lve'] = row[count]
            item['limits'][column]['lve'] = row[count + 1]
            item['faults'][column]['lve'] = row[count + 2]
            count += 3
        return item

    @staticmethod
    def __get_io_bytes(read, write):
        """
        Get io value by read and write values.
        :param int|float|None read: read speed in MEGAbytes
        :param int|float|None write: write speed in MEGAbytes
        :return int: io, BYTES
        """
        # convert None to zero
        read = read or 0
        write = write or 0

        # calculate io in bytes
        io = (read + write) * 1024 * 1024
        return int(io)

    def fill_gov_item(self, item, row):
        if row is None:
            return item
        item['usage']['cpu']['mysql'] = round(float(row[1] or 0), 1)  # pylint: disable=round-builtin
        item['limits']['cpu']['mysql'] = round(float(row[2] or 0), 1) # pylint: disable=round-builtin
        item['usage']['io']['mysql'] = self.__get_io_bytes(row[3], row[5])
        item['limits']['io']['mysql'] = self.__get_io_bytes(row[4], row[6])
        return item

    def process_data(self, lve_results, gov_results):
        from_gov_to_lve_indexes, from_lve_to_gov_indexes,\
            gov_keys, gov_results,\
            lve_keys, lve_results,\
            not_in_gov_indexes, not_in_lve_indexes = self.process_indexes(gov_results, lve_results)
        if not self.show_mysql:
            lve_results = list(lve_results)
        elif self.order_by_gov:
            lve_results = self.sort_by_another_list(from_gov_to_lve_indexes, lve_results, not_in_gov_indexes)
        elif self.order_by_lve:
            gov_results = self.sort_by_another_list(from_lve_to_gov_indexes, gov_results, not_in_lve_indexes)
        else:
            gov_results, lve_results = self.sort_by_key(gov_keys, gov_results, lve_keys, lve_results)
        lendiff = (len(lve_results) - len(gov_results))
        gov_results += [None] * lendiff
        lve_results += [None] * -lendiff
        self._lve_results = lve_results
        self._gov_results = gov_results

    @staticmethod
    def sort_by_key(gov_keys, gov_results, lve_keys, lve_results):
        # type: (List[int], List, List[int], List) -> Tuple[List, List]
        lve_pos = 0
        gov_pos = 0
        _lve_results = []
        _gov_results = []
        for _ in range(min(len(lve_results), len(gov_results))):
            if lve_keys[lve_pos] == gov_keys[gov_pos]:
                _lve_results.append(lve_results[lve_pos])
                _gov_results.append(gov_results[gov_pos])
                lve_pos += 1
                gov_pos += 1
            elif lve_keys[lve_pos] > gov_keys[gov_pos]:
                _lve_results.append(None)
                _gov_results.append(gov_results[gov_pos])
                gov_pos += 1
            else:
                _gov_results.append(None)
                _lve_results.append(lve_results[lve_pos])
                lve_pos += 1
        gov_results = _gov_results + gov_results[gov_pos:]
        lve_results = _lve_results + lve_results[lve_pos:]
        return gov_results, lve_results

    @staticmethod
    def sort_by_another_list(indexes, results, not_in_indexes):
        # type: (List[int], List, List[int]) -> List
        _results = []
        for index in indexes:
            if index is None:
                _results.append(None)
            else:
                _results.append(results[index])
        for index in not_in_indexes:
            _results.append(results[index])
        results = _results
        return results

    def process_indexes(self, gov_results, lve_results):
        # type: (List, List) -> Tuple[List[int],List[int],List,List,List,List,List,List]
        if self.by_usage_gov:
            gov_uids = [_[0] for _ in gov_results]
            lve_results = [row for row in lve_results if row[0] in gov_uids]
        if self.by_usage_lve or self.by_fault:
            lve_uids = [_[0] for _ in lve_results]
            gov_results = [row for row in gov_results if row[0] in lve_uids]
        gov_keys = [_[self._gov_key_index] for _ in gov_results]
        lve_keys = [_[self._lve_key_index] for _ in lve_results]
        from_gov_to_lve_indexes = self.get_indexes(gov_keys, lve_keys)
        not_in_lve_indexes = [i for i, _ in enumerate(from_gov_to_lve_indexes) if _ is None]
        from_lve_to_gov_indexes = self.get_indexes(lve_keys, gov_keys)
        not_in_gov_indexes = [i for i, _ in enumerate(from_lve_to_gov_indexes) if _ is None]
        return from_gov_to_lve_indexes, from_lve_to_gov_indexes,\
            gov_keys, gov_results,\
            lve_keys, lve_results,\
            not_in_gov_indexes, not_in_lve_indexes

    def get_indexes(self, first_uids, second_uids):
        # type: (List[int], List[int]) -> List[int]
        return [self._index_of(second_uids, uid) for uid in first_uids]

    def get_value(self, key, resellers):
        # type: (int, bool) -> Dict[str, Union[int,float]]
        """
        :type resellers: bool
        :type key: int
        :rtype: dict
        """
        if resellers:
            item = self.get_item(self._resellers_results[key][self._lve_key_index])
            return self.fill_lve_item(item, self._resellers_results[key])
        elif not self.limit or key < self.limit:
            if not self.show_mysql:
                item = self.get_item(self._lve_results[key][self._lve_key_index])
                return self.fill_lve_item(item, self._lve_results[key])
            elif self.order_by_gov or self.limit_gov:
                row = self._gov_results[key]
                if row is not None:
                    item = self.get_item(row[self._gov_key_index])
                    self.fill_gov_item(item, row)
                else:
                    item = self.get_item(self._lve_results[key][self._lve_key_index])
                return self.fill_lve_item(item, self._lve_results[key])
            else:
                row = self._lve_results[key]
                if row is not None:
                    item = self.get_item(row[self._lve_key_index])
                    self.fill_lve_item(item, row)
                else:
                    item = self.get_item(self._gov_results[key][self._gov_key_index])
                return self.fill_gov_item(item, self._gov_results[key])
        else:
            raise IndexError

    def get_gov_row(self, key, indexes, result):
        if key > len(indexes) + 1 or indexes[key] is None:
            return None
        return result[indexes[key]]

    def get_user_limits_safe(self, username, ignore_watched=False):
        # type: (str, bool) -> Tuple[int,int]
        """
        Get cpu and io limit for user
        :return tuple: cpu limit in %, io limit in bytes/s
        """
        cpu_limit, io_limit = 0, 0
        if self.mysql_governor.is_governor_present():
            try:
                if ignore_watched or self.mysql_governor.get_governor_status_by_username(username) == 'watched':
                    cpu_limit, io_limit = self.mysql_governor.get_limits_by_user(username)
            except MySQLGovException as e:
                self.log.info('Cannot get mysql limits for user %s. Error raised: %s', username,
                              e.message % e.context)  # pylint: disable=exception-message-attribute
        return cpu_limit, io_limit * 1024

    @staticmethod
    def normalize_by_fault(by_fault):
        # type: (Optional[str]) -> Optional[Tuple[str]]
        if by_fault:
            if by_fault == 'mem':
                by_fault = 'vmem'
            return by_fault + 'f',  # it is a tuple, so we need coma at the end

    def init_by_usage(self, by_usage):
        # type: (Optional[str]) -> None
        if by_usage:
            if by_usage.startswith("mysql"):
                self.by_usage_gov = by_usage[len('mysql_'):]
            else:
                self.by_usage_lve = 'a' + by_usage,  # it is a tuple, so we need coma at the end

    def generate_statistics(self, period_from, period_to):
        # type: (datetime, datetime) -> None
        """Generate statistics."""
        period_from_gm = dateutil.local_to_gm(period_from)
        period_to_gm = dateutil.local_to_gm(period_to)

        if self._show_columns_lve:
            history_show = HistoryShowUnion(
                dbengine=self.engine,
                period_from=period_from_gm, period_to=period_to_gm,
                uid=self.uid,
                time_unit=self.time_unit,
                show_columns=self._show_columns_lve,
                server_id=self.server_id,
                order_by=self.order_by_lve,
                by_usage=self.by_usage_lve,
                by_usage_percentage=self.by_usage_percentage,
                by_fault=self.by_fault,
                threshold=self.threshold,
                limit=self.limit_lve)
            history_show.set_normalised_output()
            lve_results = history_show.proceed()
        else:
            lve_results = list()

        if self.show_mysql:
            gov_results = HistoryShowDBGov(
                dbengine=self.engine,
                period_from=period_from_gm,
                period_to=period_to_gm,
                time_unit=self.time_unit,
                server_id=self.server_id,
                show_columns=self._show_columns_gov,
                order_by=self.order_by_gov,
                by_usage=self.by_usage_gov,
                by_usage_percentage=self.by_usage_percentage,
                limit=self.limit_gov,
                reverse=True,
                uid=self.uid,
            ).history_dbgov_show()
        else:
            gov_results = list()
        if self.is_many_uids:
            users = []
            for item in lve_results:
                uid, is_reseller = deserialize_lve_id(item[self._lve_key_index])
                if uid in self._admins_uids:
                    continue
                elif is_reseller:
                    self._resellers_results.append(item)
                else:
                    users.append(item)
            lve_results = users
        self.process_data(lve_results, gov_results)


def _get_uid_for_select(for_reseller, uid):
    # type: (str, int) -> Union[List[int], int]
    """
    Get uid that should be selected from history table;
    """

    if getuser() != 'root':
        return os.getuid()

    g_usersManager.build_users_cache(for_reseller)
    if for_reseller:
        users_list = get_users_for_reseller(for_reseller or getuser())
        id_ = list(filter(bool, list(map(_username_to_uid_local, users_list))))
        reseller_uid = reseller_to_uid(for_reseller or getuser())
        # LVESTATS-97
        # If uid not in users_list, we return empty list
        if uid:
            return uid if uid in id_ or uid == reseller_uid else []
        # select also information about reseller's containers (if exists)
        if reseller_uid is not None:
            id_.append(reseller_uid)
    else:
        id_ = uid
    return id_


def execute(engine, options, log=None):
    # type: (sqlalchemy.engine.base.Engine, object, Optional[logging.Logger]) -> Tuple[str, int]
    """
    Get command response and exitcode.
    :type log: None | logging.Logger
    :type engine: sqlalchemy.engine.base.Engine
    :rtype: (str, int)
    """
    if not options.json:
        return "Only JSON mode supported for now", -1

    id_ = _get_uid_for_select(options.for_reseller, options.id)
    statistics = get_statistics(engine, id_, log, options)
    json_data = dump_json(statistics)

    return json_data, 0


def dump_json(statistics):
    # type: (Dict) -> str
    json_data = json.dumps(statistics)
    return json_data


def get_statistics(
        engine,  # type: sqlalchemy.engine.base.Engine
        id_,  # type: Optional[Union[int, List[int], Tuple[int]]]
        log,  # type: Optional[logging.Logger]
        options,
        timestamp=dateutil.gm_datetime_to_unixtimestamp()  # type: int
        ):
    # type: (...) -> Dict[str, Union[LVEList, str, int]]
    result = dict(
        result='success',
        timestamp=timestamp)
    # LVESTATS-97
    # If both params (for_reseller and id) are used
    # and id_ list is empty, we return `Permission denied` message
    if options.for_reseller is not None and options.id is not None and id_ == []:
        error_msg = 'Permission denied. User with id ' \
                    '{} does not belong reseller `{}`'.format(options.id,
                                                              options.for_reseller)
        result['result'] = error_msg
        return result
    try:
        stat = CloudLinuxStatistics(
            engine,
            show_columns=options.show,
            order_by=options.order_by,
            server_id=options.server_id,
            by_usage=options.by_usage,
            by_usage_percentage=options.percentage / 100.,
            by_fault=options.by_fault,
            threshold=options.threshold,
            limit=options.limit,
            uid=id_,
            time_unit=options.time_unit)
        stat.generate_statistics(getattr(options, 'from'), options.to)

        lvelist = LVEList(stat)
        lvelist_resellers = LVEList(stat, resellers=True)
        if stat.is_many_uids:
            result['users'] = lvelist
            result['resellers'] = lvelist_resellers
        else:
            result['user'] = lvelist
    except Exception as e:
        result["result"] = str(e)
        if log:
            log.error(str(e))

    # add mysql status info
    status, error = func.get_governor_status()
    result['mySqlGov'] = status
    if status == GovernorStatus.ERROR:
        result['warning'] = error.message  # pylint: disable=exception-message-attribute
        result['context'] = error.context
    return result


@lvestats.lib.commons.decorators.no_sigpipe
def main(engine, argv_, server_id='localhost', log=None):
    # type: (sqlalchemy.engine.base.Engine, List[str], str, Optional[logging.Logger]) -> int
    """
    Execute command and return exitcode
    """
    if log is None:
        log = setup_logging({}, caller_name="CloudLinuxStatistics", file_level=logging.WARNING, console_level=logging.FATAL)
    options = setup_statistics_parser(argv_, server_id)

    json_str, exit_code = execute(engine, options, log=log)
    print(json_str)
    return exit_code


def get_users_and_resellers_with_faults(period='1d'):
    """
    Return amount of users and resellers with faults
    """
    cnf = config.read_config()
    dbengine = make_db_engine(cnf)
    server_id = cnf.get('server_id', 'localhost')
    opt_list = ['--by-fault', 'any', '--json', '--period=' + period]
    options = setup_statistics_parser(opt_list, server_id)
    json_str, _ = execute(dbengine, options, log=None)
    json_data = json.loads(json_str)
    resellers = json_data['resellers']
    users = json_data['users']
    return len(users), len(resellers)


def get_max_memory_for_lve(lve_id, period_seconds):
    data = main_(config.read_config(), argv_ = ['--json', '--id', str(lve_id), '--show-columns',
                                                'mPMem', '--period', str(int(math.ceil(period_seconds / 60))) + 'm'])
    if data:
        parsed_data = json.loads(data)
    try:
        maxPmem = max([x['mPMem'] for x in parsed_data['data']])
    except (ValueError, KeyError):
        maxPmem = None
    return maxPmem
Name	Size	Permission
__pycache__	---	0755
chart	---	0755
commons	---	0755
info	---	0755
parsers	---	0755
__init__.py	219 bytes	0644
cloudlinux_statistics.py	24401 bytes	0644
cloudlinux_statsnotifier.py	5334 bytes	0644
config.py	3011 bytes	0644
db_functions.py	1641 bytes	0644
dbengine.py	9508 bytes	0644
jsonhandler.py	482 bytes	0644
lve_create_db.py	1964 bytes	0644
lve_list.py	3467 bytes	0644
lveinfolib.py	40905 bytes	0644
lveinfolib_gov.py	13912 bytes	0644
lvestats_server.py	8296 bytes	0644
notifications_helper.py	5581 bytes	0644
server_id.py	5519 bytes	0644
snapshot.py	7572 bytes	0644
uidconverter.py	5229 bytes	0644
ustate.py	8860 bytes	0644
File Content: cloudlinux_statistics.py

File Content: `cloudlinux_statistics.py`
