EVOLUTION-MANAGER

Edit File: db_governor_lib.py

# -*- coding: utf-8 -*-

# db_governer_lib.py - Library for MySQL governor config files manipulating for cloudlinux-config utility

#
# Copyright © Cloud Linux GmbH & Cloud Linux Software, Inc 2010-2019 All Rights Reserved
#
# Licensed under CLOUD LINUX LICENSE AGREEMENT
# http://cloudlinux.com/docs/LICENSE.TXT

import os
import subprocess
from typing import Optional  # NOQA

from lxml import etree

from clcommon.clexception import FormattedException
from clcommon.lib.mysql_governor_lib import MySQLGovernor

from .clconfig_utils import (
    boolean_to_on_off,
    str_to_boolean,
    time_convertor_to_dict,
    time_unit_to_letter,
)

class DBGovernorException(FormattedException):
    pass

DEFAULTS_DICT = {
    'modeOfOperation': 'off',
    'gatherDataForDetailedStats': 'on',
    'save_uid': 'off',
    'logRestrictedUsersQueries': 'off',
    'scriptPath': '',
    'userMaxConnections': 30,
    'restrictType': {
        'mode': 'limit',
        'unlimit': '60s'
    },
    'slowQueries': {
        'kill': 'off',
        'logPath': '',
        'timeout': 30
    },
    'restrictLog': {
        'logPath': '',
        'format': 'LONG'
    },
    'errorLog': {
        'logPath': '',
        'format': 'ERROR'
    }
}

DbGovConfig = '/etc/container/mysql-governor.xml'

# For unit tests
def _open(file_name):
    return open(file_name, encoding='utf-8')

def get_data_from_xml(x, node, attrib, default=None):
    """
    Gets data from specified node and attribute
    :param x: Parent node object
    :param node: Node name to extract data
    :param attrib: Attribute name to extract data
    :param default: Default value to return if specified node/attribute not found
    :return: Value
    """
    try:
        path = '//' + str(node) + '/@' + str(attrib)
        data = x.xpath(path)[0]
    except (AttributeError, IndexError):
        return default
    return data

def convert_time_value_to_seconds(value):
    """
    Convert value in minutes, hours, days to seconds
    :param value: value of slow limit
    :return: value in seconds
    """
    value = str(value).lower()
    if value.endswith('m'):
        result = int(value[:-1]) * 60
    elif value.endswith('h'):
        result = int(value[:-1]) * 60 * 60
    elif value.endswith('d'):
        result = int(value[:-1]) * 60 * 60 * 24
    elif value.endswith('s'):
        result = int(value[:-1])
    else:
        result = int(value)

return result

def get_slow_queries_timeout(x, default):
    """
    Gets slow queries timeout
    :param x: Root node
    :param default: Default value to return if timeout not found
    :return: Timeout
    """
    try:
        data = x.xpath("default/limit[@name='slow']/@current")[0]
        data = convert_time_value_to_seconds(data)
        return int(data)
    except (AttributeError, IndexError, ValueError):
        return default

def set_slow_queries_timeout(node, slow_timeout):
    """
    Sets slow queries timeout
    :param node: Root node
    :param slow_timeout: new timeout value
    :return: None
    """
    path = "default/limit[@name='slow']"
    tag_list = node.xpath(path)
    if len(tag_list) == 0:
        # Create default/limit chain
        node_list = node.xpath('default')
        if len(node_list) == 0:
            new_node = etree.Element('default')
            node.append(new_node)
        # 'default' node present, but 'limit' node with name="slow" attribute is absent
        default_node = node.xpath('default')[0]
        # Create 'limit' node
        tag_to_set = etree.Element('limit')
        default_node.append(tag_to_set)
        # set name attribute
        tag_to_set.set('name', 'slow')
    else:
        tag_to_set = tag_list[0]
    # Set attribute value
    tag_to_set.set('current', str(slow_timeout))

def set_data_to_xml(x, node_name, attribute_name, value):
    """
    Sets value to specified node/attribute
    :param x: Parent node
    :param node_name: Node name
    :param attribute_name: Attribute name
    :param value: Value to set
    :return: None
    """
    path = '//' + str(node_name)
    tag_list = x.xpath(path)
    if len(tag_list) == 0:
        # Tag absent, add it
        new_node = etree.Element(node_name)
        x.append(new_node)
        tag_list = x.xpath(path)
    tag = tag_list[0]
    # Set specified attribute
    tag.set(attribute_name, value)

def _get_timedict_from_xml(x, tag_name, attr_name):
    """
    Get time from specified tag/attribute as seconds
    :param x: XML object
    :param tag_name: XML tag name
    :param attr_name: XML object name
    :return: Number of seconds. Throws DBGovernorException if any error
    """
    value = get_data_from_xml(x, tag_name, attr_name)
    try:
        return time_convertor_to_dict(value)
    except Exception as e:
        raise DBGovernorException(
            f"{DbGovConfig} has invalid value in governer/{tag_name}/@{attr_name}"
        ) from e

def get_gov_mode_operation(parsed_xml_tree=None):
    # type: (etree.ElementBase) -> Optional[str]
    """
    Get MySQL Governor mode. We return None if governor config isn't exists
    All exceptions will be catched in top level functions
    :param parsed_xml_tree: parsed governor config
    """
    if not os.path.exists(DbGovConfig):
        return None
    if parsed_xml_tree is None:
        f = _open(DbGovConfig)
        x = etree.parse(f).getroot()
    else:
        x = parsed_xml_tree
    return get_data_from_xml(x, 'lve', 'use', DEFAULTS_DICT['modeOfOperation'])

def get_db_gov_config():
    """
    Get MySQL Governer config
    :return: Dictionary with config data
    """

try:
        if not os.path.exists(DbGovConfig):
            return {}

f = _open(DbGovConfig)

x = etree.parse(f).getroot()
        result_dict = {}
        result_dict['modeOfOperation'] = get_gov_mode_operation(parsed_xml_tree=x)
        result_dict['gatherDataForDetailedStats'] = str_to_boolean(
            get_data_from_xml(x, 'statistic', 'mode', DEFAULTS_DICT['gatherDataForDetailedStats']))
        result_dict['logRestrictedUsersQueries'] = str_to_boolean(
            get_data_from_xml(x, 'logqueries', 'use', DEFAULTS_DICT['logRestrictedUsersQueries']))
        result_dict['scriptPath'] = get_data_from_xml(x, 'restrict', 'script', DEFAULTS_DICT['scriptPath'])
        result_dict['userMaxConnections'] = int(
            get_data_from_xml(x, 'restrict', 'user_max_connections', DEFAULTS_DICT['userMaxConnections']))
        result_dict['restrictedTimePeriods'] = {
            'level1': _get_timedict_from_xml(x, 'restrict', 'level1'),
            'level2': _get_timedict_from_xml(x, 'restrict', 'level2'),
            'level3': _get_timedict_from_xml(x, 'restrict', 'level3'),
            'level4': _get_timedict_from_xml(x, 'restrict', 'level4'),
            'timeout': _get_timedict_from_xml(x, 'restrict', 'timeout')
        }
        result_dict['restrictType'] = {
            'mode': get_data_from_xml(x, 'restrict_mode', 'use', DEFAULTS_DICT['restrictType']['mode']),
            'unlimit': time_convertor_to_dict(
                get_data_from_xml(x, 'restrict_mode', 'unlimit', DEFAULTS_DICT['restrictType']['unlimit']))
        }
        result_dict['slowQueries'] = {'kill': str_to_boolean(get_data_from_xml(x, 'slow_queries', 'run',
                                                                               DEFAULTS_DICT['slowQueries']['kill'])),
                                      'logPath': get_data_from_xml(x, 'slow_queries', 'log',
                                                                   DEFAULTS_DICT['slowQueries']['logPath']),
                                      'timeout': get_slow_queries_timeout(x, DEFAULTS_DICT['slowQueries']['timeout'])
                                      }
        result_dict['restrictLog'] = {
            'logPath': get_data_from_xml(x, 'restrict', 'log', DEFAULTS_DICT['restrictLog']['logPath']),
            'format': get_data_from_xml(x, 'restrict', 'format', DEFAULTS_DICT['restrictLog']['format'])
        }
        result_dict['errorLog'] = {'logPath': get_data_from_xml(x, 'log', 'file', DEFAULTS_DICT['errorLog']['logPath']),
                                   'level': get_data_from_xml(x, 'log', 'mode', DEFAULTS_DICT['errorLog']['format'])
                                   }
        result_dict.update(get_db_gov_stat_info())
        return {'mySQLGovSettings': result_dict}
    except (OSError, IOError, etree.XMLSyntaxError, etree.XMLSchemaParseError,
            etree.XMLSchemaError, DBGovernorException) as e:
        raise DBGovernorException(f"{DbGovConfig} read/parse error: {e}") from e

def set_db_gov_config(parameters_dict):
    """
    Set MySQL governer config
    :param parameters_dict: Parameters to set dictionary. For example:
        {
         u'modeOfOperation': u'abusers',
         u'scriptPath': u'',
         u'gatherDataForDetailedStats': True,
         u'userMaxConnections': 30,
         u'logRestrictedUsersQueries': False,
         u'restrictLog': {u'logPath': u'/var/log/dbgovernor-restrict.log', u'format': u'LONG'},
         u'restrictedTimePeriods': {u'level1': {u'unitOfTime': u'seconds', u'period': 60},
                                    u'level2': {u'unitOfTime': u'minutes', u'period': 15},
                                    u'level3': {u'unitOfTime': u'hours', u'period': 1},
                                    u'level4': {u'unitOfTime': u'days', u'period': 1},
                                    u'timeout': {u'unitOfTime': u'hours', u'period': 1}},
         u'errorLog': {u'logPath': u'/var/log/dbgovernor-error-my.log', u'level': u'DEBUG'},
         u'slowQueries': {u'logPath': u'/var/log/dbgovernor-kill.log', u'kill': True, u'timeout': 30},
         u'restrictType': {u'mode': u'limit', u'unlimit': {u'unitOfTime': u'seconds', u'period': 60}}
         }
        Any key in dictionary can absent, appropriate MySQL Governor parameter will not changed
    :return: None. Throws DBGovernorException if error
    """
    try:
        # Do nothing if MySQL Governor config does not exist
        if not os.path.exists(DbGovConfig):
            return
        # Open config and parse it
        f = _open(DbGovConfig)
        x = etree.parse(f).getroot()
        f.close()
        ##################################
        # 1. Process single node/attribute
        # Control data list struct:
        #  (input_dictionary_keys, xml_tag_name, xml_attrubute_name)
        control_data_list = [
            ("['modeOfOperation']", 'lve', 'use', str),  # lve/@use
            ("['scriptPath']", 'restrict', 'script', str),  # restrict/@script
            ("['gatherDataForDetailedStats']", 'statistic', 'mode', boolean_to_on_off),  # statistic/@mode
            ("['userMaxConnections']", 'restrict', 'user_max_connections', str),  # restrict/@user_max_connections
            ("['logRestrictedUsersQueries']", 'logqueries', 'use', boolean_to_on_off),  # logqueries/@use
            ("['restrictLog']['logPath']", 'restrict', 'log', str),  # restrict/@log
            ("['restrictLog']['format']", 'restrict', 'format', str),  # restrict/@format
            ("['errorLog']['logPath']", 'log', 'file', str),  # log/@file
            ("['errorLog']['level']", 'log', 'mode', str),  # log/@mode
            ("['slowQueries']['logPath']", 'slow_queries', 'log', str),  # slow_queries/@log
            ("['slowQueries']['kill']", 'slow_queries', 'run', boolean_to_on_off),  # slow_queries/@run
            ("['restrictType']['mode']", 'restrict_mode', 'use', str)  # restrict_mode/@use
        ]
        for control_data_item in control_data_list:
            try:
                input_dict_keys, xml_tag_name, xml_attribute_name, func_converter = control_data_item
                # Get input dictionary value. Generates KeyError exception if key(s) not exist
                val = eval(f"parameters_dict{input_dict_keys}")
                set_data_to_xml(x, xml_tag_name, xml_attribute_name, func_converter(val))
            except KeyError:
                pass
        ##################################
        # 2. Process values pairs
        # Item structure: (period_indexes, period_unit_of_time, xml_tag_name, xml_attrubute_name)
        control_data_pairs_list = [
            ("['restrictType']['unlimit']['period']", "['restrictType']['unlimit']['unitOfTime']",
             'restrict_mode', 'unlimit'),  # restrict_mode/@unlimit
            ("['restrictedTimePeriods']['level1']['period']", "['restrictedTimePeriods']['level1']['unitOfTime']",
             'restrict', 'level1'),  # restrict/@level1
            ("['restrictedTimePeriods']['level2']['period']", "['restrictedTimePeriods']['level2']['unitOfTime']",
             'restrict', 'level2'),  # restrict_mode/@level2
            ("['restrictedTimePeriods']['level3']['period']", "['restrictedTimePeriods']['level3']['unitOfTime']",
             'restrict', 'level3'),  # restrict_mode/@level3
            ("['restrictedTimePeriods']['level4']['period']", "['restrictedTimePeriods']['level4']['unitOfTime']",
             'restrict', 'level4'),  # restrict_mode/@level4
            ("['restrictedTimePeriods']['timeout']['period']", "['restrictedTimePeriods']['timeout']['unitOfTime']",
             'restrict', 'timeout'),  # restrict_mode/@timeout
        ]
        for control_data_item in control_data_pairs_list:
            period_indexes, period_unit_of_time_indexes, xml_tag_name, xml_attribute_name = control_data_item
            try:
                # period_indexes, period_unit_of_time_indexes, xml_tag_name, xml_attribute_name = control_data_item
                # Get input dictionary value. Generates KeyError exception if key(s) not exist
                period_val = eval(f"parameters_dict{period_indexes}")
                period_unit_of_time = time_unit_to_letter(eval(f"parameters_dict{period_unit_of_time_indexes}"))
                set_data_to_xml(x, xml_tag_name, xml_attribute_name, f"{period_val}{period_unit_of_time}")
            except KeyError:
                pass
        ##################################
        # 3. Process special values
        # Write slow queries timeout
        try:
            slow_queries_timeout = parameters_dict['slowQueries']['timeout']
            set_slow_queries_timeout(x, slow_queries_timeout)
        except KeyError:
            pass
        ##################################
        # 4. Save result
        s_xml = etree.tostring(x)
        with open(DbGovConfig, 'wb') as f:
            f.write(b'<?xml version="1.0" ?>\n%s\n' % s_xml)
    except (OSError, IOError, etree.XMLSyntaxError, etree.XMLSchemaParseError, etree.XMLSchemaError) as e:
        raise DBGovernorException({'message': "%(dbgov_cfg)s error: " + str(e),
                                   'context': {'dbgov_cfg': DbGovConfig}}) from e

##################################
    # 5. Restart governer
    # /sbin/service db_governor restart
    subprocess.run('/sbin/service db_governor restart &>/dev/null &',
                   shell=True, executable='/bin/bash', check=False)

def get_db_gov_stat_info():
    """
    Get db_governor status and version info
    :return: dict(
                'dbGovVersion': <version>,
                'dbGovStatus': enabled|error|disabled,
                'dbGovError': error details
            )
    """
    db_gov = MySQLGovernor()
    status, error = db_gov.get_governor_status()
    return {
        'dbGovVersion': db_gov.get_governor_version(),
        'dbGovStatus': status,
        'dbGovError': str(error)
    }