EVOLUTION-MANAGER

Edit File: fault_detector.py

# -*- coding: utf-8 -*-

# Copyright © Cloud Linux GmbH & Cloud Linux Software, Inc 2010-2021 All Rights Reserved
#
# Licensed under CLOUD LINUX LICENSE AGREEMENT
# http://cloudlinux.com/docs/LICENSE.TXT

"""
This module contains FaultDetector class, aimed to track throttling
for incoming requests
"""
import logging
import time
from datetime import datetime, timedelta
from threading import RLock
from typing import Tuple, Any

from xray.internal.constants import drop_after, check_period, throttling_threshold

class FaultDetector:
    """
    Fault Detector class
    """

def __init__(self):
        self.logger = logging.getLogger('fault_detector')
        self.throttling_stat = dict()
        self.lock = RLock()
        self.time_marker_file = '/usr/share/alt-php-xray/flush.latest'

def __call__(self, t_key: Any, cpu_value: int) -> Tuple[bool, int]:
        """
        Perform fault detection
        """
        with self.lock:
            try:
                saved_cpu, _ = self.throttling_stat[t_key]
            except KeyError:
                hitting_limits, throttled_time = False, 0
            else:
                hitting_limits, throttled_time = self._was_throttled_since(
                    saved_cpu, cpu_value)
                self._flush_entry(t_key)
        return hitting_limits, throttled_time

@property
    def timestamp(self) -> int:
        """
        Current Unix timestamp
        """
        return int(time.time())

@property
    def _drop_after(self) -> timedelta:
        """
        Drop after value as datetime.timedelta
        """
        return self.timedelta_in_minutes(drop_after)

@property
    def _check_period(self) -> timedelta:
        """
        Check period value as datetime.timedelta
        """
        return self.timedelta_in_minutes(check_period)

@property
    def should_flush(self) -> bool:
        """
        If throttling_stat should be flushed right now
        regarding check period value
        """
        try:
            with open(self.time_marker_file) as latest_info:
                latest_flush_time = int(latest_info.read().strip())
            return self.current_delta(latest_flush_time) > self._check_period
        except (OSError, ValueError):
            with self.lock:
                self._update_latest_flush()
            return False

@staticmethod
    def timedelta_in_minutes(mins: int) -> timedelta:
        """
        Get timedelta object for given number of minutes
        """
        return timedelta(minutes=mins)

def current_delta(self, time_stamp: int) -> timedelta:
        """
        Calculate timedelta for given time_stamp from current timestamp
        """

def _cast(ts: int) -> datetime:
            """
            Make a datetime object from ts (int)
            """
            return datetime.fromtimestamp(ts)

return _cast(self.timestamp) - _cast(time_stamp)

def save(self, key: Any, cpu: int) -> None:
        """
        Save throttling entry for given key
        """
        stamp = self.timestamp
        with self.lock:
            self.logger.debug('Adding new throttling entry %s: (%i, %i)',
                              key, cpu, stamp)
            self.throttling_stat[key] = (cpu, stamp)
            self.logger.debug('Throttling entries total %i',
                              len(self.throttling_stat))

def _flush_entry(self, key: Any) -> None:
        """
        Remove throttling entry by given key
        """
        with self.lock:
            self.logger.debug('Abandon throttling entry %s', key)
            self.throttling_stat.pop(key, 0)
            self.logger.debug('Throttling entries total %i',
                              len(self.throttling_stat))

def _was_throttled_since(self,
                             initial_value: int,
                             current_value: int) -> Tuple[bool, int]:
        """
        Check current throttling time of given lve_id vs given initial_value
        Return: tuple(fact of throttling, throttling time in ms)
        """
        # initial values of throttling time are in nanoseconds!
        spent_time = (current_value - initial_value) // 1000000  # cast to ms
        throttling_fact = spent_time > throttling_threshold
        self.logger.debug('Throttling checked with %i: %s, %i',
                          throttling_threshold,
                          throttling_fact, spent_time)
        return throttling_fact, spent_time if spent_time > 0 else 0

def flush(self) -> None:
        """
        Clear throttling stat dict
        """
        if self.should_flush:
            with self.lock:
                self.logger.debug('Flush started, total entries %i',
                                  len(self.throttling_stat))
                for key in self._expired_entries():
                    self._flush_entry(key)
                self._update_latest_flush()
                self.logger.debug('Flush finished, total entries %i',
                                  len(self.throttling_stat))

def _update_latest_flush(self) -> None:
        """
        Write current timestamp into time marker file
        """
        try:
            with open(self.time_marker_file, 'w') as latest_info:
                latest_info.write(str(self.timestamp))
        except OSError:
            self.logger.error('Failed to save timestamp of latest flush')

def _expired_entries(self) -> list:
        """
        Collect expired throttling stat entries keys
        regarding the drop after value
        """
        expired = list()
        for key, value in self.throttling_stat.items():
            _, timestamp = value
            _delta = self.current_delta(timestamp)
            if _delta > self._drop_after:
                self.logger.debug('Expired entry detected %i: %s elapsed',
                                  key, _delta)
                expired.append(key)
        return expired