311 lines
9.2 KiB
Python
311 lines
9.2 KiB
Python
# Copyright 2014 Objectif Libre
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
#
|
|
import calendar
|
|
import contextlib
|
|
import datetime
|
|
import decimal
|
|
import fractions
|
|
import importlib
|
|
import math
|
|
import shutil
|
|
from string import Template
|
|
import sys
|
|
import tempfile
|
|
import yaml
|
|
|
|
from oslo_log import log as logging
|
|
from oslo_utils import timeutils
|
|
from stevedore import extension
|
|
|
|
from cloudkitty.utils import tz as tzutils
|
|
|
|
|
|
_ISO8601_TIME_FORMAT_SUBSECOND = '%Y-%m-%dT%H:%M:%S.%f'
|
|
_ISO8601_TIME_FORMAT = '%Y-%m-%dT%H:%M:%S'
|
|
|
|
LOG = logging.getLogger(__name__)
|
|
|
|
|
|
def isotime(at=None, subsecond=False):
|
|
"""Stringify time in ISO 8601 format."""
|
|
|
|
# Python provides a similar instance method for datetime.datetime objects
|
|
# called isoformat(). The format of the strings generated by isoformat()
|
|
# have a couple of problems:
|
|
# 1) The strings generated by isotime are used in tokens and other public
|
|
# APIs that we can't change without a deprecation period. The strings
|
|
# generated by isoformat are not the same format, so we can't just
|
|
# change to it.
|
|
# 2) The strings generated by isoformat do not include the microseconds if
|
|
# the value happens to be 0. This will likely show up as random failures
|
|
# as parsers may be written to always expect microseconds, and it will
|
|
# parse correctly most of the time.
|
|
|
|
if not at:
|
|
at = timeutils.utcnow()
|
|
st = at.strftime(_ISO8601_TIME_FORMAT
|
|
if not subsecond
|
|
else _ISO8601_TIME_FORMAT_SUBSECOND)
|
|
tz = at.tzinfo.tzname(None) if at.tzinfo else 'UTC'
|
|
st += ('Z' if tz == 'UTC' else tz)
|
|
return st
|
|
|
|
|
|
def iso8601_from_timestamp(timestamp, microsecond=False):
|
|
"""Returns an iso8601 formatted date from timestamp"""
|
|
|
|
# Python provides a similar instance method for datetime.datetime
|
|
# objects called isoformat() and utcfromtimestamp(). The format
|
|
# of the strings generated by isoformat() and utcfromtimestamp()
|
|
# have a couple of problems:
|
|
# 1) The method iso8601_from_timestamp in oslo_utils is realized
|
|
# by isotime, the strings generated by isotime are used in
|
|
# tokens and other public APIs that we can't change without a
|
|
# deprecation period. The strings generated by isoformat are
|
|
# not the same format, so we can't just change to it.
|
|
# 2) The strings generated by isoformat() and utcfromtimestamp()
|
|
# do not include the microseconds if the value happens to be 0.
|
|
# This will likely show up as random failures as parsers may be
|
|
# written to always expect microseconds, and it will parse
|
|
# correctly most of the time.
|
|
|
|
return isotime(datetime.datetime.utcfromtimestamp(timestamp), microsecond)
|
|
|
|
|
|
def dt2ts(orig_dt):
|
|
"""Translate a datetime into a timestamp."""
|
|
return calendar.timegm(orig_dt.timetuple())
|
|
|
|
|
|
def iso2dt(iso_date):
|
|
"""iso8601 format to datetime."""
|
|
iso_dt = timeutils.parse_isotime(iso_date)
|
|
trans_dt = timeutils.normalize_time(iso_dt)
|
|
return trans_dt
|
|
|
|
|
|
def ts2dt(timestamp):
|
|
"""timestamp to datetime format."""
|
|
if not isinstance(timestamp, float):
|
|
timestamp = float(timestamp)
|
|
return datetime.datetime.utcfromtimestamp(timestamp)
|
|
|
|
|
|
def ts2iso(timestamp):
|
|
"""timestamp to is8601 format."""
|
|
if not isinstance(timestamp, float):
|
|
timestamp = float(timestamp)
|
|
return iso8601_from_timestamp(timestamp)
|
|
|
|
|
|
def dt2iso(orig_dt):
|
|
"""datetime to is8601 format."""
|
|
return isotime(orig_dt)
|
|
|
|
|
|
def utcnow():
|
|
"""Returns a datetime for the current utc time."""
|
|
return timeutils.utcnow()
|
|
|
|
|
|
def get_month_days(dt):
|
|
return calendar.monthrange(dt.year, dt.month)[1]
|
|
|
|
|
|
def add_days(base_dt, days, stay_on_month=True):
|
|
if stay_on_month:
|
|
max_days = get_month_days(base_dt)
|
|
if days > max_days:
|
|
return get_month_end(base_dt)
|
|
return base_dt + datetime.timedelta(days=days)
|
|
|
|
|
|
def add_month(dt, stay_on_month=True):
|
|
next_month = get_next_month(dt)
|
|
return add_days(next_month, dt.day, stay_on_month)
|
|
|
|
|
|
def sub_month(dt, stay_on_month=True):
|
|
prev_month = get_last_month(dt)
|
|
return add_days(prev_month, dt.day, stay_on_month)
|
|
|
|
|
|
def get_month_start(dt=None):
|
|
if not dt:
|
|
dt = utcnow()
|
|
month_start = datetime.datetime(dt.year, dt.month, 1)
|
|
return month_start
|
|
|
|
|
|
def get_month_start_timestamp(dt=None):
|
|
return dt2ts(get_month_start(dt))
|
|
|
|
|
|
def get_month_end(dt=None):
|
|
month_start = get_month_start(dt)
|
|
days_of_month = get_month_days(month_start)
|
|
month_end = month_start.replace(day=days_of_month)
|
|
return month_end
|
|
|
|
|
|
def get_last_month(dt=None):
|
|
if not dt:
|
|
dt = utcnow()
|
|
month_end = get_month_start(dt) - datetime.timedelta(days=1)
|
|
return get_month_start(month_end)
|
|
|
|
|
|
def get_next_month(dt=None):
|
|
month_end = get_month_end(dt)
|
|
next_month = month_end + datetime.timedelta(days=1)
|
|
return next_month
|
|
|
|
|
|
def get_next_month_timestamp(dt=None):
|
|
return dt2ts(get_next_month(dt))
|
|
|
|
|
|
def refresh_stevedore(namespace=None):
|
|
"""Trigger reload of entry points.
|
|
|
|
Useful to have dynamic loading/unloading of stevedore modules.
|
|
"""
|
|
# NOTE(sheeprine): pkg_resources doesn't support reload on python3 due to
|
|
# defining basestring which is still there on reload hence executing
|
|
# python2 related code.
|
|
try:
|
|
del sys.modules['pkg_resources'].basestring
|
|
except AttributeError:
|
|
# python2, do nothing
|
|
pass
|
|
# Force working_set reload
|
|
importlib.reload(sys.modules['pkg_resources'])
|
|
# Clear stevedore cache
|
|
cache = extension.ExtensionManager.ENTRY_POINT_CACHE
|
|
if namespace:
|
|
if namespace in cache:
|
|
del cache[namespace]
|
|
else:
|
|
cache.clear()
|
|
|
|
|
|
def check_time_state(timestamp=None, period=0, wait_periods=0):
|
|
"""Checks the state of a timestamp compared to the current time.
|
|
|
|
Returns the next timestamp based on the current timestamp and the period if
|
|
the next timestamp is inferior to the current time and the waiting period
|
|
or None if not.
|
|
|
|
:param timestamp: Current timestamp
|
|
:type timestamp: datetime.datetime
|
|
:param period: Period, in seconds
|
|
:type period: int
|
|
:param wait_periods: periods to wait before the current timestamp.
|
|
:type wait_periods: int
|
|
:rtype: datetime.datetime
|
|
"""
|
|
if not timestamp:
|
|
return tzutils.get_month_start()
|
|
|
|
period_delta = datetime.timedelta(seconds=period)
|
|
next_timestamp = tzutils.add_delta(timestamp, period_delta)
|
|
wait_time = wait_periods * period_delta
|
|
if tzutils.add_delta(next_timestamp, wait_time) < tzutils.localized_now():
|
|
return next_timestamp
|
|
return None
|
|
|
|
|
|
def load_conf(conf_path):
|
|
"""Loads the metric collection configuration.
|
|
|
|
:param conf_path: Path of the file to load
|
|
:type conf_path: str
|
|
:rtype: dict
|
|
"""
|
|
with open(conf_path) as conf:
|
|
res = yaml.safe_load(conf)
|
|
return res or {}
|
|
|
|
|
|
@contextlib.contextmanager
|
|
def tempdir(**kwargs):
|
|
tmpdir = tempfile.mkdtemp(**kwargs)
|
|
try:
|
|
yield tmpdir
|
|
finally:
|
|
try:
|
|
shutil.rmtree(tmpdir)
|
|
except OSError as e:
|
|
LOG.debug('Could not remove tmpdir: %s', e)
|
|
|
|
|
|
def mutate(value, mode='NONE'):
|
|
"""Mutate value according provided mode."""
|
|
|
|
if mode == 'NUMBOOL':
|
|
return float(value != 0.0)
|
|
|
|
if mode == 'FLOOR':
|
|
return math.floor(value)
|
|
|
|
if mode == 'CEIL':
|
|
return math.ceil(value)
|
|
|
|
return value
|
|
|
|
|
|
def num2decimal(num):
|
|
"""Converts a number into a decimal.Decimal.
|
|
|
|
The number may be an str in float, int or fraction format;
|
|
a fraction.Fraction, a decimal.Decimal, an int or a float.
|
|
"""
|
|
if isinstance(num, decimal.Decimal):
|
|
return num
|
|
if isinstance(num, str):
|
|
if '/' in num:
|
|
num = float(fractions.Fraction(num))
|
|
if isinstance(num, fractions.Fraction):
|
|
num = float(num)
|
|
return decimal.Decimal(num)
|
|
|
|
|
|
def convert_unit(value, factor, offset):
|
|
"""Return converted value depending on the provided factor and offset."""
|
|
return num2decimal(value) * num2decimal(factor) + num2decimal(offset)
|
|
|
|
|
|
def flat_dict(item, parent=None):
|
|
"""Returns a flat version of the nested dict item"""
|
|
if not parent:
|
|
parent = dict()
|
|
for k, val in item.items():
|
|
if isinstance(val, dict):
|
|
parent = flat_dict(val, parent)
|
|
else:
|
|
parent[k] = val
|
|
return parent
|
|
|
|
|
|
def template_str_substitute(string, replace_map):
|
|
"""Returns a string with subtituted patterns."""
|
|
try:
|
|
tmp = Template(string)
|
|
return tmp.substitute(replace_map)
|
|
except (KeyError, ValueError) as e:
|
|
LOG.error("Error when trying to substitute the string placeholders. \
|
|
Please, check your metrics configuration.", e)
|
|
raise
|