340 lines
10 KiB
Python
340 lines
10 KiB
Python
# -*- coding: utf-8 -*-
|
|
# Copyright 2014 Objectif Libre
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
#
|
|
# @author: Stéphane Albert
|
|
#
|
|
"""
|
|
Time calculations functions
|
|
|
|
We're mostly using oslo_utils for time calculations but we're encapsulating it
|
|
to ease maintenance in case of library modifications.
|
|
"""
|
|
import calendar
|
|
import contextlib
|
|
import datetime
|
|
import decimal
|
|
import fractions
|
|
import shutil
|
|
import six
|
|
import sys
|
|
import tempfile
|
|
import yaml
|
|
|
|
from oslo_config import cfg
|
|
from oslo_log import log as logging
|
|
|
|
from oslo_utils import timeutils
|
|
from six import moves
|
|
from stevedore import extension
|
|
|
|
|
|
COLLECTORS_NAMESPACE = 'cloudkitty.collector.backends'
|
|
|
|
_ISO8601_TIME_FORMAT_SUBSECOND = '%Y-%m-%dT%H:%M:%S.%f'
|
|
_ISO8601_TIME_FORMAT = '%Y-%m-%dT%H:%M:%S'
|
|
|
|
LOG = logging.getLogger(__name__)
|
|
|
|
collect_opts = [
|
|
cfg.StrOpt('fetcher',
|
|
default='keystone',
|
|
deprecated_for_removal=True,
|
|
help='Project fetcher.'),
|
|
cfg.StrOpt('collector',
|
|
default='gnocchi',
|
|
deprecated_for_removal=True,
|
|
help='Data collector.'),
|
|
cfg.IntOpt('window',
|
|
default=1800,
|
|
deprecated_for_removal=True,
|
|
help='Number of samples to collect per call.'),
|
|
cfg.IntOpt('period',
|
|
default=3600,
|
|
deprecated_for_removal=True,
|
|
help='Rating period in seconds.'),
|
|
cfg.IntOpt('wait_periods',
|
|
default=2,
|
|
deprecated_for_removal=True,
|
|
help='Wait for N periods before collecting new data.'),
|
|
cfg.ListOpt('services',
|
|
default=[
|
|
'compute',
|
|
'volume',
|
|
'network.bw.in',
|
|
'network.bw.out',
|
|
'network.floating',
|
|
'image',
|
|
],
|
|
deprecated_for_removal=True,
|
|
help='Services to monitor.'),
|
|
cfg.StrOpt('metrics_conf',
|
|
default='/etc/cloudkitty/metrics.yml',
|
|
help='Metrology configuration file.'),
|
|
]
|
|
|
|
storage_opts = [
|
|
cfg.StrOpt('backend',
|
|
default='sqlalchemy',
|
|
help='Name of the storage backend driver.')
|
|
]
|
|
|
|
CONF = cfg.CONF
|
|
CONF.register_opts(collect_opts, 'collect')
|
|
CONF.register_opts(storage_opts, 'storage')
|
|
|
|
|
|
def isotime(at=None, subsecond=False):
|
|
"""Stringify time in ISO 8601 format."""
|
|
|
|
# Python provides a similar instance method for datetime.datetime objects
|
|
# called isoformat(). The format of the strings generated by isoformat()
|
|
# have a couple of problems:
|
|
# 1) The strings generated by isotime are used in tokens and other public
|
|
# APIs that we can't change without a deprecation period. The strings
|
|
# generated by isoformat are not the same format, so we can't just
|
|
# change to it.
|
|
# 2) The strings generated by isoformat do not include the microseconds if
|
|
# the value happens to be 0. This will likely show up as random failures
|
|
# as parsers may be written to always expect microseconds, and it will
|
|
# parse correctly most of the time.
|
|
|
|
if not at:
|
|
at = timeutils.utcnow()
|
|
st = at.strftime(_ISO8601_TIME_FORMAT
|
|
if not subsecond
|
|
else _ISO8601_TIME_FORMAT_SUBSECOND)
|
|
tz = at.tzinfo.tzname(None) if at.tzinfo else 'UTC'
|
|
st += ('Z' if tz == 'UTC' else tz)
|
|
return st
|
|
|
|
|
|
def iso8601_from_timestamp(timestamp, microsecond=False):
|
|
"""Returns an iso8601 formatted date from timestamp"""
|
|
|
|
# Python provides a similar instance method for datetime.datetime
|
|
# objects called isoformat() and utcfromtimestamp(). The format
|
|
# of the strings generated by isoformat() and utcfromtimestamp()
|
|
# have a couple of problems:
|
|
# 1) The method iso8601_from_timestamp in oslo_utils is realized
|
|
# by isotime, the strings generated by isotime are used in
|
|
# tokens and other public APIs that we can't change without a
|
|
# deprecation period. The strings generated by isoformat are
|
|
# not the same format, so we can't just change to it.
|
|
# 2) The strings generated by isoformat() and utcfromtimestamp()
|
|
# do not include the microseconds if the value happens to be 0.
|
|
# This will likely show up as random failures as parsers may be
|
|
# written to always expect microseconds, and it will parse
|
|
# correctly most of the time.
|
|
|
|
return isotime(datetime.datetime.utcfromtimestamp(timestamp), microsecond)
|
|
|
|
|
|
def dt2ts(orig_dt):
|
|
"""Translate a datetime into a timestamp."""
|
|
return calendar.timegm(orig_dt.timetuple())
|
|
|
|
|
|
def iso2dt(iso_date):
|
|
"""iso8601 format to datetime."""
|
|
iso_dt = timeutils.parse_isotime(iso_date)
|
|
trans_dt = timeutils.normalize_time(iso_dt)
|
|
return trans_dt
|
|
|
|
|
|
def ts2dt(timestamp):
|
|
"""timestamp to datetime format."""
|
|
if not isinstance(timestamp, float):
|
|
timestamp = float(timestamp)
|
|
return datetime.datetime.utcfromtimestamp(timestamp)
|
|
|
|
|
|
def ts2iso(timestamp):
|
|
"""timestamp to is8601 format."""
|
|
if not isinstance(timestamp, float):
|
|
timestamp = float(timestamp)
|
|
return iso8601_from_timestamp(timestamp)
|
|
|
|
|
|
def dt2iso(orig_dt):
|
|
"""datetime to is8601 format."""
|
|
return isotime(orig_dt)
|
|
|
|
|
|
def utcnow():
|
|
"""Returns a datetime for the current utc time."""
|
|
return timeutils.utcnow()
|
|
|
|
|
|
def utcnow_ts():
|
|
"""Returns a timestamp for the current utc time."""
|
|
return timeutils.utcnow_ts()
|
|
|
|
|
|
def get_month_days(dt):
|
|
return calendar.monthrange(dt.year, dt.month)[1]
|
|
|
|
|
|
def add_days(base_dt, days, stay_on_month=True):
|
|
if stay_on_month:
|
|
max_days = get_month_days(base_dt)
|
|
if days > max_days:
|
|
return get_month_end(base_dt)
|
|
return base_dt + datetime.timedelta(days=days)
|
|
|
|
|
|
def add_month(dt, stay_on_month=True):
|
|
next_month = get_next_month(dt)
|
|
return add_days(next_month, dt.day, stay_on_month)
|
|
|
|
|
|
def sub_month(dt, stay_on_month=True):
|
|
prev_month = get_last_month(dt)
|
|
return add_days(prev_month, dt.day, stay_on_month)
|
|
|
|
|
|
def get_month_start(dt=None):
|
|
if not dt:
|
|
dt = utcnow()
|
|
month_start = datetime.datetime(dt.year, dt.month, 1)
|
|
return month_start
|
|
|
|
|
|
def get_month_start_timestamp(dt=None):
|
|
return dt2ts(get_month_start(dt))
|
|
|
|
|
|
def get_month_end(dt=None):
|
|
month_start = get_month_start(dt)
|
|
days_of_month = get_month_days(month_start)
|
|
month_end = month_start.replace(day=days_of_month)
|
|
return month_end
|
|
|
|
|
|
def get_last_month(dt=None):
|
|
if not dt:
|
|
dt = utcnow()
|
|
month_end = get_month_start(dt) - datetime.timedelta(days=1)
|
|
return get_month_start(month_end)
|
|
|
|
|
|
def get_next_month(dt=None):
|
|
month_end = get_month_end(dt)
|
|
next_month = month_end + datetime.timedelta(days=1)
|
|
return next_month
|
|
|
|
|
|
def get_next_month_timestamp(dt=None):
|
|
return dt2ts(get_next_month(dt))
|
|
|
|
|
|
def refresh_stevedore(namespace=None):
|
|
"""Trigger reload of entry points.
|
|
|
|
Useful to have dynamic loading/unloading of stevedore modules.
|
|
"""
|
|
# NOTE(sheeprine): pkg_resources doesn't support reload on python3 due to
|
|
# defining basestring which is still there on reload hence executing
|
|
# python2 related code.
|
|
try:
|
|
del sys.modules['pkg_resources'].basestring
|
|
except AttributeError:
|
|
# python2, do nothing
|
|
pass
|
|
# Force working_set reload
|
|
moves.reload_module(sys.modules['pkg_resources'])
|
|
# Clear stevedore cache
|
|
cache = extension.ExtensionManager.ENTRY_POINT_CACHE
|
|
if namespace:
|
|
if namespace in cache:
|
|
del cache[namespace]
|
|
else:
|
|
cache.clear()
|
|
|
|
|
|
def check_time_state(timestamp=None, period=0, wait_time=0):
|
|
if not timestamp:
|
|
return get_month_start_timestamp()
|
|
|
|
now = utcnow_ts()
|
|
next_timestamp = timestamp + period
|
|
if next_timestamp + wait_time < now:
|
|
return next_timestamp
|
|
return 0
|
|
|
|
|
|
def get_metrics_conf(conf_path):
|
|
"""Return loaded yaml metrology configuration.
|
|
|
|
In case not found metrics.yml file,
|
|
return an empty dict.
|
|
"""
|
|
# NOTE(mc): We can not raise any exception in this function as it called
|
|
# at some file imports. Default values should be used instead. This is
|
|
# done for the docs and tests in gerrit which does not copy yaml conf file.
|
|
try:
|
|
with open(conf_path) as conf:
|
|
res = yaml.safe_load(conf)
|
|
res.update({'storage': CONF.storage.backend})
|
|
return res or {}
|
|
except Exception:
|
|
LOG.warning('Error when trying to retrieve yaml metrology conf file.')
|
|
return {}
|
|
|
|
|
|
@contextlib.contextmanager
|
|
def tempdir(**kwargs):
|
|
tmpdir = tempfile.mkdtemp(**kwargs)
|
|
try:
|
|
yield tmpdir
|
|
finally:
|
|
try:
|
|
shutil.rmtree(tmpdir)
|
|
except OSError as e:
|
|
LOG.debug('Could not remove tmpdir: %s',
|
|
six.text_type(e))
|
|
|
|
|
|
def num2decimal(num):
|
|
"""Converts a number into a decimal.Decimal.
|
|
|
|
The number may be an str in float, int or fraction format;
|
|
a fraction.Fraction, a decimal.Decimal, an int or a float.
|
|
"""
|
|
if isinstance(num, decimal.Decimal):
|
|
return num
|
|
if isinstance(num, str):
|
|
if '/' in num:
|
|
num = float(fractions.Fraction(num))
|
|
if isinstance(num, fractions.Fraction):
|
|
num = float(num)
|
|
return decimal.Decimal(num)
|
|
|
|
|
|
def convert_unit(value, factor=1, offset=0):
|
|
"""Return converted value depending on the provided factor and offset."""
|
|
return num2decimal(value) * num2decimal(factor) + num2decimal(offset)
|
|
|
|
|
|
def flat_dict(item, parent=None):
|
|
"""Returns a flat version of the nested dict item"""
|
|
if not parent:
|
|
parent = dict()
|
|
for k, val in item.items():
|
|
if isinstance(val, dict):
|
|
parent = flat_dict(val, parent)
|
|
else:
|
|
parent[k] = val
|
|
return parent
|