Merge "Added support for an hybrid gnocchi storage"
This commit is contained in:
commit
56cdfe9a9e
cloudkitty
collector
orchestrator.pystorage
tests
transformer
@ -19,7 +19,9 @@ import abc
|
|||||||
|
|
||||||
from oslo_config import cfg
|
from oslo_config import cfg
|
||||||
import six
|
import six
|
||||||
|
from stevedore import driver
|
||||||
|
|
||||||
|
from cloudkitty import transformer
|
||||||
import cloudkitty.utils as ck_utils
|
import cloudkitty.utils as ck_utils
|
||||||
|
|
||||||
collect_opts = [
|
collect_opts = [
|
||||||
@ -44,7 +46,24 @@ collect_opts = [
|
|||||||
'network.floating'],
|
'network.floating'],
|
||||||
help='Services to monitor.'), ]
|
help='Services to monitor.'), ]
|
||||||
|
|
||||||
cfg.CONF.register_opts(collect_opts, 'collect')
|
CONF = cfg.CONF
|
||||||
|
CONF.register_opts(collect_opts, 'collect')
|
||||||
|
|
||||||
|
COLLECTORS_NAMESPACE = 'cloudkitty.collector.backends'
|
||||||
|
|
||||||
|
|
||||||
|
def get_collector(transformers=None):
|
||||||
|
if not transformers:
|
||||||
|
transformers = transformer.get_transformers()
|
||||||
|
collector_args = {
|
||||||
|
'period': CONF.collect.period,
|
||||||
|
'transformers': transformers}
|
||||||
|
collector = driver.DriverManager(
|
||||||
|
COLLECTORS_NAMESPACE,
|
||||||
|
CONF.collect.collector,
|
||||||
|
invoke_on_load=True,
|
||||||
|
invoke_kwds=collector_args).driver
|
||||||
|
return collector
|
||||||
|
|
||||||
|
|
||||||
class TransformerDependencyError(Exception):
|
class TransformerDependencyError(Exception):
|
||||||
|
@ -26,13 +26,14 @@ from oslo_config import cfg
|
|||||||
from oslo_log import log as logging
|
from oslo_log import log as logging
|
||||||
import oslo_messaging as messaging
|
import oslo_messaging as messaging
|
||||||
from stevedore import driver
|
from stevedore import driver
|
||||||
from stevedore import extension
|
|
||||||
from tooz import coordination
|
from tooz import coordination
|
||||||
|
|
||||||
from cloudkitty import collector
|
from cloudkitty import collector
|
||||||
from cloudkitty.common import rpc
|
from cloudkitty.common import rpc
|
||||||
from cloudkitty import config # noqa
|
from cloudkitty import config # noqa
|
||||||
from cloudkitty import extension_manager
|
from cloudkitty import extension_manager
|
||||||
|
from cloudkitty import storage
|
||||||
|
from cloudkitty import transformer
|
||||||
from cloudkitty import utils as ck_utils
|
from cloudkitty import utils as ck_utils
|
||||||
|
|
||||||
eventlet.monkey_patch()
|
eventlet.monkey_patch()
|
||||||
@ -40,7 +41,6 @@ eventlet.monkey_patch()
|
|||||||
LOG = logging.getLogger(__name__)
|
LOG = logging.getLogger(__name__)
|
||||||
|
|
||||||
CONF = cfg.CONF
|
CONF = cfg.CONF
|
||||||
CONF.import_opt('backend', 'cloudkitty.storage', 'storage')
|
|
||||||
CONF.import_opt('backend', 'cloudkitty.tenant_fetcher', 'tenant_fetcher')
|
CONF.import_opt('backend', 'cloudkitty.tenant_fetcher', 'tenant_fetcher')
|
||||||
|
|
||||||
orchestrator_opts = [
|
orchestrator_opts = [
|
||||||
@ -51,11 +51,8 @@ orchestrator_opts = [
|
|||||||
]
|
]
|
||||||
CONF.register_opts(orchestrator_opts, group='orchestrator')
|
CONF.register_opts(orchestrator_opts, group='orchestrator')
|
||||||
|
|
||||||
COLLECTORS_NAMESPACE = 'cloudkitty.collector.backends'
|
|
||||||
FETCHERS_NAMESPACE = 'cloudkitty.tenant.fetchers'
|
FETCHERS_NAMESPACE = 'cloudkitty.tenant.fetchers'
|
||||||
TRANSFORMERS_NAMESPACE = 'cloudkitty.transformers'
|
|
||||||
PROCESSORS_NAMESPACE = 'cloudkitty.rating.processors'
|
PROCESSORS_NAMESPACE = 'cloudkitty.rating.processors'
|
||||||
STORAGES_NAMESPACE = 'cloudkitty.storage.backends'
|
|
||||||
|
|
||||||
|
|
||||||
class RatingEndpoint(object):
|
class RatingEndpoint(object):
|
||||||
@ -226,24 +223,9 @@ class Orchestrator(object):
|
|||||||
CONF.tenant_fetcher.backend,
|
CONF.tenant_fetcher.backend,
|
||||||
invoke_on_load=True).driver
|
invoke_on_load=True).driver
|
||||||
|
|
||||||
# Transformers
|
self.transformers = transformer.get_transformers()
|
||||||
self.transformers = {}
|
self.collector = collector.get_collector(self.transformers)
|
||||||
self._load_transformers()
|
self.storage = storage.get_storage(self.collector)
|
||||||
|
|
||||||
collector_args = {'transformers': self.transformers,
|
|
||||||
'period': CONF.collect.period}
|
|
||||||
self.collector = driver.DriverManager(
|
|
||||||
COLLECTORS_NAMESPACE,
|
|
||||||
CONF.collect.collector,
|
|
||||||
invoke_on_load=True,
|
|
||||||
invoke_kwds=collector_args).driver
|
|
||||||
|
|
||||||
storage_args = {'period': CONF.collect.period}
|
|
||||||
self.storage = driver.DriverManager(
|
|
||||||
STORAGES_NAMESPACE,
|
|
||||||
CONF.storage.backend,
|
|
||||||
invoke_on_load=True,
|
|
||||||
invoke_kwds=storage_args).driver
|
|
||||||
|
|
||||||
# RPC
|
# RPC
|
||||||
self.server = None
|
self.server = None
|
||||||
@ -287,17 +269,6 @@ class Orchestrator(object):
|
|||||||
return next_timestamp
|
return next_timestamp
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
def _load_transformers(self):
|
|
||||||
self.transformers = {}
|
|
||||||
transformers = extension.ExtensionManager(
|
|
||||||
TRANSFORMERS_NAMESPACE,
|
|
||||||
invoke_on_load=True)
|
|
||||||
|
|
||||||
for transformer in transformers:
|
|
||||||
t_name = transformer.name
|
|
||||||
t_obj = transformer.obj
|
|
||||||
self.transformers[t_name] = t_obj
|
|
||||||
|
|
||||||
def process_messages(self):
|
def process_messages(self):
|
||||||
# TODO(sheeprine): Code kept to handle threading and asynchronous
|
# TODO(sheeprine): Code kept to handle threading and asynchronous
|
||||||
# reloading
|
# reloading
|
||||||
|
@ -21,23 +21,25 @@ from oslo_config import cfg
|
|||||||
import six
|
import six
|
||||||
from stevedore import driver
|
from stevedore import driver
|
||||||
|
|
||||||
|
from cloudkitty import collector as ck_collector
|
||||||
from cloudkitty import utils as ck_utils
|
from cloudkitty import utils as ck_utils
|
||||||
|
|
||||||
STORAGES_NAMESPACE = 'cloudkitty.storage.backends'
|
|
||||||
CONF = cfg.CONF
|
|
||||||
|
|
||||||
storage_opts = [
|
storage_opts = [
|
||||||
cfg.StrOpt('backend',
|
cfg.StrOpt('backend',
|
||||||
default='sqlalchemy',
|
default='sqlalchemy',
|
||||||
help='Name of the storage backend driver.')
|
help='Name of the storage backend driver.')
|
||||||
]
|
]
|
||||||
|
|
||||||
|
CONF = cfg.CONF
|
||||||
CONF.register_opts(storage_opts, group='storage')
|
CONF.register_opts(storage_opts, group='storage')
|
||||||
CONF.import_opt('period', 'cloudkitty.collector', 'collect')
|
|
||||||
|
STORAGES_NAMESPACE = 'cloudkitty.storage.backends'
|
||||||
|
|
||||||
|
|
||||||
def get_storage():
|
def get_storage(collector=None):
|
||||||
storage_args = {'period': cfg.CONF.collect.period}
|
storage_args = {
|
||||||
|
'period': CONF.collect.period,
|
||||||
|
'collector': collector if collector else ck_collector.get_collector()}
|
||||||
backend = driver.DriverManager(
|
backend = driver.DriverManager(
|
||||||
STORAGES_NAMESPACE,
|
STORAGES_NAMESPACE,
|
||||||
cfg.CONF.storage.backend,
|
cfg.CONF.storage.backend,
|
||||||
@ -60,8 +62,9 @@ class BaseStorage(object):
|
|||||||
|
|
||||||
Handle incoming data from the global orchestrator, and store them.
|
Handle incoming data from the global orchestrator, and store them.
|
||||||
"""
|
"""
|
||||||
def __init__(self, period=CONF.collect.period):
|
def __init__(self, **kwargs):
|
||||||
self._period = period
|
self._period = kwargs.get('period', CONF.collect.period)
|
||||||
|
self._collector = kwargs.get('collector')
|
||||||
|
|
||||||
# State vars
|
# State vars
|
||||||
self.usage_start = {}
|
self.usage_start = {}
|
||||||
|
69
cloudkitty/storage/gnocchi_hybrid/__init__.py
Normal file
69
cloudkitty/storage/gnocchi_hybrid/__init__.py
Normal file
@ -0,0 +1,69 @@
|
|||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
# Copyright 2015 Objectif Libre
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
#
|
||||||
|
# @author: Stéphane Albert
|
||||||
|
#
|
||||||
|
import decimal
|
||||||
|
|
||||||
|
from oslo_log import log
|
||||||
|
|
||||||
|
from cloudkitty.storage.gnocchi_hybrid import migration
|
||||||
|
from cloudkitty.storage.gnocchi_hybrid import models
|
||||||
|
from cloudkitty.storage import sqlalchemy as sql_storage
|
||||||
|
|
||||||
|
LOG = log.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class GnocchiHybridStorage(sql_storage.SQLAlchemyStorage):
|
||||||
|
"""Gnocchi Hybrid Storage Backend
|
||||||
|
|
||||||
|
Driver used to add support for gnocchi until the creation of custom
|
||||||
|
resources is supported in gnocchi.
|
||||||
|
"""
|
||||||
|
frame_model = models.HybridRatedDataframe
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def init():
|
||||||
|
migration.upgrade('head')
|
||||||
|
|
||||||
|
def _append_time_frame(self, res_type, frame, tenant_id):
|
||||||
|
rating_dict = frame.get('rating', {})
|
||||||
|
rate = rating_dict.get('price')
|
||||||
|
if not rate:
|
||||||
|
rate = decimal.Decimal(0)
|
||||||
|
resource_ref = frame.get('resource_id')
|
||||||
|
if not resource_ref:
|
||||||
|
LOG.warn('Trying to store data collected outside of gnocchi. '
|
||||||
|
'This driver can only be used with the gnocchi collector.'
|
||||||
|
' Data not stored!')
|
||||||
|
return
|
||||||
|
self.add_time_frame(begin=self.usage_start_dt.get(tenant_id),
|
||||||
|
end=self.usage_end_dt.get(tenant_id),
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
res_type=res_type,
|
||||||
|
resource_ref=resource_ref,
|
||||||
|
rate=rate)
|
||||||
|
|
||||||
|
def add_time_frame(self, **kwargs):
|
||||||
|
"""Create a new time frame.
|
||||||
|
|
||||||
|
:param begin: Start of the dataframe.
|
||||||
|
:param end: End of the dataframe.
|
||||||
|
:param res_type: Type of the resource.
|
||||||
|
:param rate: Calculated rate for this dataframe.
|
||||||
|
:param tenant_id: tenant_id of the dataframe owner.
|
||||||
|
:param resource_ref: Reference to the gnocchi metric (UUID).
|
||||||
|
"""
|
||||||
|
super(GnocchiHybridStorage, self).add_time_frame(**kwargs)
|
25
cloudkitty/storage/gnocchi_hybrid/alembic/env.py
Normal file
25
cloudkitty/storage/gnocchi_hybrid/alembic/env.py
Normal file
@ -0,0 +1,25 @@
|
|||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
# Copyright 2015 Objectif Libre
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
#
|
||||||
|
# @author: Stéphane Albert
|
||||||
|
#
|
||||||
|
from cloudkitty.common.db.alembic import env # noqa
|
||||||
|
from cloudkitty.storage.gnocchi_hybrid import models
|
||||||
|
|
||||||
|
target_metadata = models.Base.metadata
|
||||||
|
version_table = 'storage_gnocchi_hybrid_alembic'
|
||||||
|
|
||||||
|
|
||||||
|
env.run_migrations_online(target_metadata, version_table)
|
22
cloudkitty/storage/gnocchi_hybrid/alembic/script.py.mako
Normal file
22
cloudkitty/storage/gnocchi_hybrid/alembic/script.py.mako
Normal file
@ -0,0 +1,22 @@
|
|||||||
|
"""${message}
|
||||||
|
|
||||||
|
Revision ID: ${up_revision}
|
||||||
|
Revises: ${down_revision}
|
||||||
|
Create Date: ${create_date}
|
||||||
|
|
||||||
|
"""
|
||||||
|
|
||||||
|
# revision identifiers, used by Alembic.
|
||||||
|
revision = ${repr(up_revision)}
|
||||||
|
down_revision = ${repr(down_revision)}
|
||||||
|
|
||||||
|
from alembic import op
|
||||||
|
import sqlalchemy as sa
|
||||||
|
${imports if imports else ""}
|
||||||
|
|
||||||
|
def upgrade():
|
||||||
|
${upgrades if upgrades else "pass"}
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade():
|
||||||
|
${downgrades if downgrades else "pass"}
|
@ -0,0 +1,32 @@
|
|||||||
|
"""Initial migration.
|
||||||
|
|
||||||
|
Revision ID: 4c2f20df7491
|
||||||
|
Revises: None
|
||||||
|
Create Date: 2015-11-18 11:44:09.175326
|
||||||
|
|
||||||
|
"""
|
||||||
|
|
||||||
|
# revision identifiers, used by Alembic.
|
||||||
|
revision = '4c2f20df7491'
|
||||||
|
down_revision = None
|
||||||
|
|
||||||
|
from alembic import op
|
||||||
|
import sqlalchemy as sa
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade():
|
||||||
|
op.create_table('ghybrid_dataframes',
|
||||||
|
sa.Column('id', sa.Integer(), nullable=False),
|
||||||
|
sa.Column('begin', sa.DateTime(), nullable=False),
|
||||||
|
sa.Column('end', sa.DateTime(), nullable=False),
|
||||||
|
sa.Column('res_type', sa.String(length=255), nullable=False),
|
||||||
|
sa.Column('rate', sa.Numeric(precision=20, scale=8), nullable=False),
|
||||||
|
sa.Column('resource_ref', sa.String(length=32), nullable=False),
|
||||||
|
sa.Column('tenant_id', sa.String(length=32), nullable=True),
|
||||||
|
sa.PrimaryKeyConstraint('id'),
|
||||||
|
mysql_charset='utf8',
|
||||||
|
mysql_engine='InnoDB')
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade():
|
||||||
|
op.drop_table('ghybrid_dataframes')
|
47
cloudkitty/storage/gnocchi_hybrid/migration.py
Normal file
47
cloudkitty/storage/gnocchi_hybrid/migration.py
Normal file
@ -0,0 +1,47 @@
|
|||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
# Copyright 2015 Objectif Libre
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
#
|
||||||
|
# @author: Stéphane Albert
|
||||||
|
#
|
||||||
|
import os
|
||||||
|
|
||||||
|
from cloudkitty.common.db.alembic import migration
|
||||||
|
|
||||||
|
ALEMBIC_REPO = os.path.join(os.path.dirname(__file__), 'alembic')
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade(revision):
|
||||||
|
config = migration.load_alembic_config(ALEMBIC_REPO)
|
||||||
|
return migration.upgrade(config, revision)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade(revision):
|
||||||
|
config = migration.load_alembic_config(ALEMBIC_REPO)
|
||||||
|
return migration.downgrade(config, revision)
|
||||||
|
|
||||||
|
|
||||||
|
def version():
|
||||||
|
config = migration.load_alembic_config(ALEMBIC_REPO)
|
||||||
|
return migration.version(config)
|
||||||
|
|
||||||
|
|
||||||
|
def revision(message, autogenerate):
|
||||||
|
config = migration.load_alembic_config(ALEMBIC_REPO)
|
||||||
|
return migration.revision(config, message, autogenerate)
|
||||||
|
|
||||||
|
|
||||||
|
def stamp(revision):
|
||||||
|
config = migration.load_alembic_config(ALEMBIC_REPO)
|
||||||
|
return migration.stamp(config, revision)
|
86
cloudkitty/storage/gnocchi_hybrid/models.py
Normal file
86
cloudkitty/storage/gnocchi_hybrid/models.py
Normal file
@ -0,0 +1,86 @@
|
|||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
# Copyright 2015 Objectif Libre
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
#
|
||||||
|
# @author: Stéphane Albert
|
||||||
|
#
|
||||||
|
from oslo_db.sqlalchemy import models
|
||||||
|
import sqlalchemy
|
||||||
|
from sqlalchemy.ext import declarative
|
||||||
|
|
||||||
|
from cloudkitty import utils as ck_utils
|
||||||
|
|
||||||
|
Base = declarative.declarative_base()
|
||||||
|
|
||||||
|
|
||||||
|
class HybridRatedDataframe(Base, models.ModelBase):
|
||||||
|
"""An hybrid rated dataframe.
|
||||||
|
|
||||||
|
"""
|
||||||
|
__table_args__ = {'mysql_charset': "utf8",
|
||||||
|
'mysql_engine': "InnoDB"}
|
||||||
|
__tablename__ = 'ghybrid_dataframes'
|
||||||
|
|
||||||
|
id = sqlalchemy.Column(sqlalchemy.Integer,
|
||||||
|
primary_key=True)
|
||||||
|
begin = sqlalchemy.Column(sqlalchemy.DateTime,
|
||||||
|
nullable=False)
|
||||||
|
end = sqlalchemy.Column(sqlalchemy.DateTime,
|
||||||
|
nullable=False)
|
||||||
|
res_type = sqlalchemy.Column(sqlalchemy.String(255),
|
||||||
|
nullable=False)
|
||||||
|
rate = sqlalchemy.Column(sqlalchemy.Numeric(20, 8),
|
||||||
|
nullable=False)
|
||||||
|
resource_ref = sqlalchemy.Column(sqlalchemy.String(32),
|
||||||
|
nullable=False)
|
||||||
|
tenant_id = sqlalchemy.Column(sqlalchemy.String(32),
|
||||||
|
nullable=True)
|
||||||
|
|
||||||
|
def to_cloudkitty(self, collector=None):
|
||||||
|
if not collector:
|
||||||
|
raise Exception('Gnocchi storage needs a reference '
|
||||||
|
'to the collector.')
|
||||||
|
# Rating informations
|
||||||
|
rating_dict = {}
|
||||||
|
rating_dict['price'] = self.rate
|
||||||
|
|
||||||
|
# Resource information from gnocchi
|
||||||
|
resource_data = collector.resource_info(
|
||||||
|
resource_type=self.res_type,
|
||||||
|
start=self.begin,
|
||||||
|
end=self.end,
|
||||||
|
resource_id=self.resource_ref,
|
||||||
|
project_id=self.tenant_id)
|
||||||
|
|
||||||
|
# Encapsulate informations in a resource dict
|
||||||
|
res_dict = {}
|
||||||
|
res_dict['desc'] = resource_data['desc']
|
||||||
|
res_dict['vol'] = resource_data['vol']
|
||||||
|
res_dict['rating'] = rating_dict
|
||||||
|
res_dict['tenant_id'] = self.tenant_id
|
||||||
|
|
||||||
|
# Add resource to the usage dict
|
||||||
|
usage_dict = {}
|
||||||
|
usage_dict[self.res_type] = [res_dict]
|
||||||
|
|
||||||
|
# Time informations
|
||||||
|
period_dict = {}
|
||||||
|
period_dict['begin'] = ck_utils.dt2iso(self.begin)
|
||||||
|
period_dict['end'] = ck_utils.dt2iso(self.end)
|
||||||
|
|
||||||
|
# Add period to the resource informations
|
||||||
|
ck_dict = {}
|
||||||
|
ck_dict['period'] = period_dict
|
||||||
|
ck_dict['usage'] = usage_dict
|
||||||
|
return ck_dict
|
@ -15,6 +15,7 @@
|
|||||||
#
|
#
|
||||||
# @author: Stéphane Albert
|
# @author: Stéphane Albert
|
||||||
#
|
#
|
||||||
|
import decimal
|
||||||
import json
|
import json
|
||||||
|
|
||||||
from oslo_db.sqlalchemy import utils
|
from oslo_db.sqlalchemy import utils
|
||||||
@ -31,8 +32,10 @@ class SQLAlchemyStorage(storage.BaseStorage):
|
|||||||
"""SQLAlchemy Storage Backend
|
"""SQLAlchemy Storage Backend
|
||||||
|
|
||||||
"""
|
"""
|
||||||
def __init__(self, period=3600):
|
frame_model = models.RatedDataFrame
|
||||||
super(SQLAlchemyStorage, self).__init__(period)
|
|
||||||
|
def __init__(self, **kwargs):
|
||||||
|
super(SQLAlchemyStorage, self).__init__(**kwargs)
|
||||||
self._session = {}
|
self._session = {}
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
@ -69,22 +72,18 @@ class SQLAlchemyStorage(storage.BaseStorage):
|
|||||||
def get_state(self, tenant_id=None):
|
def get_state(self, tenant_id=None):
|
||||||
session = db.get_session()
|
session = db.get_session()
|
||||||
q = utils.model_query(
|
q = utils.model_query(
|
||||||
models.RatedDataFrame,
|
self.frame_model,
|
||||||
session
|
session)
|
||||||
)
|
|
||||||
if tenant_id:
|
if tenant_id:
|
||||||
q = q.filter(
|
q = q.filter(
|
||||||
models.RatedDataFrame.tenant_id == tenant_id
|
self.frame_model.tenant_id == tenant_id)
|
||||||
)
|
q = q.order_by(
|
||||||
r = q.order_by(
|
self.frame_model.begin.desc())
|
||||||
models.RatedDataFrame.begin.desc()
|
r = q.first()
|
||||||
).first()
|
|
||||||
if r:
|
if r:
|
||||||
return ck_utils.dt2ts(r.begin)
|
return ck_utils.dt2ts(r.begin)
|
||||||
|
|
||||||
def get_total(self, begin=None, end=None, tenant_id=None, service=None):
|
def get_total(self, begin=None, end=None, tenant_id=None, service=None):
|
||||||
model = models.RatedDataFrame
|
|
||||||
|
|
||||||
# Boundary calculation
|
# Boundary calculation
|
||||||
if not begin:
|
if not begin:
|
||||||
begin = ck_utils.get_month_start()
|
begin = ck_utils.get_month_start()
|
||||||
@ -93,22 +92,20 @@ class SQLAlchemyStorage(storage.BaseStorage):
|
|||||||
|
|
||||||
session = db.get_session()
|
session = db.get_session()
|
||||||
q = session.query(
|
q = session.query(
|
||||||
sqlalchemy.func.sum(model.rate).label('rate'))
|
sqlalchemy.func.sum(self.frame_model.rate).label('rate'))
|
||||||
if tenant_id:
|
if tenant_id:
|
||||||
q = q.filter(
|
q = q.filter(
|
||||||
models.RatedDataFrame.tenant_id == tenant_id)
|
self.frame_model.tenant_id == tenant_id)
|
||||||
if service:
|
if service:
|
||||||
q = q.filter(
|
q = q.filter(
|
||||||
models.RatedDataFrame.res_type == service)
|
self.frame_model.res_type == service)
|
||||||
q = q.filter(
|
q = q.filter(
|
||||||
model.begin >= begin,
|
self.frame_model.begin >= begin,
|
||||||
model.end <= end)
|
self.frame_model.end <= end)
|
||||||
rate = q.scalar()
|
rate = q.scalar()
|
||||||
return rate
|
return rate
|
||||||
|
|
||||||
def get_tenants(self, begin=None, end=None):
|
def get_tenants(self, begin=None, end=None):
|
||||||
model = models.RatedDataFrame
|
|
||||||
|
|
||||||
# Boundary calculation
|
# Boundary calculation
|
||||||
if not begin:
|
if not begin:
|
||||||
begin = ck_utils.get_month_start()
|
begin = ck_utils.get_month_start()
|
||||||
@ -117,65 +114,64 @@ class SQLAlchemyStorage(storage.BaseStorage):
|
|||||||
|
|
||||||
session = db.get_session()
|
session = db.get_session()
|
||||||
q = utils.model_query(
|
q = utils.model_query(
|
||||||
model,
|
self.frame_model,
|
||||||
session
|
session)
|
||||||
).filter(
|
q = q.filter(
|
||||||
model.begin >= begin,
|
self.frame_model.begin >= begin,
|
||||||
model.end <= end
|
self.frame_model.end <= end)
|
||||||
)
|
|
||||||
tenants = q.distinct().values(
|
tenants = q.distinct().values(
|
||||||
model.tenant_id
|
self.frame_model.tenant_id)
|
||||||
)
|
|
||||||
return [tenant.tenant_id for tenant in tenants]
|
return [tenant.tenant_id for tenant in tenants]
|
||||||
|
|
||||||
def get_time_frame(self, begin, end, **filters):
|
def get_time_frame(self, begin, end, **filters):
|
||||||
model = models.RatedDataFrame
|
|
||||||
session = db.get_session()
|
session = db.get_session()
|
||||||
q = utils.model_query(
|
q = utils.model_query(
|
||||||
model,
|
self.frame_model,
|
||||||
session
|
session)
|
||||||
).filter(
|
q = q.filter(
|
||||||
model.begin >= ck_utils.ts2dt(begin),
|
self.frame_model.begin >= ck_utils.ts2dt(begin),
|
||||||
model.end <= ck_utils.ts2dt(end)
|
self.frame_model.end <= ck_utils.ts2dt(end))
|
||||||
)
|
|
||||||
for filter_name, filter_value in filters.items():
|
for filter_name, filter_value in filters.items():
|
||||||
if filter_value:
|
if filter_value:
|
||||||
q = q.filter(getattr(model, filter_name) == filter_value)
|
q = q.filter(
|
||||||
|
getattr(self.frame_model, filter_name) == filter_value)
|
||||||
if not filters.get('res_type'):
|
if not filters.get('res_type'):
|
||||||
q = q.filter(model.res_type != '_NO_DATA_')
|
q = q.filter(self.frame_model.res_type != '_NO_DATA_')
|
||||||
count = q.count()
|
count = q.count()
|
||||||
if not count:
|
if not count:
|
||||||
raise storage.NoTimeFrame()
|
raise storage.NoTimeFrame()
|
||||||
r = q.all()
|
r = q.all()
|
||||||
return [entry.to_cloudkitty() for entry in r]
|
return [entry.to_cloudkitty(self._collector) for entry in r]
|
||||||
|
|
||||||
def _append_time_frame(self, res_type, frame, tenant_id):
|
def _append_time_frame(self, res_type, frame, tenant_id):
|
||||||
vol_dict = frame['vol']
|
vol_dict = frame['vol']
|
||||||
qty = vol_dict['qty']
|
qty = vol_dict['qty']
|
||||||
unit = vol_dict['unit']
|
unit = vol_dict['unit']
|
||||||
rating_dict = frame['rating']
|
rating_dict = frame.get('rating', {})
|
||||||
rate = rating_dict['price']
|
rate = rating_dict.get('price')
|
||||||
|
if not rate:
|
||||||
|
rate = decimal.Decimal(0)
|
||||||
desc = json.dumps(frame['desc'])
|
desc = json.dumps(frame['desc'])
|
||||||
self.add_time_frame(self.usage_start_dt.get(tenant_id),
|
self.add_time_frame(begin=self.usage_start_dt.get(tenant_id),
|
||||||
self.usage_end_dt.get(tenant_id),
|
end=self.usage_end_dt.get(tenant_id),
|
||||||
tenant_id,
|
tenant_id=tenant_id,
|
||||||
unit,
|
unit=unit,
|
||||||
qty,
|
qty=qty,
|
||||||
res_type,
|
res_type=res_type,
|
||||||
rate,
|
rate=rate,
|
||||||
desc)
|
desc=desc)
|
||||||
|
|
||||||
def add_time_frame(self, begin, end, tenant_id, unit, qty, res_type,
|
def add_time_frame(self, **kwargs):
|
||||||
rate, desc):
|
|
||||||
"""Create a new time frame.
|
"""Create a new time frame.
|
||||||
|
|
||||||
|
:param begin: Start of the dataframe.
|
||||||
|
:param end: End of the dataframe.
|
||||||
|
:param tenant_id: tenant_id of the dataframe owner.
|
||||||
|
:param unit: Unit of the metric.
|
||||||
|
:param qty: Quantity of the metric.
|
||||||
|
:param res_type: Type of the resource.
|
||||||
|
:param rate: Calculated rate for this dataframe.
|
||||||
|
:param desc: Resource description (metadata).
|
||||||
"""
|
"""
|
||||||
frame = models.RatedDataFrame(begin=begin,
|
frame = self.frame_model(**kwargs)
|
||||||
end=end,
|
self._session[kwargs.get('tenant_id')].add(frame)
|
||||||
tenant_id=tenant_id,
|
|
||||||
unit=unit,
|
|
||||||
qty=qty,
|
|
||||||
res_type=res_type,
|
|
||||||
rate=rate,
|
|
||||||
desc=desc)
|
|
||||||
self._session[tenant_id].add(frame)
|
|
||||||
|
@ -53,7 +53,7 @@ class RatedDataFrame(Base, models.ModelBase):
|
|||||||
desc = sqlalchemy.Column(sqlalchemy.Text(),
|
desc = sqlalchemy.Column(sqlalchemy.Text(),
|
||||||
nullable=False)
|
nullable=False)
|
||||||
|
|
||||||
def to_cloudkitty(self):
|
def to_cloudkitty(self, collector=None):
|
||||||
# Rating informations
|
# Rating informations
|
||||||
rating_dict = {}
|
rating_dict = {}
|
||||||
rating_dict['price'] = self.rate
|
rating_dict['price'] = self.rate
|
||||||
|
@ -17,6 +17,7 @@
|
|||||||
#
|
#
|
||||||
import decimal
|
import decimal
|
||||||
|
|
||||||
|
import mock
|
||||||
from oslo_config import fixture as config_fixture
|
from oslo_config import fixture as config_fixture
|
||||||
from oslotest import base
|
from oslotest import base
|
||||||
import testscenarios
|
import testscenarios
|
||||||
@ -75,7 +76,19 @@ class TestCase(testscenarios.TestWithScenarios, base.BaseTestCase):
|
|||||||
self.conn = ck_db_api.get_instance()
|
self.conn = ck_db_api.get_instance()
|
||||||
migration = self.conn.get_migration()
|
migration = self.conn.get_migration()
|
||||||
migration.upgrade('head')
|
migration.upgrade('head')
|
||||||
|
auth = mock.patch(
|
||||||
|
'keystoneauth1.loading.load_auth_from_conf_options',
|
||||||
|
return_value=dict())
|
||||||
|
auth.start()
|
||||||
|
self.auth = auth
|
||||||
|
session = mock.patch(
|
||||||
|
'keystoneauth1.loading.load_session_from_conf_options',
|
||||||
|
return_value=dict())
|
||||||
|
session.start()
|
||||||
|
self.session = session
|
||||||
|
|
||||||
def tearDown(self):
|
def tearDown(self):
|
||||||
db.get_engine().dispose()
|
db.get_engine().dispose()
|
||||||
|
self.auth.stop()
|
||||||
|
self.session.stop()
|
||||||
super(TestCase, self).tearDown()
|
super(TestCase, self).tearDown()
|
||||||
|
@ -274,7 +274,15 @@ class BaseStorageDataFixture(fixture.GabbiFixture):
|
|||||||
return data
|
return data
|
||||||
|
|
||||||
def start_fixture(self):
|
def start_fixture(self):
|
||||||
self.storage = storage.get_storage()
|
auth = mock.patch(
|
||||||
|
'keystoneauth1.loading.load_auth_from_conf_options',
|
||||||
|
return_value=dict())
|
||||||
|
session = mock.patch(
|
||||||
|
'keystoneauth1.loading.load_session_from_conf_options',
|
||||||
|
return_value=dict())
|
||||||
|
with auth:
|
||||||
|
with session:
|
||||||
|
self.storage = storage.get_storage()
|
||||||
self.storage.init()
|
self.storage.init()
|
||||||
self.initialize_data()
|
self.initialize_data()
|
||||||
|
|
||||||
@ -350,4 +358,10 @@ class CORSConfigFixture(fixture.GabbiFixture):
|
|||||||
|
|
||||||
def setup_app():
|
def setup_app():
|
||||||
rpc.init()
|
rpc.init()
|
||||||
return app.load_app()
|
# FIXME(sheeprine): Extension fixtures are interacting with transformers
|
||||||
|
# loading, since collectors are not needed here we shunt them
|
||||||
|
no_collector = mock.patch(
|
||||||
|
'cloudkitty.collector.get_collector',
|
||||||
|
return_value=None)
|
||||||
|
with no_collector:
|
||||||
|
return app.load_app()
|
||||||
|
@ -1,6 +1,6 @@
|
|||||||
fixtures:
|
fixtures:
|
||||||
- PyScriptsConfigFixture
|
- PyScriptsConfigFixture
|
||||||
- UUIDFixture
|
- UUIDFixture
|
||||||
|
|
||||||
tests:
|
tests:
|
||||||
|
|
||||||
|
@ -18,6 +18,21 @@
|
|||||||
import abc
|
import abc
|
||||||
|
|
||||||
import six
|
import six
|
||||||
|
from stevedore import extension
|
||||||
|
|
||||||
|
TRANSFORMERS_NAMESPACE = 'cloudkitty.transformers'
|
||||||
|
|
||||||
|
|
||||||
|
def get_transformers():
|
||||||
|
transformers = {}
|
||||||
|
transformer_exts = extension.ExtensionManager(
|
||||||
|
TRANSFORMERS_NAMESPACE,
|
||||||
|
invoke_on_load=True)
|
||||||
|
for transformer in transformer_exts:
|
||||||
|
t_name = transformer.name
|
||||||
|
t_obj = transformer.obj
|
||||||
|
transformers[t_name] = t_obj
|
||||||
|
return transformers
|
||||||
|
|
||||||
|
|
||||||
@six.add_metaclass(abc.ABCMeta)
|
@six.add_metaclass(abc.ABCMeta)
|
||||||
|
@ -58,6 +58,7 @@ cloudkitty.rating.processors =
|
|||||||
|
|
||||||
cloudkitty.storage.backends =
|
cloudkitty.storage.backends =
|
||||||
sqlalchemy = cloudkitty.storage.sqlalchemy:SQLAlchemyStorage
|
sqlalchemy = cloudkitty.storage.sqlalchemy:SQLAlchemyStorage
|
||||||
|
gnocchihybrid = cloudkitty.storage.gnocchi_hybrid:GnocchiHybridStorage
|
||||||
|
|
||||||
cloudkitty.output.writers =
|
cloudkitty.output.writers =
|
||||||
osrf = cloudkitty.writer.osrf:OSRFBackend
|
osrf = cloudkitty.writer.osrf:OSRFBackend
|
||||||
|
Loading…
Reference in New Issue
Block a user