openstack/cloudkitty
Code Issues Proposed changes

Transitioned from importutils to stevedore

Browse Source 
Billing modules now use stevedore.
Collection modules now use stevedore.
Writer modules now use stevedore.
Billing modules now reports if they are enabled or not.

Change-Id: I8d7609ef8f227722a5a897e045d780d8440ac4fa
This commit is contained in:
Stéphane Albert 2014-08-08 14:02:40 +02:00
parent 4c440984d6
commit 2b575ced77
8 changed files with 122 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
cloudkitty/billing/__init__.py
View File

@ -26,6 +26,13 @@ class BillingProcessorBase(object):
def __init__(self):
pass
@abc.abstractproperty
def enabled(self):
"""Check if the module is enabled
:returns: bool if module is enabled
"""
@abc.abstractmethod
def process(self, data):
"""Add billing informations to data

5
cloudkitty/billing/hash.py
View File

@ -25,6 +25,11 @@ class BasicHashMap(billing.BillingProcessorBase):
self._billing_info = {}
self._load_billing_rates()
@property
def enabled(self):
# TODO(sheeprine): Implement real feature
return True
def _load_billing_rates(self):
# FIXME We should use another path
self._billing_info = json.loads(open('billing_info.json').read())

4
cloudkitty/billing/noop.py
View File

@ -22,6 +22,10 @@ class Noop(billing.BillingProcessorBase):
def __init__(self):
pass
@property
def enabled(self):
return True
def process(self, data):
for cur_data in data:
cur_usage = cur_data['usage']

15
cloudkitty/config.py
View File

@ -16,6 +16,10 @@
# @author: Stéphane Albert
#
from oslo.config import cfg
from oslo.db import options as db_options # noqa
from oslo.messaging import opts # noqa
from cloudkitty.openstack.common import log # noqa
auth_opts = [
@ -37,7 +41,7 @@ auth_opts = [
collect_opts = [
cfg.StrOpt('collector',
default='cloudkitty.collector.ceilometer.CeilometerCollector',
default='ceilometer',
help='Data collector.'),
cfg.IntOpt('window',
default=1800,
@ -57,12 +61,6 @@ state_opts = [
default='/var/lib/cloudkitty/states/',
help='Storage directory for the file state backend.'), ]
billing_opts = [
cfg.ListOpt('pipeline',
default=['cloudkitty.billing.hash.BasicHashMap',
'cloudkitty.billing.noop.Noop'],
help='Billing pipeline modules.'), ]
output_opts = [
cfg.StrOpt('backend',
default='cloudkitty.backend.file.FileBackend',
@ -71,12 +69,11 @@ output_opts = [
default='/var/lib/cloudkitty/states/',
help='Storage directory for the file output backend.'),
cfg.ListOpt('pipeline',
default=['cloudkitty.writer.osrf.OSRFBackend'],
default=['osrf'],
help='Output pipeline'), ]
cfg.CONF.register_opts(auth_opts, 'auth')
cfg.CONF.register_opts(collect_opts, 'collect')
cfg.CONF.register_opts(state_opts, 'state')
cfg.CONF.register_opts(billing_opts, 'billing')
cfg.CONF.register_opts(output_opts, 'output')

42
cloudkitty/extension_manager.py Normal file
View File

@ -0,0 +1,42 @@
# -*- coding: utf-8 -*-
# Copyright 2014 Objectif Libre
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
# @author: Stéphane Albert
#
from stevedore import enabled
class EnabledExtensionManager(enabled.EnabledExtensionManager):
"""CloudKitty Billing processor manager
Override default EnabledExtensionManager to check for an internal
object property in the extension.
"""
def __init__(self, namespace, invoke_args=(), invoke_kwds={}):
def check_enabled(ext):
"""Check if extension is enabled.
"""
return ext.obj.enabled
super(EnabledExtensionManager, self).__init__(
namespace=namespace,
check_func=check_enabled,
invoke_on_load=True,
invoke_args=invoke_args,
invoke_kwds=invoke_kwds,
)

72
cloudkitty/orchestrator.py
View File

@ -23,56 +23,65 @@ import time
from keystoneclient.v2_0 import client as kclient
from oslo.config import cfg
from stevedore import driver
from stevedore import named
import cloudkitty.config # NOQA
import cloudkitty.openstack.common.importutils as i_utils
from cloudkitty import config # NOQA
from cloudkitty import extension_manager
from cloudkitty.openstack.common import importutils as i_utils
from cloudkitty.openstack.common import log as logging
from cloudkitty import state
from cloudkitty import write_orchestrator as w_orch
LOG = logging.getLogger(__name__)
CONF = cfg.CONF
class Orchestrator(object):
def __init__(self):
# Billing settings
self.billing_pipeline = []
for billing_processor in CONF.billing.pipeline:
processor = i_utils.import_class(billing_processor)
self.billing_pipeline.append(processor)
# Output settings
self.output_pipeline = []
for writer in CONF.output.pipeline:
self.output_pipeline.append(i_utils.import_class(writer))
self.keystone = kclient.Client(username=CONF.auth.username,
password=CONF.auth.password,
tenant_name=CONF.auth.tenant,
region_name=CONF.auth.region,
auth_url=CONF.auth.url)
self.sm = state.StateManager(i_utils.import_class(CONF.state.backend),
s_backend = i_utils.import_class(CONF.state.backend)
self.sm = state.StateManager(s_backend,
CONF.state.basepath,
self.keystone.user_id,
'osrtf')
collector = i_utils.import_class(CONF.collect.collector)
self.collector = collector(user=CONF.auth.username,
password=CONF.auth.password,
tenant=CONF.auth.tenant,
region=CONF.auth.region,
keystone_url=CONF.auth.url,
period=CONF.collect.period)
collector_args = {'user': CONF.auth.username,
'password': CONF.auth.password,
'tenant': CONF.auth.tenant,
'region': CONF.auth.region,
'keystone_url': CONF.auth.url,
'period': CONF.collect.period}
self.collector = driver.DriverManager(
'cloudkitty.collector.backends',
CONF.collect.collector,
invoke_on_load=True,
invoke_kwds=collector_args).driver
w_backend = i_utils.import_class(CONF.output.backend)
s_backend = i_utils.import_class(CONF.state.backend)
self.wo = w_orch.WriteOrchestrator(w_backend,
s_backend,
self.keystone.user_id,
self.sm)
for writer in self.output_pipeline:
self.wo.add_writer(writer)
# Billing processors
self.b_processors = {}
self._load_billing_processors()
# Output settings
output_pipeline = named.NamedExtensionManager(
'cloudkitty.output.writers',
CONF.output.pipeline)
for writer in output_pipeline:
self.wo.add_writer(writer.plugin)
def _check_state(self):
def _get_this_month_timestamp():
@ -101,6 +110,17 @@ class Orchestrator(object):
'usage': raw_data}]
return timed_data
def _load_billing_processors(self):
self.b_processors = {}
processors = extension_manager.EnabledExtensionManager(
'cloudkitty.billing.processors',
)
for processor in processors:
b_name = processor.name
b_obj = processor.obj
self.b_processors[b_name] = b_obj
def process(self):
while True:
timestamp = self._check_state()
@ -112,9 +132,8 @@ class Orchestrator(object):
data = self._collect(service, timestamp)
# Billing
for b_proc in self.billing_pipeline:
b_obj = b_proc()
data = b_obj.process(data)
for processor in self.b_processors.values():
processor.process(data)
# Writing
self.wo.append(data)
@ -127,6 +146,7 @@ class Orchestrator(object):
def main():
CONF(sys.argv[1:], project='cloudkitty')
logging.setup('cloudkitty')
orchestrator = Orchestrator()
orchestrator.process()

14
etc/cloudkitty/cloudkitty.conf.sample
View File

@ -116,16 +116,6 @@
#url=
[billing]
#
# Options defined in cloudkitty.config
#
# Billing pipeline modules. (list value)
#pipeline=cloudkitty.billing.hash.BasicHashMap,cloudkitty.billing.noop.Noop
[collect]
#
@ -133,7 +123,7 @@
#
# Data collector. (string value)
#collector=cloudkitty.collector.ceilometer.CeilometerCollector
#collector=ceilometer
# Number of samples to collect per call. (integer value)
#window=1800
@ -159,7 +149,7 @@
#basepath=/var/lib/cloudkitty/states/
# Output pipeline (list value)
#pipeline=cloudkitty.writer.osrf.OSRFBackend
#pipeline=osrf
[state]

10
setup.cfg
View File

@ -22,6 +22,16 @@ packages =
console_scripts =
cloudkitty-processor = cloudkitty.orchestrator:main
cloudkitty.collector.backends =
ceilometer = cloudkitty.collector.ceilometer:CeilometerCollector
cloudkitty.billing.processors =
noop = cloudkitty.billing.noop:Noop
hashmap = cloudkitty.billing.hash:BasicHashMap
cloudkitty.output.writers =
osrf = cloudkitty.writer.osrf:OSRFBackend
[build_sphinx]
all_files = 1
build-dir = doc/build