From 2f3d9a8c5549404a4686c42d038b5c4cb5a2c5b5 Mon Sep 17 00:00:00 2001 From: Mehdi Abaakouk Date: Thu, 16 Jul 2015 12:10:32 +0200 Subject: [PATCH] Move gnocchi resources definition in yaml file Transformation of samples to gnocchi resources/metrics is just json2json think, so do it in a declarative ways like we do for event and declarative notification. Implements blueprint gnocchi-declarative-resources Change-Id: I5a202c30614d06821063e243d4e2330736aba5fd --- ceilometer/dispatcher/gnocchi.py | 235 +++++++++++++----- ceilometer/dispatcher/resources/__init__.py | 0 ceilometer/dispatcher/resources/base.py | 40 --- .../dispatcher/resources/ceph_account.py | 31 --- ceilometer/dispatcher/resources/identity.py | 44 ---- ceilometer/dispatcher/resources/image.py | 30 --- ceilometer/dispatcher/resources/instance.py | 54 ---- ceilometer/dispatcher/resources/ipmi.py | 30 --- ceilometer/dispatcher/resources/network.py | 41 --- .../dispatcher/resources/orchestration.py | 30 --- .../dispatcher/resources/swift_account.py | 33 --- ceilometer/dispatcher/resources/volume.py | 38 --- ceilometer/tests/dispatcher/test_gnocchi.py | 62 ++++- devstack/plugin.sh | 2 + etc/ceilometer/gnocchi_resources.yaml | 121 +++++++++ setup.cfg | 11 - 16 files changed, 344 insertions(+), 458 deletions(-) delete mode 100644 ceilometer/dispatcher/resources/__init__.py delete mode 100644 ceilometer/dispatcher/resources/base.py delete mode 100644 ceilometer/dispatcher/resources/ceph_account.py delete mode 100644 ceilometer/dispatcher/resources/identity.py delete mode 100644 ceilometer/dispatcher/resources/image.py delete mode 100644 ceilometer/dispatcher/resources/instance.py delete mode 100644 ceilometer/dispatcher/resources/ipmi.py delete mode 100644 ceilometer/dispatcher/resources/network.py delete mode 100644 ceilometer/dispatcher/resources/orchestration.py delete mode 100644 ceilometer/dispatcher/resources/swift_account.py delete mode 100644 ceilometer/dispatcher/resources/volume.py create mode 100644 etc/ceilometer/gnocchi_resources.yaml diff --git a/ceilometer/dispatcher/gnocchi.py b/ceilometer/dispatcher/gnocchi.py index 177ab3eb..c79220da 100644 --- a/ceilometer/dispatcher/gnocchi.py +++ b/ceilometer/dispatcher/gnocchi.py @@ -16,22 +16,22 @@ # License for the specific language governing permissions and limitations # under the License. import fnmatch -import threading - import itertools import json import operator import os -import yaml +import threading -from ceilometer import dispatcher -from ceilometer.i18n import _ -from ceilometer import keystone_client +import jsonpath_rw from oslo_config import cfg from oslo_log import log import requests import six -import stevedore.dispatch +import yaml + +from ceilometer import dispatcher +from ceilometer.i18n import _, _LE +from ceilometer import keystone_client LOG = log.getLogger(__name__) @@ -53,8 +53,13 @@ dispatcher_opts = [ 'create a new metric.'), cfg.StrOpt('archive_policy_file', default='gnocchi_archive_policy_map.yaml', + deprecated_for_removal=True, help=_('The Yaml file that defines per metric archive ' 'policies.')), + cfg.StrOpt('resources_definition_file', + default='gnocchi_resources.yaml', + help=_('The Yaml file that defines mapping between samples ' + 'and gnocchi resources/metrics')), ] cfg.CONF.register_opts(dispatcher_opts, group="dispatcher_gnocchi") @@ -93,6 +98,98 @@ def log_and_ignore_unexpected_workflow_error(func): return log_and_ignore +class LegacyArchivePolicyDefinition(object): + def __init__(self, definition_cfg): + self.cfg = definition_cfg + if self.cfg is None: + LOG.debug(_("No archive policy file found!" + " Using default config.")) + + def get(self, metric_name): + if self.cfg is not None: + for metric, policy in self.cfg.items(): + # Support wild cards such as disk.* + if fnmatch.fnmatch(metric_name, metric): + return policy + + +class ResourcesDefinitionException(Exception): + def __init__(self, message, definition_cfg): + super(ResourcesDefinitionException, self).__init__(message) + self.definition_cfg = definition_cfg + + def __str__(self): + return '%s %s: %s' % (self.__class__.__name__, + self.definition_cfg, self.message) + + +class ResourcesDefinition(object): + + MANDATORY_FIELDS = {'resource_type': six.string_types, + 'metrics': list} + + def __init__(self, definition_cfg, default_archive_policy, + legacy_archive_policy_defintion): + self._default_archive_policy = default_archive_policy + self._legacy_archive_policy_defintion = legacy_archive_policy_defintion + self.cfg = definition_cfg + self._validate() + + def match(self, metric_name): + for t in self.cfg['metrics']: + if fnmatch.fnmatch(metric_name, t): + return True + return False + + def attributes(self, sample): + attrs = {} + for attribute_info in self.cfg.get('attributes', []): + for attr, field in attribute_info.items(): + value = self._parse_field(field, sample) + if value is not None: + attrs[attr] = value + return attrs + + def metrics(self): + metrics = {} + for t in self.cfg['metrics']: + archive_policy = self.cfg.get( + 'archive_policy', + self._legacy_archive_policy_defintion.get(t)) + metrics[t] = dict(archive_policy_name=archive_policy or + self._default_archive_policy) + return metrics + + def _parse_field(self, field, sample): + # TODO(sileht): share this with + # https://review.openstack.org/#/c/197633/ + if not field: + return + if isinstance(field, six.integer_types): + return field + try: + parts = jsonpath_rw.parse(field) + except Exception as e: + raise ResourcesDefinitionException( + _LE("Parse error in JSONPath specification " + "'%(jsonpath)s': %(err)s") + % dict(jsonpath=field, err=e), self.cfg) + values = [match.value for match in parts.find(sample) + if match.value is not None] + if values: + return values[0] + + def _validate(self): + for field, field_type in self.MANDATORY_FIELDS.items(): + if field not in self.cfg: + raise ResourcesDefinitionException( + _LE("Required field %s not specified") % field, self.cfg) + if not isinstance(self.cfg[field], field_type): + raise ResourcesDefinitionException( + _LE("Required field %(field)s should be a %(type)s") % + {'field': field, 'type': field_type}, self.cfg) + + class GnocchiDispatcher(dispatcher.Base): def __init__(self, conf): super(GnocchiDispatcher, self).__init__(conf) @@ -101,12 +198,8 @@ class GnocchiDispatcher(dispatcher.Base): conf.dispatcher_gnocchi.filter_service_activity) self._ks_client = keystone_client.get_client() self.gnocchi_url = conf.dispatcher_gnocchi.url - self.gnocchi_archive_policy_default = ( - conf.dispatcher_gnocchi.archive_policy) self.gnocchi_archive_policy_data = self._load_archive_policy(conf) - self.mgmr = stevedore.dispatch.DispatchExtensionManager( - 'ceilometer.dispatcher.resource', lambda x: True, - invoke_on_load=True) + self.resources_definition = self._load_resources_definitions(conf) self._gnocchi_project_id = None self._gnocchi_project_id_lock = threading.Lock() @@ -119,8 +212,36 @@ class GnocchiDispatcher(dispatcher.Base): 'X-Auth-Token': self._ks_client.auth_token, } - def _load_archive_policy(self, conf): - policy_config_file = self._get_config_file(conf) + # TODO(sileht): Share yaml loading with + # event converter and declarative notification + + @staticmethod + def _get_config_file(conf, config_file): + if not os.path.exists(config_file): + config_file = cfg.CONF.find_file(config_file) + return config_file + + @classmethod + def _load_resources_definitions(cls, conf): + res_def_file = cls._get_config_file( + conf, conf.dispatcher_gnocchi.resources_definition_file) + data = {} + if res_def_file is not None: + with open(res_def_file) as data_file: + try: + data = yaml.safe_load(data_file) + except ValueError: + data = {} + + legacy_archive_policies = cls._load_archive_policy(conf) + return [ResourcesDefinition(r, conf.dispatcher_gnocchi.archive_policy, + legacy_archive_policies) + for r in data.get('resources', [])] + + @classmethod + def _load_archive_policy(cls, conf): + policy_config_file = cls._get_config_file( + conf, conf.dispatcher_gnocchi.archive_policy_file) data = {} if policy_config_file is not None: with open(policy_config_file) as data_file: @@ -128,35 +249,7 @@ class GnocchiDispatcher(dispatcher.Base): data = yaml.safe_load(data_file) except ValueError: data = {} - return data - - def get_archive_policy(self, metric_name): - - archive_policy = {} - if self.gnocchi_archive_policy_data is not None: - policy_match = self._match_metric(metric_name) - archive_policy['archive_policy_name'] = ( - policy_match or self.gnocchi_archive_policy_default) - else: - LOG.debug(_("No archive policy file found!" - " Using default config.")) - archive_policy['archive_policy_name'] = ( - self.gnocchi_archive_policy_default) - - return archive_policy - - @staticmethod - def _get_config_file(conf): - config_file = conf.dispatcher_gnocchi.archive_policy_file - if not os.path.exists(config_file): - config_file = cfg.CONF.find_file(config_file) - return config_file - - def _match_metric(self, metric_name): - for metric, policy in self.gnocchi_archive_policy_data.items(): - # Support wild cards such as disk.* - if fnmatch.fnmatch(metric_name, metric): - return policy + return LegacyArchivePolicyDefinition(data) @property def gnocchi_project_id(self): @@ -192,16 +285,25 @@ class GnocchiDispatcher(dispatcher.Base): return self._gnocchi_api + def _is_swift_account_sample(self, sample): + return bool([rd for rd in self.resources_definition + if rd.cfg['resource_type'] == 'swift_account' + and rd.match(sample['counter_name'])]) + def _is_gnocchi_activity(self, sample): return (self.filter_service_activity and ( # avoid anything from the user used by gnocchi sample['project_id'] == self.gnocchi_project_id or # avoid anything in the swift account used by gnocchi (sample['resource_id'] == self.gnocchi_project_id and - sample['counter_name'] in - self.mgmr['swift_account'].obj.get_metrics_names()) + self._is_swift_account_sample(sample)) )) + def _get_resource_definition(self, metric_name): + for rd in self.resources_definition: + if rd.match(metric_name): + return rd + def record_metering_data(self, data): # NOTE(sileht): skip sample generated by gnocchi itself data = [s for s in data if not self._is_gnocchi_activity(s)] @@ -223,11 +325,15 @@ class GnocchiDispatcher(dispatcher.Base): list(samples_of_resource), key=operator.itemgetter('counter_name')) for metric_name, samples in metric_grouped_samples: - for ext in self.mgmr: - if metric_name in ext.obj.get_metrics_names(): - self._process_samples( - ext, resource_id, metric_name, list(samples), - resource_need_to_be_updated) + samples = list(samples) + rd = self._get_resource_definition(metric_name) + if rd: + self._process_samples(rd, resource_id, metric_name, + samples, + resource_need_to_be_updated) + else: + LOG.warn("metric %s is not handled by gnocchi" % + metric_name) # FIXME(sileht): Does it reasonable to skip the resource # update here ? Does differents kind of counter_name @@ -238,9 +344,9 @@ class GnocchiDispatcher(dispatcher.Base): # resource_need_to_be_updated = False @log_and_ignore_unexpected_workflow_error - def _process_samples(self, ext, resource_id, metric_name, samples, + def _process_samples(self, resource_def, resource_id, metric_name, samples, resource_need_to_be_updated): - resource_type = ext.name + resource_type = resource_def.cfg['resource_type'] measure_attributes = [{'timestamp': sample['timestamp'], 'value': sample['counter_volume']} for sample in samples] @@ -253,14 +359,15 @@ class GnocchiDispatcher(dispatcher.Base): # they more chance that the resource doesn't exists than the metric # is missing, the should be reduce the number of resource API call resource_attributes = self._get_resource_attributes( - ext, resource_id, metric_name, samples) + resource_def, resource_id, metric_name, samples) try: self._create_resource(resource_type, resource_id, resource_attributes) except ResourceAlreadyExists: try: + archive_policy = (resource_def.metrics()[metric_name]) self._create_metric(resource_type, resource_id, - metric_name) + metric_name, archive_policy) except MetricAlreadyExists: # NOTE(sileht): Just ignore the metric have been created in # the meantime. @@ -278,25 +385,22 @@ class GnocchiDispatcher(dispatcher.Base): if resource_need_to_be_updated: resource_attributes = self._get_resource_attributes( - ext, resource_id, metric_name, samples, for_update=True) + resource_def, resource_id, metric_name, samples, + for_update=True) if resource_attributes: self._update_resource(resource_type, resource_id, resource_attributes) - def _get_resource_attributes(self, ext, resource_id, metric_name, samples, - for_update=False): + def _get_resource_attributes(self, resource_def, resource_id, metric_name, + samples, for_update=False): # FIXME(sileht): Should I merge attibutes of all samples ? # Or keep only the last one is sufficient ? - attributes = ext.obj.get_resource_extra_attributes( - samples[-1]) + attributes = resource_def.attributes(samples[-1]) if not for_update: attributes["id"] = resource_id attributes["user_id"] = samples[-1]['user_id'] attributes["project_id"] = samples[-1]['project_id'] - attributes["metrics"] = dict( - (metric_name, self.get_archive_policy(metric_name)) - for metric_name in ext.obj.get_metrics_names() - ) + attributes["metrics"] = resource_def.metrics() return attributes def _post_measure(self, resource_type, resource_id, metric_name, @@ -365,8 +469,9 @@ class GnocchiDispatcher(dispatcher.Base): else: LOG.debug("Resource %s updated", resource_id) - def _create_metric(self, resource_type, resource_id, metric_name): - params = {metric_name: self.get_archive_policy(metric_name)} + def _create_metric(self, resource_type, resource_id, metric_name, + archive_policy): + params = {metric_name: archive_policy} r = self.gnocchi_api.post("%s/v1/resource/%s/%s/metric" % (self.gnocchi_url, resource_type, resource_id), diff --git a/ceilometer/dispatcher/resources/__init__.py b/ceilometer/dispatcher/resources/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/ceilometer/dispatcher/resources/base.py b/ceilometer/dispatcher/resources/base.py deleted file mode 100644 index 37d1ed70..00000000 --- a/ceilometer/dispatcher/resources/base.py +++ /dev/null @@ -1,40 +0,0 @@ -# -# Copyright 2014 eNovance -# -# Authors: Mehdi Abaakouk -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import abc - -import six - - -@six.add_metaclass(abc.ABCMeta) -class ResourceBase(object): - """Base class for resource.""" - - @abc.abstractmethod - def get_resource_extra_attributes(self, sample): - """Extract the metadata from a ceilometer sample. - - :param sample: The ceilometer sample - :returns: the resource attributes - """ - - @abc.abstractmethod - def get_metrics_names(self): - """Return the metric handled by this resource. - - :returns: list of metric names - """ diff --git a/ceilometer/dispatcher/resources/ceph_account.py b/ceilometer/dispatcher/resources/ceph_account.py deleted file mode 100644 index dae50f94..00000000 --- a/ceilometer/dispatcher/resources/ceph_account.py +++ /dev/null @@ -1,31 +0,0 @@ -# -# Copyright 2015 Mirantis Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -from ceilometer.dispatcher.resources import base - - -class CephAccount(base.ResourceBase): - @staticmethod - def get_resource_extra_attributes(sample): - return {} - - @staticmethod - def get_metrics_names(): - return ['radosgw.api.request', - 'radosgw.objects.size', - 'radosgw.objects', - 'radosgw.objects.containers', - 'radosgw.containers.objects', - 'radosgw.containers.objects.size', - ] diff --git a/ceilometer/dispatcher/resources/identity.py b/ceilometer/dispatcher/resources/identity.py deleted file mode 100644 index 2cee8519..00000000 --- a/ceilometer/dispatcher/resources/identity.py +++ /dev/null @@ -1,44 +0,0 @@ -# -# Copyright 2015 Mirantis Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -from ceilometer.dispatcher.resources import base - - -class Identity(base.ResourceBase): - @staticmethod - def get_resource_extra_attributes(sample): - return {} - - @staticmethod - def get_metrics_names(): - return ['identity.authenticate.success', - 'identity.authenticate.pending', - 'identity.authenticate.failure', - 'identity.user.created', - 'identity.user.deleted', - 'identity.user.updated', - 'identity.group.created', - 'identity.group.deleted', - 'identity.group.updated', - 'identity.role.created', - 'identity.role.deleted', - 'identity.role.updated', - 'identity.project.created', - 'identity.project.deleted', - 'identity.project.updated', - 'identity.trust.created', - 'identity.trust.deleted', - 'identity.role_assignment.created', - 'identity.role_assignment.deleted', - ] diff --git a/ceilometer/dispatcher/resources/image.py b/ceilometer/dispatcher/resources/image.py deleted file mode 100644 index 66de2b44..00000000 --- a/ceilometer/dispatcher/resources/image.py +++ /dev/null @@ -1,30 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from ceilometer.dispatcher.resources import base - - -class Image(base.ResourceBase): - @staticmethod - def get_resource_extra_attributes(sample): - metadata = sample['resource_metadata'] - params = { - "name": metadata['name'], - "container_format": metadata["container_format"], - "disk_format": metadata["disk_format"] - } - return params - - @staticmethod - def get_metrics_names(): - return ['image', - 'image.size'] diff --git a/ceilometer/dispatcher/resources/instance.py b/ceilometer/dispatcher/resources/instance.py deleted file mode 100644 index 61251625..00000000 --- a/ceilometer/dispatcher/resources/instance.py +++ /dev/null @@ -1,54 +0,0 @@ -# -# Copyright 2014 eNovance -# -# Authors: Julien Danjou -# Mehdi Abaakouk -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from ceilometer.dispatcher.resources import base - - -class Instance(base.ResourceBase): - @staticmethod - def get_resource_extra_attributes(sample): - metadata = sample['resource_metadata'] - params = { - "host": metadata['host'], - "image_ref": metadata['image_ref_url'], - "display_name": metadata['display_name'], - } - if "instance_flavor_id" in metadata: - params["flavor_id"] = metadata['instance_flavor_id'] - else: - # NOTE(sileht): instance.exists have the flavor here - params["flavor_id"] = metadata["flavor"]["id"] - - server_group = metadata.get('user_metadata', {}).get('server_group') - if server_group: - params["server_group"] = server_group - - return params - - @staticmethod - def get_metrics_names(): - # NOTE(sileht): Can we generate the list by loading ceilometer - # plugin ? - return ['instance', - 'disk.root.size', - 'disk.ephemeral.size', - 'memory', - 'memory.usage', - 'vcpus', - 'cpu', - 'cpu_util'] diff --git a/ceilometer/dispatcher/resources/ipmi.py b/ceilometer/dispatcher/resources/ipmi.py deleted file mode 100644 index b7f92fdb..00000000 --- a/ceilometer/dispatcher/resources/ipmi.py +++ /dev/null @@ -1,30 +0,0 @@ -# -# Copyright 2015 Mirantis Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -from ceilometer.dispatcher.resources import base - - -class IPMI(base.ResourceBase): - @staticmethod - def get_resource_extra_attributes(sample): - return {} - - @staticmethod - def get_metrics_names(): - return ['hardware.ipmi.node.power', - 'hardware.ipmi.node.temperature', - 'hardware.ipmi.node.fan', - 'hardware.ipmi.node.current', - 'hardware.ipmi.node.voltage', - ] diff --git a/ceilometer/dispatcher/resources/network.py b/ceilometer/dispatcher/resources/network.py deleted file mode 100644 index edf854f9..00000000 --- a/ceilometer/dispatcher/resources/network.py +++ /dev/null @@ -1,41 +0,0 @@ -# -# Copyright 2015 Mirantis Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -from ceilometer.dispatcher.resources import base - - -class Network(base.ResourceBase): - @staticmethod - def get_resource_extra_attributes(sample): - return {} - - @staticmethod - def get_metrics_names(): - return ['bandwidth', - 'network', - 'network.create', - 'network.update', - 'subnet', - 'subnet.create', - 'subnet.update', - 'port', - 'port.create', - 'port.update', - 'router', - 'router.create', - 'router.update', - 'ip.floating', - 'ip.floating.create', - 'ip.floating.update', - ] diff --git a/ceilometer/dispatcher/resources/orchestration.py b/ceilometer/dispatcher/resources/orchestration.py deleted file mode 100644 index f4af3bca..00000000 --- a/ceilometer/dispatcher/resources/orchestration.py +++ /dev/null @@ -1,30 +0,0 @@ -# -# Copyright 2015 Mirantis Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -from ceilometer.dispatcher.resources import base - - -class Stack(base.ResourceBase): - @staticmethod - def get_resource_extra_attributes(sample): - return {} - - @staticmethod - def get_metrics_names(): - return ['stack.create', - 'stack.update', - 'stack.delete', - 'stack.resume', - 'stack.suspend', - ] diff --git a/ceilometer/dispatcher/resources/swift_account.py b/ceilometer/dispatcher/resources/swift_account.py deleted file mode 100644 index e8538ecf..00000000 --- a/ceilometer/dispatcher/resources/swift_account.py +++ /dev/null @@ -1,33 +0,0 @@ -# -# Copyright 2014 eNovance -# -# Authors: Julien Danjou -# Mehdi Abaakouk -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -from ceilometer.dispatcher.resources import base - - -class SwiftAccount(base.ResourceBase): - @staticmethod - def get_resource_extra_attributes(sample): - return {} - - @staticmethod - def get_metrics_names(): - return ['storage.objects.incoming.bytes', - 'storage.objects.outgoing.bytes', - 'storage.api.request', - 'storage.objects.size', - 'storage.objects', - 'storage.objects.containers'] diff --git a/ceilometer/dispatcher/resources/volume.py b/ceilometer/dispatcher/resources/volume.py deleted file mode 100644 index d6abf91a..00000000 --- a/ceilometer/dispatcher/resources/volume.py +++ /dev/null @@ -1,38 +0,0 @@ -# -# Copyright 2015 Mirantis Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from ceilometer.dispatcher.resources import base - - -class Volume(base.ResourceBase): - @staticmethod - def get_resource_extra_attributes(sample): - metadata = sample['resource_metadata'] - params = { - "display_name": metadata['display_name'], - } - return params - - @staticmethod - def get_metrics_names(): - return ['volume', - 'volume.size', - 'volume.create', - 'volume.delete', - 'volume.update', - 'volume.resize', - 'volume.attach', - 'volume.detach', - ] diff --git a/ceilometer/tests/dispatcher/test_gnocchi.py b/ceilometer/tests/dispatcher/test_gnocchi.py index 65380e38..70318b61 100644 --- a/ceilometer/tests/dispatcher/test_gnocchi.py +++ b/ceilometer/tests/dispatcher/test_gnocchi.py @@ -16,13 +16,15 @@ # under the License. import json +import os import uuid import mock from oslo_config import fixture as config_fixture -from oslotest import base +from oslo_utils import fileutils from oslotest import mockpatch import requests +import six import six.moves.urllib.parse as urlparse import tempfile import testscenarios @@ -30,6 +32,7 @@ import yaml from ceilometer.dispatcher import gnocchi from ceilometer import service as ceilometer_service +from ceilometer.tests import base load_tests = testscenarios.load_tests_apply_scenarios @@ -51,6 +54,11 @@ class DispatcherTest(base.BaseTestCase): super(DispatcherTest, self).setUp() self.conf = self.useFixture(config_fixture.Config()) ceilometer_service.prepare_service([]) + self.conf.config( + resources_definition_file=self.path_get( + 'etc/ceilometer/gnocchi_resources.yaml'), + group="dispatcher_gnocchi" + ) self.resource_id = str(uuid.uuid4()) self.samples = [{ 'counter_name': 'disk.root.size', @@ -92,12 +100,43 @@ class DispatcherTest(base.BaseTestCase): return_value=ks_client)) self.conf.conf.dispatcher_gnocchi.filter_service_activity = True - def test_extensions_load(self): + def test_config_load(self): self.conf.config(filter_service_activity=False, group='dispatcher_gnocchi') d = gnocchi.GnocchiDispatcher(self.conf.conf) - self.assertIn('instance', d.mgmr.names()) - self.assertIn('volume', d.mgmr.names()) + names = [rd.cfg['resource_type'] for rd in d.resources_definition] + self.assertIn('instance', names) + self.assertIn('volume', names) + + def test_broken_config_load(self): + contents = [("---\n" + "resources:\n" + " - resource_type: foobar\n"), + ("---\n" + "resources:\n" + " - resource_type: 0\n"), + ("---\n" + "resources:\n" + " - sample_types: ['foo', 'bar']\n"), + ("---\n" + "resources:\n" + " - sample_types: foobar\n" + " - resource_type: foobar\n"), + ] + + for content in contents: + if six.PY3: + content = content.encode('utf-8') + + temp = fileutils.write_to_tempfile(content=content, + prefix='gnocchi_resources', + suffix='.yaml') + self.addCleanup(os.remove, temp) + self.conf.config(filter_service_activity=False, + resources_definition_file=temp, + group='dispatcher_gnocchi') + self.assertRaises(gnocchi.ResourcesDefinitionException, + gnocchi.GnocchiDispatcher, self.conf.conf) @mock.patch('ceilometer.dispatcher.gnocchi.GnocchiDispatcher' '._process_samples') @@ -110,10 +149,6 @@ class DispatcherTest(base.BaseTestCase): expected_samples, True, ) - def test_archive_policy_default(self): - d = gnocchi.GnocchiDispatcher(self.conf.conf) - self.assertEqual(d.gnocchi_archive_policy_default, "low") - def test_archive_policy_map_config(self): archive_policy_map = yaml.dump({ 'foo.*': 'low' @@ -125,9 +160,8 @@ class DispatcherTest(base.BaseTestCase): self.conf.conf.dispatcher_gnocchi.archive_policy_file = ( archive_policy_cfg_file.name) d = gnocchi.GnocchiDispatcher(self.conf.conf) - self.assertEqual( - d.get_archive_policy( - 'foo.disk.rate')['archive_policy_name'], "low") + legacy = d._load_archive_policy(self.conf.conf) + self.assertEqual(legacy.get('foo.disk.rate'), "low") archive_policy_cfg_file.close() def test_activity_filter_match_project_id(self): @@ -265,6 +299,12 @@ class DispatcherWorkflowTest(base.BaseTestCase, return_value=ks_client)) ceilometer_service.prepare_service([]) + self.conf.config( + resources_definition_file=self.path_get( + 'etc/ceilometer/gnocchi_resources.yaml'), + group="dispatcher_gnocchi" + ) + self.dispatcher = gnocchi.GnocchiDispatcher(self.conf.conf) self.sample['resource_id'] = str(uuid.uuid4()) diff --git a/devstack/plugin.sh b/devstack/plugin.sh index 7a501c74..8d3268f5 100644 --- a/devstack/plugin.sh +++ b/devstack/plugin.sh @@ -255,6 +255,8 @@ function configure_ceilometer { cp $CEILOMETER_DIR/etc/ceilometer/event_pipeline.yaml $CEILOMETER_CONF_DIR cp $CEILOMETER_DIR/etc/ceilometer/api_paste.ini $CEILOMETER_CONF_DIR cp $CEILOMETER_DIR/etc/ceilometer/event_definitions.yaml $CEILOMETER_CONF_DIR + cp $CEILOMETER_DIR/etc/ceilometer/gnocchi_archive_policy_map.yaml $CEILOMETER_CONF_DIR + cp $CEILOMETER_DIR/etc/ceilometer/gnocchi_resources.yaml $CEILOMETER_CONF_DIR if [ "$CEILOMETER_PIPELINE_INTERVAL" ]; then sed -i "s/interval:.*/interval: ${CEILOMETER_PIPELINE_INTERVAL}/" $CEILOMETER_CONF_DIR/pipeline.yaml diff --git a/etc/ceilometer/gnocchi_resources.yaml b/etc/ceilometer/gnocchi_resources.yaml new file mode 100644 index 00000000..a7509312 --- /dev/null +++ b/etc/ceilometer/gnocchi_resources.yaml @@ -0,0 +1,121 @@ +--- + +resources: + - resource_type: identity + archive_policy: low + metrics: + - 'identity.authenticate.success' + - 'identity.authenticate.pending' + - 'identity.authenticate.failure' + - 'identity.user.created' + - 'identity.user.deleted' + - 'identity.user.updated' + - 'identity.group.created' + - 'identity.group.deleted' + - 'identity.group.updated' + + - resource_type: identity + archive_policy: low + metrics: + - 'identity.role.created' + - 'identity.role.deleted' + - 'identity.role.updated' + - 'identity.project.created' + - 'identity.project.deleted' + - 'identity.project.updated' + - 'identity.trust.created' + - 'identity.trust.deleted' + - 'identity.role_assignment.created' + - 'identity.role_assignment.deleted' + + - resource_type: ceph_account + metrics: + - 'stack.create' + - 'stack.update' + - 'stack.delete' + - 'stack.resume' + - 'stack.suspend' + + - resource_type: instance + metrics: + - 'instance' + - 'disk.root.size' + - 'disk.ephemeral.size' + - 'memory' + - 'memory.usage' + - 'vcpus' + - 'cpu' + - 'cpu_util' + attributes: + - host: resource_metadata.host + - image_ref: resource_metadata.image_ref_url + - display_name: resource_metadata.display_name + - flavor_id: resource_metadata.instance_flavor_id + - flavor_id: resource_metadata.flavor.id + - server_group: resource_metadata.user_metadata.server_group + + - resource_type: image + metrics: + - 'image' + - 'image.size' + attributes: + - name: resource_metadata.name + - container_format: resource_metadata.container_format + - disk_format: resource_metadata.disk_format + + - resource_type: ipmi + metrics: + - 'hardware.ipmi.node.power' + - 'hardware.ipmi.node.temperature' + - 'hardware.ipmi.node.fan' + - 'hardware.ipmi.node.current' + - 'hardware.ipmi.node.voltage' + + - resource_type: network + metrics: + - 'bandwidth' + - 'network' + - 'network.create' + - 'network.update' + - 'subnet' + - 'subnet.create' + - 'subnet.update' + - 'port' + - 'port.create' + - 'port.update' + - 'router' + - 'router.create' + - 'router.update' + - 'ip.floating' + - 'ip.floating.create' + - 'ip.floating.update' + + - resource_type: orchestration + metrics: + - 'stack.create' + - 'stack.update' + - 'stack.delete' + - 'stack.resume' + - 'stack.suspend' + + - resource_type: swift_account + metrics: + - 'storage.objects.incoming.bytes' + - 'storage.objects.outgoing.bytes' + - 'storage.api.request' + - 'storage.objects.size' + - 'storage.objects' + - 'storage.objects.containers' + + - resource_type: volume + metrics: + - 'volume' + - 'volume.size' + - 'volume.create' + - 'volume.delete' + - 'volume.update' + - 'volume.resize' + - 'volume.attach' + - 'volume.detach' + attributes: + - display_name: resource_metadata.display_name diff --git a/setup.cfg b/setup.cfg index d678b4c7..88d02c90 100644 --- a/setup.cfg +++ b/setup.cfg @@ -346,17 +346,6 @@ ceilometer.dispatcher = http = ceilometer.dispatcher.http:HttpDispatcher gnocchi = ceilometer.dispatcher.gnocchi:GnocchiDispatcher -ceilometer.dispatcher.resource = - instance = ceilometer.dispatcher.resources.instance:Instance - swift_account = ceilometer.dispatcher.resources.swift_account:SwiftAccount - volume = ceilometer.dispatcher.resources.volume:Volume - ceph_account = ceilometer.dispatcher.resources.ceph_account:CephAccount - network = ceilometer.dispatcher.resources.network:Network - identity = ceilometer.dispatcher.resources.identity:Identity - ipmi = ceilometer.dispatcher.resources.ipmi:IPMI - stack = ceilometer.dispatcher.resources.orchestration:Stack - image = ceilometer.dispatcher.resources.image:Image - network.statistics.drivers = opendaylight = ceilometer.network.statistics.opendaylight.driver:OpenDayLightDriver opencontrail = ceilometer.network.statistics.opencontrail.driver:OpencontrailDriver