Move gnocchi resources definition in yaml file

Transformation of samples to gnocchi resources/metrics is
just json2json think, so do it in a declarative ways like we do
for event and declarative notification.

Implements blueprint gnocchi-declarative-resources

Change-Id: I5a202c30614d06821063e243d4e2330736aba5fd

ceilometer/dispatcher/gnocchi.py

@@ -16,22 +16,22 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 import fnmatch
-import threading
-
 import itertools
 import json
 import operator
 import os
-import yaml
+import threading
 
-from ceilometer import dispatcher
-from ceilometer.i18n import _
-from ceilometer import keystone_client
+import jsonpath_rw
 from oslo_config import cfg
 from oslo_log import log
 import requests
 import six
-import stevedore.dispatch
+import yaml
+
+from ceilometer import dispatcher
+from ceilometer.i18n import _, _LE
+from ceilometer import keystone_client
 
 LOG = log.getLogger(__name__)
 
@@ -53,8 +53,13 @@ dispatcher_opts = [
                'create a new metric.'),
     cfg.StrOpt('archive_policy_file',
                default='gnocchi_archive_policy_map.yaml',
+               deprecated_for_removal=True,
                help=_('The Yaml file that defines per metric archive '
                       'policies.')),
+    cfg.StrOpt('resources_definition_file',
+               default='gnocchi_resources.yaml',
+               help=_('The Yaml file that defines mapping between samples '
+                      'and gnocchi resources/metrics')),
 ]
 
 cfg.CONF.register_opts(dispatcher_opts, group="dispatcher_gnocchi")
@@ -93,6 +98,98 @@ def log_and_ignore_unexpected_workflow_error(func):
     return log_and_ignore
 
 
+class LegacyArchivePolicyDefinition(object):
+    def __init__(self, definition_cfg):
+        self.cfg = definition_cfg
+        if self.cfg is None:
+            LOG.debug(_("No archive policy file found!"
+                      " Using default config."))
+
+    def get(self, metric_name):
+        if self.cfg is not None:
+            for metric, policy in self.cfg.items():
+                # Support wild cards such as disk.*
+                if fnmatch.fnmatch(metric_name, metric):
+                    return policy
+
+
+class ResourcesDefinitionException(Exception):
+    def __init__(self, message, definition_cfg):
+        super(ResourcesDefinitionException, self).__init__(message)
+        self.definition_cfg = definition_cfg
+
+    def __str__(self):
+        return '%s %s: %s' % (self.__class__.__name__,
+                              self.definition_cfg, self.message)
+
+
+class ResourcesDefinition(object):
+
+    MANDATORY_FIELDS = {'resource_type': six.string_types,
+                        'metrics': list}
+
+    def __init__(self, definition_cfg, default_archive_policy,
+                 legacy_archive_policy_defintion):
+        self._default_archive_policy = default_archive_policy
+        self._legacy_archive_policy_defintion = legacy_archive_policy_defintion
+        self.cfg = definition_cfg
+        self._validate()
+
+    def match(self, metric_name):
+        for t in self.cfg['metrics']:
+            if fnmatch.fnmatch(metric_name, t):
+                return True
+        return False
+
+    def attributes(self, sample):
+        attrs = {}
+        for attribute_info in self.cfg.get('attributes', []):
+            for attr, field in attribute_info.items():
+                value = self._parse_field(field, sample)
+                if value is not None:
+                    attrs[attr] = value
+        return attrs
+
+    def metrics(self):
+        metrics = {}
+        for t in self.cfg['metrics']:
+            archive_policy = self.cfg.get(
+                'archive_policy',
+                self._legacy_archive_policy_defintion.get(t))
+            metrics[t] = dict(archive_policy_name=archive_policy or
+                              self._default_archive_policy)
+        return metrics
+
+    def _parse_field(self, field, sample):
+        # TODO(sileht): share this with
+        # https://review.openstack.org/#/c/197633/
+        if not field:
+            return
+        if isinstance(field, six.integer_types):
+            return field
+        try:
+            parts = jsonpath_rw.parse(field)
+        except Exception as e:
+            raise ResourcesDefinitionException(
+                _LE("Parse error in JSONPath specification "
+                    "'%(jsonpath)s': %(err)s")
+                % dict(jsonpath=field, err=e), self.cfg)
+        values = [match.value for match in parts.find(sample)
+                  if match.value is not None]
+        if values:
+            return values[0]
+
+    def _validate(self):
+        for field, field_type in self.MANDATORY_FIELDS.items():
+            if field not in self.cfg:
+                raise ResourcesDefinitionException(
+                    _LE("Required field %s not specified") % field, self.cfg)
+            if not isinstance(self.cfg[field], field_type):
+                raise ResourcesDefinitionException(
+                    _LE("Required field %(field)s should be a %(type)s") %
+                    {'field': field, 'type': field_type}, self.cfg)
+
+
 class GnocchiDispatcher(dispatcher.Base):
     def __init__(self, conf):
         super(GnocchiDispatcher, self).__init__(conf)
@@ -101,12 +198,8 @@ class GnocchiDispatcher(dispatcher.Base):
             conf.dispatcher_gnocchi.filter_service_activity)
         self._ks_client = keystone_client.get_client()
         self.gnocchi_url = conf.dispatcher_gnocchi.url
-        self.gnocchi_archive_policy_default = (
-            conf.dispatcher_gnocchi.archive_policy)
         self.gnocchi_archive_policy_data = self._load_archive_policy(conf)
-        self.mgmr = stevedore.dispatch.DispatchExtensionManager(
-            'ceilometer.dispatcher.resource', lambda x: True,
-            invoke_on_load=True)
+        self.resources_definition = self._load_resources_definitions(conf)
 
         self._gnocchi_project_id = None
         self._gnocchi_project_id_lock = threading.Lock()
@@ -119,8 +212,36 @@ class GnocchiDispatcher(dispatcher.Base):
             'X-Auth-Token': self._ks_client.auth_token,
         }
 
-    def _load_archive_policy(self, conf):
-        policy_config_file = self._get_config_file(conf)
+    # TODO(sileht): Share yaml loading with
+    # event converter and declarative notification
+
+    @staticmethod
+    def _get_config_file(conf, config_file):
+        if not os.path.exists(config_file):
+            config_file = cfg.CONF.find_file(config_file)
+        return config_file
+
+    @classmethod
+    def _load_resources_definitions(cls, conf):
+        res_def_file = cls._get_config_file(
+            conf, conf.dispatcher_gnocchi.resources_definition_file)
+        data = {}
+        if res_def_file is not None:
+            with open(res_def_file) as data_file:
+                try:
+                    data = yaml.safe_load(data_file)
+                except ValueError:
+                    data = {}
+
+        legacy_archive_policies = cls._load_archive_policy(conf)
+        return [ResourcesDefinition(r, conf.dispatcher_gnocchi.archive_policy,
+                                    legacy_archive_policies)
+                for r in data.get('resources', [])]
+
+    @classmethod
+    def _load_archive_policy(cls, conf):
+        policy_config_file = cls._get_config_file(
+            conf, conf.dispatcher_gnocchi.archive_policy_file)
         data = {}
         if policy_config_file is not None:
             with open(policy_config_file) as data_file:
@@ -128,35 +249,7 @@ class GnocchiDispatcher(dispatcher.Base):
                     data = yaml.safe_load(data_file)
                 except ValueError:
                     data = {}
-        return data
-
-    def get_archive_policy(self, metric_name):
-
-        archive_policy = {}
-        if self.gnocchi_archive_policy_data is not None:
-            policy_match = self._match_metric(metric_name)
-            archive_policy['archive_policy_name'] = (
-                policy_match or self.gnocchi_archive_policy_default)
-        else:
-            LOG.debug(_("No archive policy file found!"
-                      " Using default config."))
-            archive_policy['archive_policy_name'] = (
-                self.gnocchi_archive_policy_default)
-
-        return archive_policy
-
-    @staticmethod
-    def _get_config_file(conf):
-        config_file = conf.dispatcher_gnocchi.archive_policy_file
-        if not os.path.exists(config_file):
-            config_file = cfg.CONF.find_file(config_file)
-        return config_file
-
-    def _match_metric(self, metric_name):
-        for metric, policy in self.gnocchi_archive_policy_data.items():
-            # Support wild cards such as disk.*
-            if fnmatch.fnmatch(metric_name, metric):
-                return policy
+        return LegacyArchivePolicyDefinition(data)
 
     @property
     def gnocchi_project_id(self):
@@ -192,16 +285,25 @@ class GnocchiDispatcher(dispatcher.Base):
 
             return self._gnocchi_api
 
+    def _is_swift_account_sample(self, sample):
+        return bool([rd for rd in self.resources_definition
+                     if rd.cfg['resource_type'] == 'swift_account'
+                     and rd.match(sample['counter_name'])])
+
     def _is_gnocchi_activity(self, sample):
         return (self.filter_service_activity and (
             # avoid anything from the user used by gnocchi
             sample['project_id'] == self.gnocchi_project_id or
             # avoid anything in the swift account used by gnocchi
             (sample['resource_id'] == self.gnocchi_project_id and
-             sample['counter_name'] in
-             self.mgmr['swift_account'].obj.get_metrics_names())
+             self._is_swift_account_sample(sample))
         ))
 
+    def _get_resource_definition(self, metric_name):
+        for rd in self.resources_definition:
+            if rd.match(metric_name):
+                return rd
+
     def record_metering_data(self, data):
         # NOTE(sileht): skip sample generated by gnocchi itself
         data = [s for s in data if not self._is_gnocchi_activity(s)]
@@ -223,11 +325,15 @@ class GnocchiDispatcher(dispatcher.Base):
                 list(samples_of_resource),
                 key=operator.itemgetter('counter_name'))
             for metric_name, samples in metric_grouped_samples:
-                for ext in self.mgmr:
-                    if metric_name in ext.obj.get_metrics_names():
-                        self._process_samples(
-                            ext, resource_id, metric_name, list(samples),
+                samples = list(samples)
+                rd = self._get_resource_definition(metric_name)
+                if rd:
+                    self._process_samples(rd, resource_id, metric_name,
+                                          samples,
                                           resource_need_to_be_updated)
+                else:
+                    LOG.warn("metric %s is not handled by gnocchi" %
+                             metric_name)
 
                 # FIXME(sileht): Does it reasonable to skip the resource
                 # update here ? Does differents kind of counter_name
@@ -238,9 +344,9 @@ class GnocchiDispatcher(dispatcher.Base):
                 # resource_need_to_be_updated = False
 
     @log_and_ignore_unexpected_workflow_error
-    def _process_samples(self, ext, resource_id, metric_name, samples,
+    def _process_samples(self, resource_def, resource_id, metric_name, samples,
                          resource_need_to_be_updated):
-        resource_type = ext.name
+        resource_type = resource_def.cfg['resource_type']
         measure_attributes = [{'timestamp': sample['timestamp'],
                                'value': sample['counter_volume']}
                               for sample in samples]
@@ -253,14 +359,15 @@ class GnocchiDispatcher(dispatcher.Base):
             # they more chance that the resource doesn't exists than the metric
             # is missing, the should be reduce the number of resource API call
             resource_attributes = self._get_resource_attributes(
-                ext, resource_id, metric_name, samples)
+                resource_def, resource_id, metric_name, samples)
             try:
                 self._create_resource(resource_type, resource_id,
                                       resource_attributes)
             except ResourceAlreadyExists:
                 try:
+                    archive_policy = (resource_def.metrics()[metric_name])
                     self._create_metric(resource_type, resource_id,
-                                        metric_name)
+                                        metric_name, archive_policy)
                 except MetricAlreadyExists:
                     # NOTE(sileht): Just ignore the metric have been created in
                     # the meantime.
@@ -278,25 +385,22 @@ class GnocchiDispatcher(dispatcher.Base):
 
         if resource_need_to_be_updated:
             resource_attributes = self._get_resource_attributes(
-                ext, resource_id, metric_name, samples, for_update=True)
+                resource_def, resource_id, metric_name, samples,
+                for_update=True)
             if resource_attributes:
                 self._update_resource(resource_type, resource_id,
                                       resource_attributes)
 
-    def _get_resource_attributes(self, ext, resource_id, metric_name, samples,
-                                 for_update=False):
+    def _get_resource_attributes(self, resource_def, resource_id, metric_name,
+                                 samples, for_update=False):
         # FIXME(sileht): Should I merge attibutes of all samples ?
         # Or keep only the last one is sufficient ?
-        attributes = ext.obj.get_resource_extra_attributes(
-            samples[-1])
+        attributes = resource_def.attributes(samples[-1])
         if not for_update:
             attributes["id"] = resource_id
             attributes["user_id"] = samples[-1]['user_id']
             attributes["project_id"] = samples[-1]['project_id']
-            attributes["metrics"] = dict(
-                (metric_name, self.get_archive_policy(metric_name))
-                for metric_name in ext.obj.get_metrics_names()
-            )
+            attributes["metrics"] = resource_def.metrics()
         return attributes
 
     def _post_measure(self, resource_type, resource_id, metric_name,
@@ -365,8 +469,9 @@ class GnocchiDispatcher(dispatcher.Base):
         else:
             LOG.debug("Resource %s updated", resource_id)
 
-    def _create_metric(self, resource_type, resource_id, metric_name):
-        params = {metric_name: self.get_archive_policy(metric_name)}
+    def _create_metric(self, resource_type, resource_id, metric_name,
+                       archive_policy):
+        params = {metric_name: archive_policy}
         r = self.gnocchi_api.post("%s/v1/resource/%s/%s/metric"
                                   % (self.gnocchi_url, resource_type,
                                      resource_id),

ceilometer/dispatcher/resources/base.py

@@ -1,40 +0,0 @@
-#
-# Copyright 2014 eNovance
-#
-# Authors: Mehdi Abaakouk <mehdi.abaakouk@enovance.com>
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-import abc
-
-import six
-
-
-@six.add_metaclass(abc.ABCMeta)
-class ResourceBase(object):
-    """Base class for resource."""
-
-    @abc.abstractmethod
-    def get_resource_extra_attributes(self, sample):
-        """Extract the metadata from a ceilometer sample.
-
-        :param sample: The ceilometer sample
-        :returns: the resource attributes
-        """
-
-    @abc.abstractmethod
-    def get_metrics_names(self):
-        """Return the metric handled by this resource.
-
-        :returns: list of metric names
-        """

ceilometer/dispatcher/resources/ceph_account.py

@@ -1,31 +0,0 @@
-#
-# Copyright 2015 Mirantis Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-from ceilometer.dispatcher.resources import base
-
-
-class CephAccount(base.ResourceBase):
-    @staticmethod
-    def get_resource_extra_attributes(sample):
-        return {}
-
-    @staticmethod
-    def get_metrics_names():
-        return ['radosgw.api.request',
-                'radosgw.objects.size',
-                'radosgw.objects',
-                'radosgw.objects.containers',
-                'radosgw.containers.objects',
-                'radosgw.containers.objects.size',
-                ]

ceilometer/dispatcher/resources/identity.py

@@ -1,44 +0,0 @@
-#
-# Copyright 2015 Mirantis Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-from ceilometer.dispatcher.resources import base
-
-
-class Identity(base.ResourceBase):
-    @staticmethod
-    def get_resource_extra_attributes(sample):
-        return {}
-
-    @staticmethod
-    def get_metrics_names():
-        return ['identity.authenticate.success',
-                'identity.authenticate.pending',
-                'identity.authenticate.failure',
-                'identity.user.created',
-                'identity.user.deleted',
-                'identity.user.updated',
-                'identity.group.created',
-                'identity.group.deleted',
-                'identity.group.updated',
-                'identity.role.created',
-                'identity.role.deleted',
-                'identity.role.updated',
-                'identity.project.created',
-                'identity.project.deleted',
-                'identity.project.updated',
-                'identity.trust.created',
-                'identity.trust.deleted',
-                'identity.role_assignment.created',
-                'identity.role_assignment.deleted',
-                ]

ceilometer/dispatcher/resources/image.py

@@ -1,30 +0,0 @@
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-from ceilometer.dispatcher.resources import base
-
-
-class Image(base.ResourceBase):
-    @staticmethod
-    def get_resource_extra_attributes(sample):
-        metadata = sample['resource_metadata']
-        params = {
-            "name": metadata['name'],
-            "container_format": metadata["container_format"],
-            "disk_format": metadata["disk_format"]
-        }
-        return params
-
-    @staticmethod
-    def get_metrics_names():
-        return ['image',
-                'image.size']

ceilometer/dispatcher/resources/instance.py

@@ -1,54 +0,0 @@
-#
-# Copyright 2014 eNovance
-#
-# Authors: Julien Danjou <julien@danjou.info>
-#          Mehdi Abaakouk <mehdi.abaakouk@enovance.com>
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-from ceilometer.dispatcher.resources import base
-
-
-class Instance(base.ResourceBase):
-    @staticmethod
-    def get_resource_extra_attributes(sample):
-        metadata = sample['resource_metadata']
-        params = {
-            "host": metadata['host'],
-            "image_ref": metadata['image_ref_url'],
-            "display_name": metadata['display_name'],
-        }
-        if "instance_flavor_id" in metadata:
-            params["flavor_id"] = metadata['instance_flavor_id']
-        else:
-            # NOTE(sileht): instance.exists have the flavor here
-            params["flavor_id"] = metadata["flavor"]["id"]
-
-        server_group = metadata.get('user_metadata', {}).get('server_group')
-        if server_group:
-            params["server_group"] = server_group
-
-        return params
-
-    @staticmethod
-    def get_metrics_names():
-        # NOTE(sileht): Can we generate the list by loading ceilometer
-        # plugin ?
-        return ['instance',
-                'disk.root.size',
-                'disk.ephemeral.size',
-                'memory',
-                'memory.usage',
-                'vcpus',
-                'cpu',
-                'cpu_util']

ceilometer/dispatcher/resources/ipmi.py

@@ -1,30 +0,0 @@
-#
-# Copyright 2015 Mirantis Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-from ceilometer.dispatcher.resources import base
-
-
-class IPMI(base.ResourceBase):
-    @staticmethod
-    def get_resource_extra_attributes(sample):
-        return {}
-
-    @staticmethod
-    def get_metrics_names():
-        return ['hardware.ipmi.node.power',
-                'hardware.ipmi.node.temperature',
-                'hardware.ipmi.node.fan',
-                'hardware.ipmi.node.current',
-                'hardware.ipmi.node.voltage',
-                ]

ceilometer/dispatcher/resources/network.py

@@ -1,41 +0,0 @@
-#
-# Copyright 2015 Mirantis Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-from ceilometer.dispatcher.resources import base
-
-
-class Network(base.ResourceBase):
-    @staticmethod
-    def get_resource_extra_attributes(sample):
-        return {}
-
-    @staticmethod
-    def get_metrics_names():
-        return ['bandwidth',
-                'network',
-                'network.create',
-                'network.update',
-                'subnet',
-                'subnet.create',
-                'subnet.update',
-                'port',
-                'port.create',
-                'port.update',
-                'router',
-                'router.create',
-                'router.update',
-                'ip.floating',
-                'ip.floating.create',
-                'ip.floating.update',
-                ]

ceilometer/dispatcher/resources/orchestration.py

@@ -1,30 +0,0 @@
-#
-# Copyright 2015 Mirantis Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-from ceilometer.dispatcher.resources import base
-
-
-class Stack(base.ResourceBase):
-    @staticmethod
-    def get_resource_extra_attributes(sample):
-        return {}
-
-    @staticmethod
-    def get_metrics_names():
-        return ['stack.create',
-                'stack.update',
-                'stack.delete',
-                'stack.resume',
-                'stack.suspend',
-                ]

ceilometer/dispatcher/resources/swift_account.py

@@ -1,33 +0,0 @@
-#
-# Copyright 2014 eNovance
-#
-# Authors: Julien Danjou <julien@danjou.info>
-#          Mehdi Abaakouk <mehdi.abaakouk@enovance.com>
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-from ceilometer.dispatcher.resources import base
-
-
-class SwiftAccount(base.ResourceBase):
-    @staticmethod
-    def get_resource_extra_attributes(sample):
-        return {}
-
-    @staticmethod
-    def get_metrics_names():
-        return ['storage.objects.incoming.bytes',
-                'storage.objects.outgoing.bytes',
-                'storage.api.request',
-                'storage.objects.size',
-                'storage.objects',
-                'storage.objects.containers']

ceilometer/dispatcher/resources/volume.py

@@ -1,38 +0,0 @@
-#
-# Copyright 2015 Mirantis Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-from ceilometer.dispatcher.resources import base
-
-
-class Volume(base.ResourceBase):
-    @staticmethod
-    def get_resource_extra_attributes(sample):
-        metadata = sample['resource_metadata']
-        params = {
-            "display_name": metadata['display_name'],
-        }
-        return params
-
-    @staticmethod
-    def get_metrics_names():
-        return ['volume',
-                'volume.size',
-                'volume.create',
-                'volume.delete',
-                'volume.update',
-                'volume.resize',
-                'volume.attach',
-                'volume.detach',
-                ]

ceilometer/tests/dispatcher/test_gnocchi.py

@@ -16,13 +16,15 @@
 # under the License.
 
 import json
+import os
 import uuid
 
 import mock
 from oslo_config import fixture as config_fixture
-from oslotest import base
+from oslo_utils import fileutils
 from oslotest import mockpatch
 import requests
+import six
 import six.moves.urllib.parse as urlparse
 import tempfile
 import testscenarios
@@ -30,6 +32,7 @@ import yaml
 
 from ceilometer.dispatcher import gnocchi
 from ceilometer import service as ceilometer_service
+from ceilometer.tests import base
 
 load_tests = testscenarios.load_tests_apply_scenarios
 
@@ -51,6 +54,11 @@ class DispatcherTest(base.BaseTestCase):
         super(DispatcherTest, self).setUp()
         self.conf = self.useFixture(config_fixture.Config())
         ceilometer_service.prepare_service([])
+        self.conf.config(
+            resources_definition_file=self.path_get(
+                'etc/ceilometer/gnocchi_resources.yaml'),
+            group="dispatcher_gnocchi"
+        )
         self.resource_id = str(uuid.uuid4())
         self.samples = [{
             'counter_name': 'disk.root.size',
@@ -92,12 +100,43 @@ class DispatcherTest(base.BaseTestCase):
             return_value=ks_client))
         self.conf.conf.dispatcher_gnocchi.filter_service_activity = True
 
-    def test_extensions_load(self):
+    def test_config_load(self):
         self.conf.config(filter_service_activity=False,
                          group='dispatcher_gnocchi')
         d = gnocchi.GnocchiDispatcher(self.conf.conf)
-        self.assertIn('instance', d.mgmr.names())
-        self.assertIn('volume', d.mgmr.names())
+        names = [rd.cfg['resource_type'] for rd in d.resources_definition]
+        self.assertIn('instance', names)
+        self.assertIn('volume', names)
+
+    def test_broken_config_load(self):
+        contents = [("---\n"
+                     "resources:\n"
+                     "  - resource_type: foobar\n"),
+                    ("---\n"
+                     "resources:\n"
+                     "  - resource_type: 0\n"),
+                    ("---\n"
+                     "resources:\n"
+                     "  - sample_types: ['foo', 'bar']\n"),
+                    ("---\n"
+                     "resources:\n"
+                     "  - sample_types: foobar\n"
+                     "  - resource_type: foobar\n"),
+                    ]
+
+        for content in contents:
+            if six.PY3:
+                content = content.encode('utf-8')
+
+            temp = fileutils.write_to_tempfile(content=content,
+                                               prefix='gnocchi_resources',
+                                               suffix='.yaml')
+            self.addCleanup(os.remove, temp)
+            self.conf.config(filter_service_activity=False,
+                             resources_definition_file=temp,
+                             group='dispatcher_gnocchi')
+            self.assertRaises(gnocchi.ResourcesDefinitionException,
+                              gnocchi.GnocchiDispatcher, self.conf.conf)
 
     @mock.patch('ceilometer.dispatcher.gnocchi.GnocchiDispatcher'
                 '._process_samples')
@@ -110,10 +149,6 @@ class DispatcherTest(base.BaseTestCase):
             expected_samples, True,
         )
 
-    def test_archive_policy_default(self):
-        d = gnocchi.GnocchiDispatcher(self.conf.conf)
-        self.assertEqual(d.gnocchi_archive_policy_default, "low")
-
     def test_archive_policy_map_config(self):
         archive_policy_map = yaml.dump({
             'foo.*': 'low'
@@ -125,9 +160,8 @@ class DispatcherTest(base.BaseTestCase):
         self.conf.conf.dispatcher_gnocchi.archive_policy_file = (
             archive_policy_cfg_file.name)
         d = gnocchi.GnocchiDispatcher(self.conf.conf)
-        self.assertEqual(
-            d.get_archive_policy(
-                'foo.disk.rate')['archive_policy_name'], "low")
+        legacy = d._load_archive_policy(self.conf.conf)
+        self.assertEqual(legacy.get('foo.disk.rate'), "low")
         archive_policy_cfg_file.close()
 
     def test_activity_filter_match_project_id(self):
@@ -265,6 +299,12 @@ class DispatcherWorkflowTest(base.BaseTestCase,
             return_value=ks_client))
 
         ceilometer_service.prepare_service([])
+        self.conf.config(
+            resources_definition_file=self.path_get(
+                'etc/ceilometer/gnocchi_resources.yaml'),
+            group="dispatcher_gnocchi"
+        )
+
         self.dispatcher = gnocchi.GnocchiDispatcher(self.conf.conf)
         self.sample['resource_id'] = str(uuid.uuid4())
 

devstack/plugin.sh

@@ -255,6 +255,8 @@ function configure_ceilometer {
     cp $CEILOMETER_DIR/etc/ceilometer/event_pipeline.yaml $CEILOMETER_CONF_DIR
     cp $CEILOMETER_DIR/etc/ceilometer/api_paste.ini $CEILOMETER_CONF_DIR
     cp $CEILOMETER_DIR/etc/ceilometer/event_definitions.yaml $CEILOMETER_CONF_DIR
+    cp $CEILOMETER_DIR/etc/ceilometer/gnocchi_archive_policy_map.yaml $CEILOMETER_CONF_DIR
+    cp $CEILOMETER_DIR/etc/ceilometer/gnocchi_resources.yaml $CEILOMETER_CONF_DIR
 
     if [ "$CEILOMETER_PIPELINE_INTERVAL" ]; then
         sed -i "s/interval:.*/interval: ${CEILOMETER_PIPELINE_INTERVAL}/" $CEILOMETER_CONF_DIR/pipeline.yaml

etc/ceilometer/gnocchi_resources.yaml

@@ -0,0 +1,121 @@
+---
+
+resources:
+  - resource_type: identity
+    archive_policy: low
+    metrics:
+      - 'identity.authenticate.success'
+      - 'identity.authenticate.pending'
+      - 'identity.authenticate.failure'
+      - 'identity.user.created'
+      - 'identity.user.deleted'
+      - 'identity.user.updated'
+      - 'identity.group.created'
+      - 'identity.group.deleted'
+      - 'identity.group.updated'
+
+  - resource_type: identity
+    archive_policy: low
+    metrics:
+      - 'identity.role.created'
+      - 'identity.role.deleted'
+      - 'identity.role.updated'
+      - 'identity.project.created'
+      - 'identity.project.deleted'
+      - 'identity.project.updated'
+      - 'identity.trust.created'
+      - 'identity.trust.deleted'
+      - 'identity.role_assignment.created'
+      - 'identity.role_assignment.deleted'
+
+  - resource_type: ceph_account
+    metrics:
+      - 'stack.create'
+      - 'stack.update'
+      - 'stack.delete'
+      - 'stack.resume'
+      - 'stack.suspend'
+
+  - resource_type: instance
+    metrics:
+      - 'instance'
+      - 'disk.root.size'
+      - 'disk.ephemeral.size'
+      - 'memory'
+      - 'memory.usage'
+      - 'vcpus'
+      - 'cpu'
+      - 'cpu_util'
+    attributes:
+      - host: resource_metadata.host
+      - image_ref: resource_metadata.image_ref_url
+      - display_name: resource_metadata.display_name
+      - flavor_id: resource_metadata.instance_flavor_id
+      - flavor_id: resource_metadata.flavor.id
+      - server_group: resource_metadata.user_metadata.server_group
+
+  - resource_type: image
+    metrics:
+      - 'image'
+      - 'image.size'
+    attributes:
+      - name: resource_metadata.name
+      - container_format: resource_metadata.container_format
+      - disk_format: resource_metadata.disk_format
+
+  - resource_type: ipmi
+    metrics:
+      - 'hardware.ipmi.node.power'
+      - 'hardware.ipmi.node.temperature'
+      - 'hardware.ipmi.node.fan'
+      - 'hardware.ipmi.node.current'
+      - 'hardware.ipmi.node.voltage'
+
+  - resource_type: network
+    metrics:
+      - 'bandwidth'
+      - 'network'
+      - 'network.create'
+      - 'network.update'
+      - 'subnet'
+      - 'subnet.create'
+      - 'subnet.update'
+      - 'port'
+      - 'port.create'
+      - 'port.update'
+      - 'router'
+      - 'router.create'
+      - 'router.update'
+      - 'ip.floating'
+      - 'ip.floating.create'
+      - 'ip.floating.update'
+
+  - resource_type: orchestration
+    metrics:
+      - 'stack.create'
+      - 'stack.update'
+      - 'stack.delete'
+      - 'stack.resume'
+      - 'stack.suspend'
+
+  - resource_type: swift_account
+    metrics:
+      - 'storage.objects.incoming.bytes'
+      - 'storage.objects.outgoing.bytes'
+      - 'storage.api.request'
+      - 'storage.objects.size'
+      - 'storage.objects'
+      - 'storage.objects.containers'
+
+  - resource_type: volume
+    metrics:
+      - 'volume'
+      - 'volume.size'
+      - 'volume.create'
+      - 'volume.delete'
+      - 'volume.update'
+      - 'volume.resize'
+      - 'volume.attach'
+      - 'volume.detach'
+    attributes:
+      - display_name: resource_metadata.display_name

setup.cfg

@@ -346,17 +346,6 @@ ceilometer.dispatcher =
     http = ceilometer.dispatcher.http:HttpDispatcher
     gnocchi = ceilometer.dispatcher.gnocchi:GnocchiDispatcher
 
-ceilometer.dispatcher.resource =
-    instance = ceilometer.dispatcher.resources.instance:Instance
-    swift_account = ceilometer.dispatcher.resources.swift_account:SwiftAccount
-    volume = ceilometer.dispatcher.resources.volume:Volume
-    ceph_account = ceilometer.dispatcher.resources.ceph_account:CephAccount
-    network = ceilometer.dispatcher.resources.network:Network
-    identity = ceilometer.dispatcher.resources.identity:Identity
-    ipmi = ceilometer.dispatcher.resources.ipmi:IPMI
-    stack = ceilometer.dispatcher.resources.orchestration:Stack
-    image = ceilometer.dispatcher.resources.image:Image
-
 network.statistics.drivers =
     opendaylight = ceilometer.network.statistics.opendaylight.driver:OpenDayLightDriver
     opencontrail = ceilometer.network.statistics.opencontrail.driver:OpencontrailDriver