Merge "Use resource discovery for Network Services"

ceilometer/network/services/base.py

@@ -0,0 +1,51 @@
+#
+# Copyright 2014 Cisco Systems,Inc.
+#
+# Author: Pradeep Kilambi <pkilambi@cisco.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from ceilometer import neutron_client
+from ceilometer.openstack.common import log
+from ceilometer import plugin
+
+LOG = log.getLogger(__name__)
+
+
+# status map for converting metric status to volume int
+STATUS = {
+    'inactive': 0,
+    'active': 1,
+    'pending_create': 2,
+}
+
+
+class BaseServicesPollster(plugin.PollsterBase):
+
+    FIELDS = []
+    nc = neutron_client.Client()
+
+    def _iter_cache(self, cache, meter_name, method):
+        if meter_name not in cache:
+            cache[meter_name] = list(method())
+        return iter(cache[meter_name])
+
+    def extract_metadata(self, metric):
+        return dict((k, metric[k]) for k in self.FIELDS)
+
+    @staticmethod
+    def get_status_id(value):
+        status = value.lower()
+        if status not in STATUS:
+            return -1
+        return STATUS[status]

ceilometer/network/services/discovery.py

@@ -0,0 +1,78 @@
+# -*- encoding: utf-8 -*-
+#
+# Copyright © 2014 Cisco Systems, Inc
+#
+# Author:Pradeep Kilambi <pkilambi@cisco.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from ceilometer import neutron_client
+from ceilometer import plugin
+
+
+class _BaseServicesDiscovery(plugin.DiscoveryBase):
+
+    def __init__(self):
+        super(_BaseServicesDiscovery, self).__init__()
+        self.neutron_cli = neutron_client.Client()
+
+
+class LBPoolsDiscovery(_BaseServicesDiscovery):
+
+    def __init__(self):
+        super(LBPoolsDiscovery, self).__init__()
+
+    def discover(self, param=None):
+        """Discover resources to monitor."""
+
+        pools = self.neutron_cli.pool_get_all()
+        return [i for i in pools
+                if i.get('status') != 'error']
+
+
+class LBVipsDiscovery(_BaseServicesDiscovery):
+
+    def __init__(self):
+        super(LBVipsDiscovery, self).__init__()
+
+    def discover(self, param=None):
+        """Discover resources to monitor."""
+
+        vips = self.neutron_cli.vip_get_all()
+        return [i for i in vips
+                if i.get('status', None) != 'error']
+
+
+class LBMembersDiscovery(_BaseServicesDiscovery):
+
+    def __init__(self):
+        super(LBMembersDiscovery, self).__init__()
+
+    def discover(self, param=None):
+        """Discover resources to monitor."""
+
+        members = self.neutron_cli.member_get_all()
+        return [i for i in members
+                if i.get('status', None) != 'error']
+
+
+class LBHealthMonitorsDiscovery(_BaseServicesDiscovery):
+
+    def __init__(self):
+        super(LBHealthMonitorsDiscovery, self).__init__()
+
+    def discover(self, param=None):
+        """Discover resources to monitor."""
+
+        probes = self.neutron_cli.health_monitor_get_all()
+        return probes

ceilometer/network/services/lbaas.py

@@ -19,11 +19,10 @@ import abc
 import collections
 import six
 
-from ceilometer import neutron_client
+from ceilometer.network.services import base
 from ceilometer.openstack.common.gettextutils import _
 from ceilometer.openstack.common import log
 from ceilometer.openstack.common import timeutils
-from ceilometer import plugin
 from ceilometer import sample
 
 LOG = log.getLogger(__name__)
@@ -33,37 +32,10 @@ LBStatsData = collections.namedtuple(
     ['active_connections', 'total_connections', 'bytes_in', 'bytes_out']
 )
 
-# status map for converting metric status to volume int
-STATUS = {
-    'inactive': 0,
-    'active': 1,
-    'pending_create': 2,
-}
 
-
+class LBPoolPollster(base.BaseServicesPollster):
-class _BasePollster(plugin.PollsterBase):
-
-    FIELDS = []
-    nc = neutron_client.Client()
-
-    def _iter_cache(self, cache, meter_name, method):
-        if meter_name not in cache:
-            cache[meter_name] = list(method())
-        return iter(cache[meter_name])
-
-    def extract_metadata(self, metric):
-        return dict((k, metric[k]) for k in self.FIELDS)
-
-    @staticmethod
-    def get_status_id(value):
-        status = value.lower()
-        if status not in STATUS:
-            return -1
-        return STATUS[status]
-
-
-class LBPoolPollster(_BasePollster):
     """Pollster to capture Load Balancer pool status samples."""
+
     FIELDS = ['admin_state_up',
               'description',
               'lb_method',
@@ -76,11 +48,8 @@ class LBPoolPollster(_BasePollster):
               'vip_id'
               ]
 
-    def _get_lb_pools(self):
-        return self.nc.pool_get_all()
-
     def get_samples(self, manager, cache, resources=None):
-        for pool in self._iter_cache(cache, 'pool', self._get_lb_pools):
+        for pool in resources:
             LOG.debug("Load Balancer Pool : %s" % pool)
             status = self.get_status_id(pool['status'])
             if status == -1:
@@ -102,8 +71,9 @@ class LBPoolPollster(_BasePollster):
             )
 
 
-class LBVipPollster(_BasePollster):
+class LBVipPollster(base.BaseServicesPollster):
     """Pollster to capture Load Balancer Vip status samples."""
+
     FIELDS = ['admin_state_up',
               'address',
               'connection_limit',
@@ -119,11 +89,8 @@ class LBVipPollster(_BasePollster):
               'session_persistence',
               ]
 
-    def _get_lb_vips(self):
-        return self.nc.vip_get_all()
-
     def get_samples(self, manager, cache, resources=None):
-        for vip in self._iter_cache(cache, 'vip', self._get_lb_vips):
+        for vip in resources:
             LOG.debug("Load Balancer Vip : %s" % vip)
             status = self.get_status_id(vip['status'])
             if status == -1:
@@ -145,8 +112,9 @@ class LBVipPollster(_BasePollster):
             )
 
 
-class LBMemberPollster(_BasePollster):
+class LBMemberPollster(base.BaseServicesPollster):
     """Pollster to capture Load Balancer Member status samples."""
+
     FIELDS = ['admin_state_up',
               'address',
               'pool_id',
@@ -156,11 +124,8 @@ class LBMemberPollster(_BasePollster):
               'weight',
               ]
 
-    def _get_lb_members(self):
-        return self.nc.member_get_all()
-
     def get_samples(self, manager, cache, resources=None):
-        for member in self._iter_cache(cache, 'member', self._get_lb_members):
+        for member in resources:
             LOG.debug("Load Balancer Member : %s" % member)
             status = self.get_status_id(member['status'])
             if status == -1:
@@ -180,8 +145,9 @@ class LBMemberPollster(_BasePollster):
             )
 
 
-class LBHealthMonitorPollster(_BasePollster):
+class LBHealthMonitorPollster(base.BaseServicesPollster):
     """Pollster to capture Load Balancer Health probes status samples."""
+
     FIELDS = ['admin_state_up',
               'delay',
               'max_retries',
@@ -190,12 +156,8 @@ class LBHealthMonitorPollster(_BasePollster):
               'type'
               ]
 
-    def _get_lb_health_probes(self):
-        return self.nc.health_monitor_get_all()
-
     def get_samples(self, manager, cache, resources=None):
-        for probe in self._iter_cache(cache, 'monitor',
+        for probe in resources:
-                                      self._get_lb_health_probes):
             LOG.debug("Load Balancer Health probe : %s" % probe)
             yield sample.Sample(
                 name='network.services.lb.health_monitor',
@@ -211,7 +173,7 @@ class LBHealthMonitorPollster(_BasePollster):
 
 
 @six.add_metaclass(abc.ABCMeta)
-class _LBStatsPollster(_BasePollster):
+class _LBStatsPollster(base.BaseServicesPollster):
     """Base Statistics pollster.
 
      It is capturing the statistics info and yielding samples for connections

ceilometer/neutron_client.py

@@ -79,6 +79,7 @@ class Client(object):
 
     @logged
     def pool_get_all(self):
+        LOG.debug("NEUTRON POOL GET")
         resp = self.client.list_pools()
         return resp.get('pools')
 

ceilometer/pipeline.py

@@ -98,7 +98,6 @@ class Source(object):
         except KeyError as err:
             raise PipelineException(
                 "Required field %s not specified" % err.args[0], cfg)
-
         if self.interval <= 0:
             raise PipelineException("Interval value should > 0", cfg)
 
@@ -109,7 +108,6 @@ class Source(object):
         self.discovery = cfg.get('discovery') or []
         if not isinstance(self.discovery, list):
             raise PipelineException("Discovery should be a list", cfg)
-
         self._check_meters()
 
     def __str__(self):

ceilometer/tests/network/services/test_lbaas.py

@@ -18,6 +18,7 @@
 import mock
 
 from ceilometer.central import manager
+from ceilometer.network.services import discovery
 from ceilometer.network.services import lbaas
 from ceilometer.openstack.common import context
 from ceilometer.openstack.common.fixture import mockpatch
@@ -106,26 +107,56 @@ class TestLBPoolPollster(_BaseTestLBPollster):
                  'subnet_id': 'bbe3d818-bdcb-4e4b-b47f-5650dc8a9d7a',
                  'tenant_id': 'a4eb9f4938bb418bbc4f8eb31802fefa',
                  'health_monitors_status': []},
+                {'status': 'error',
+                 'lb_method': 'ROUND_ROBIN',
+                 'protocol': 'HTTP',
+                 'description': '',
+                 'health_monitors': [],
+                 'members': [],
+                 'provider': 'haproxy',
+                 'status_description': None,
+                 'id': 'fe7rad36-437d-4c84-aee1-186027d3bdcd',
+                 'vip_id': 'cd6a6fee-e2fa-4e6c-b3c2-bfbe395752c1',
+                 'name': 'mylb_error',
+                 'admin_state_up': True,
+                 'subnet_id': 'bbe3d818-bdcb-4e4b-b47f-5650dc8a9d7a',
+                 'tenant_id': 'a4eb9f4938bb418bbc4f8eb31802fefa',
+                 'health_monitors_status': []},
                 ]
 
     def test_pool_get_samples(self):
-        samples = list(self.pollster.get_samples(self.manager, {}))
+        samples = list(self.pollster.get_samples(
+            self.manager, {},
+            resources=self.fake_get_pools()))
         self.assertEqual(3, len(samples))
         for field in self.pollster.FIELDS:
             self.assertEqual(self.fake_get_pools()[0][field],
                              samples[0].resource_metadata[field])
 
     def test_pool_volume(self):
-        samples = list(self.pollster.get_samples(self.manager, {}))
+        samples = list(self.pollster.get_samples(
+            self.manager, {},
+            resources=self.fake_get_pools()))
         self.assertEqual(1, samples[0].volume)
         self.assertEqual(0, samples[1].volume)
         self.assertEqual(2, samples[2].volume)
 
     def test_get_pool_meter_names(self):
-        samples = list(self.pollster.get_samples(self.manager, {}))
+        samples = list(self.pollster.get_samples(
+            self.manager, {},
+            resources=self.fake_get_pools()))
         self.assertEqual(set(['network.services.lb.pool']),
                          set([s.name for s in samples]))
 
+    def test_pool_discovery(self):
+        discovered_pools = discovery.LBPoolsDiscovery().discover()
+        self.assertEqual(4, len(discovered_pools))
+        for pool in self.fake_get_pools():
+            if pool['status'] == 'error':
+                self.assertTrue(pool not in discovered_pools)
+            else:
+                self.assertTrue(pool in discovered_pools)
+
 
 class TestLBVipPollster(_BaseTestLBPollster):
 
@@ -199,26 +230,56 @@ class TestLBVipPollster(_BaseTestLBPollster):
                  'port_id': '3df3c4de-b32e-4ca1-a7f4-84323ba5f291',
                  'id': 'fg6a6fee-e2fa-4e6c-b3c2-bfbe395752c1',
                  'name': 'myvip03'},
+                {'status': 'error',
+                 'status_description': None,
+                 'protocol': 'HTTP',
+                 'description': '',
+                 'admin_state_up': True,
+                 'subnet_id': 'bbe3d818-bdcb-4e4b-b47f-5650dc8a9d7a',
+                 'tenant_id': 'a4eb9f4938bb418bbc4f8eb31802fefa',
+                 'connection_limit': -1,
+                 'pool_id': 'ce73ad36-437d-4c84-aee1-186027d3da9a',
+                 'session_persistence': None,
+                 'address': '10.0.0.8',
+                 'protocol_port': 80,
+                 'port_id': '3df3c4de-b32e-4ca1-a7f4-84323ba5f291',
+                 'id': 'fg6a6fee-e2fa-4e6c-b3c2-bfbe395752c1',
+                 'name': 'myvip_error'},
                 ]
 
     def test_vip_get_samples(self):
-        samples = list(self.pollster.get_samples(self.manager, {}))
+        samples = list(self.pollster.get_samples(
+            self.manager, {},
+            resources=self.fake_get_vips()))
         self.assertEqual(3, len(samples))
         for field in self.pollster.FIELDS:
             self.assertEqual(self.fake_get_vips()[0][field],
                              samples[0].resource_metadata[field])
 
     def test_pool_volume(self):
-        samples = list(self.pollster.get_samples(self.manager, {}))
+        samples = list(self.pollster.get_samples(
+            self.manager, {},
+            resources=self.fake_get_vips()))
         self.assertEqual(1, samples[0].volume)
         self.assertEqual(0, samples[1].volume)
         self.assertEqual(2, samples[2].volume)
 
     def test_get_vip_meter_names(self):
-        samples = list(self.pollster.get_samples(self.manager, {}))
+        samples = list(self.pollster.get_samples(
+            self.manager, {},
+            resources=self.fake_get_vips()))
         self.assertEqual(set(['network.services.lb.vip']),
                          set([s.name for s in samples]))
 
+    def test_vip_discovery(self):
+        discovered_vips = discovery.LBVipsDiscovery().discover()
+        self.assertEqual(4, len(discovered_vips))
+        for pool in self.fake_get_vips():
+            if pool['status'] == 'error':
+                self.assertTrue(pool not in discovered_vips)
+            else:
+                self.assertTrue(pool in discovered_vips)
+
 
 class TestLBMemberPollster(_BaseTestLBPollster):
 
@@ -268,26 +329,50 @@ class TestLBMemberPollster(_BaseTestLBPollster):
                  'address': '10.0.0.6',
                  'status_description': None,
                  'id': '45630b61eb-07bc-4372-9fbf-36459dd0f96b'},
+                {'status': 'error',
+                 'protocol_port': 80,
+                 'weight': 1,
+                 'admin_state_up': True,
+                 'tenant_id': 'a4eb9f4938bb418bbc4f8eb31802fefa',
+                 'pool_id': 'ce73ad36-437d-4c84-aee1-186027d3da9a',
+                 'address': '10.0.0.6',
+                 'status_description': None,
+                 'id': '45630b61eb-07bc-4372-9fbf-36459dd0f96b'},
                 ]
 
     def test_get_samples_not_empty(self):
-        samples = list(self.pollster.get_samples(self.manager, {}))
+        samples = list(self.pollster.get_samples(
+            self.manager, {},
+            self.fake_get_members()))
         self.assertEqual(3, len(samples))
         for field in self.pollster.FIELDS:
             self.assertEqual(self.fake_get_members()[0][field],
                              samples[0].resource_metadata[field])
 
     def test_pool_volume(self):
-        samples = list(self.pollster.get_samples(self.manager, {}))
+        samples = list(self.pollster.get_samples(
+            self.manager, {},
+            self.fake_get_members()))
         self.assertEqual(1, samples[0].volume)
         self.assertEqual(0, samples[1].volume)
         self.assertEqual(2, samples[2].volume)
 
     def test_get_meter_names(self):
-        samples = list(self.pollster.get_samples(self.manager, {}))
+        samples = list(self.pollster.get_samples(
+            self.manager, {},
+            self.fake_get_members()))
         self.assertEqual(set(['network.services.lb.member']),
                          set([s.name for s in samples]))
 
+    def test_members_discovery(self):
+        discovered_members = discovery.LBMembersDiscovery().discover()
+        self.assertEqual(4, len(discovered_members))
+        for pool in self.fake_get_members():
+            if pool['status'] == 'error':
+                self.assertTrue(pool not in discovered_members)
+            else:
+                self.assertTrue(pool in discovered_members)
+
 
 class TestLBHealthProbePollster(_BaseTestLBPollster):
 
@@ -312,17 +397,25 @@ class TestLBHealthProbePollster(_BaseTestLBPollster):
                  }]
 
     def test_get_samples_not_empty(self):
-        samples = list(self.pollster.get_samples(self.manager, {}))
+        samples = list(self.pollster.get_samples(
+            self.manager, {},
+            self.fake_get_health_monitor()))
         self.assertEqual(1, len(samples))
         for field in self.pollster.FIELDS:
             self.assertEqual(self.fake_get_health_monitor()[0][field],
                              samples[0].resource_metadata[field])
 
     def test_get_meter_names(self):
-        samples = list(self.pollster.get_samples(self.manager, {}))
+        samples = list(self.pollster.get_samples(
+            self.manager, {},
+            self.fake_get_health_monitor()))
         self.assertEqual(set(['network.services.lb.health_monitor']),
                          set([s.name for s in samples]))
 
+    def test_probes_discovery(self):
+        discovered_probes = discovery.LBHealthMonitorsDiscovery().discover()
+        self.assertEqual(discovered_probes, self.fake_get_health_monitor())
+
 
 class TestLBStatsPollster(_BaseTestLBPollster):
 

etc/ceilometer/pipeline.yaml

@@ -30,6 +30,38 @@ sources:
           - "network.outgoing.packets"
       sinks:
           - network_sink
+    - name: lb_pool_source
+      interval: 600
+      meters:
+          - "network.services.lb.pool"
+      discovery:
+          - "lb_pools"
+      sinks:
+          - meter_sink
+    - name: lb_health_monitor_source
+      interval: 600
+      meters:
+         - "network.services.lb.health_monitor"
+      discovery:
+         - "lb_health_probes"
+      sinks:
+          - meter_sink
+    - name: lb_vip_source
+      interval: 600
+      meters:
+          - "network.services.lb.vip"
+      discovery:
+          - "lb_vips"
+      sinks:
+          - meter_sink
+    - name: lb_member_source
+      interval: 600
+      meters:
+          - "network.services.lb.member"
+      discovery:
+          - "lb_members"
+      sinks:
+          - meter_sink
 sinks:
     - name: meter_sink
       transformers:

setup.cfg

@@ -69,6 +69,11 @@ ceilometer.notification =
 
 ceilometer.discover =
     local_instances = ceilometer.compute.discovery:InstanceDiscovery
+    lb_pools = ceilometer.network.services.discovery:LBPoolsDiscovery
+    lb_vips = ceilometer.network.services.discovery:LBVipsDiscovery
+    lb_members = ceilometer.network.services.discovery:LBMembersDiscovery
+    lb_health_probes = ceilometer.network.services.discovery:LBHealthMonitorsDiscovery
+
 
 ceilometer.poll.compute =
     disk.read.requests = ceilometer.compute.pollsters.disk:ReadRequestsPollster