openstack/ceilometer
Code Issues Proposed changes

Merge "Drop timestamping in pollsters"

Browse Source
This commit is contained in:
Jenkins 2016-06-06 15:24:22 +00:00 committed by Gerrit Code Review
commit 0569b75b10
20 changed files with 46 additions and 121 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
ceilometer/compute/pollsters/util.py
View File

@ -13,7 +13,6 @@
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from oslo_utils import timeutils
from ceilometer.compute import util as compute_util
from ceilometer import sample
@ -88,7 +87,6 @@ def make_sample_from_instance(instance, name, type, unit, volume,
user_id=instance.user_id,
project_id=instance.tenant_id,
resource_id=resource_id or instance.id,
timestamp=timeutils.utcnow().isoformat(),
resource_metadata=resource_metadata,
)

6
ceilometer/energy/kwapi.py
View File

@ -12,8 +12,6 @@
# License for the specific language governing permissions and limitations
# under the License.
import datetime
from keystoneauth1 import exceptions
from oslo_config import cfg
from oslo_log import log
@ -104,8 +102,6 @@ class EnergyPollster(_Base):
user_id=None,
project_id=None,
resource_id=probe['id'],
timestamp=datetime.datetime.fromtimestamp(
probe['timestamp']).isoformat(),
resource_metadata={}
)
@ -124,7 +120,5 @@ class PowerPollster(_Base):
user_id=None,
project_id=None,
resource_id=probe['id'],
timestamp=datetime.datetime.fromtimestamp(
probe['timestamp']).isoformat(),
resource_metadata={}
)

2
ceilometer/hardware/pollsters/util.py
View File

@ -20,7 +20,6 @@
import copy
from oslo_utils import timeutils
from six.moves.urllib import parse as urlparse
from ceilometer import sample
@ -59,7 +58,6 @@ def make_sample_from_host(host_url, name, sample_type, unit, volume,
user_id=user_id or extra.get('user_id'),
project_id=project_id or extra.get('project_id'),
resource_id=res_id,
timestamp=timeutils.utcnow().isoformat(),
resource_metadata=resource_metadata,
source='hardware',
)

3
ceilometer/image/glance.py
View File

@ -19,7 +19,6 @@ from __future__ import absolute_import
import glanceclient
from oslo_config import cfg
from oslo_utils import timeutils
from ceilometer.agent import plugin_base
from ceilometer import keystone_client
@ -110,7 +109,6 @@ class ImagePollster(_Base):
user_id=None,
project_id=image.owner,
resource_id=image.id,
timestamp=timeutils.utcnow().isoformat(),
resource_metadata=self.extract_image_metadata(image),
)
@ -127,6 +125,5 @@ class ImageSizePollster(_Base):
user_id=None,
project_id=image.owner,
resource_id=image.id,
timestamp=timeutils.utcnow().isoformat(),
resource_metadata=self.extract_image_metadata(image),
)

2
ceilometer/ipmi/pollsters/node.py
View File

@ -16,7 +16,6 @@ import abc
from oslo_config import cfg
from oslo_log import log
from oslo_utils import timeutils
import six
from ceilometer.agent import plugin_base
@ -89,7 +88,6 @@ class _Base(plugin_base.PollsterBase):
user_id=None,
project_id=None,
resource_id=CONF.host,
timestamp=timeutils.utcnow().isoformat(),
resource_metadata=metadata)

2
ceilometer/ipmi/pollsters/sensor.py
View File

@ -14,7 +14,6 @@
from oslo_config import cfg
from oslo_log import log
from oslo_utils import timeutils
from ceilometer.agent import plugin_base
from ceilometer.i18n import _
@ -112,7 +111,6 @@ class SensorPollster(plugin_base.PollsterBase):
user_id=None,
project_id=None,
resource_id=resource_id,
timestamp=timeutils.utcnow().isoformat(),
resource_metadata=metadata)

2
ceilometer/network/floatingip.py
View File

@ -18,7 +18,6 @@
from oslo_config import cfg
from oslo_log import log
from oslo_utils import timeutils
from ceilometer.i18n import _LW
from ceilometer.network.services import base
@ -59,6 +58,5 @@ class FloatingIPPollster(base.BaseServicesPollster):
user_id=fip.get('user_id'),
project_id=fip['tenant_id'],
resource_id=fip['id'],
timestamp=timeutils.utcnow().isoformat(),
resource_metadata=self.extract_metadata(fip)
)

3
ceilometer/network/services/fwaas.py
View File

@ -14,7 +14,6 @@
# under the License.
from oslo_log import log
from oslo_utils import timeutils
from ceilometer.i18n import _
from ceilometer.network.services import base
@ -58,7 +57,6 @@ class FirewallPollster(base.BaseServicesPollster):
user_id=None,
project_id=fw['tenant_id'],
resource_id=fw['id'],
timestamp=timeutils.utcnow().isoformat(),
resource_metadata=self.extract_metadata(fw)
)
@ -92,6 +90,5 @@ class FirewallPolicyPollster(base.BaseServicesPollster):
user_id=None,
project_id=fw['tenant_id'],
resource_id=fw['id'],
timestamp=timeutils.utcnow().isoformat(),
resource_metadata=self.extract_metadata(fw)
)

9
ceilometer/network/services/lbaas.py
View File

@ -18,7 +18,6 @@ import collections
from oslo_config import cfg
from oslo_log import log
from oslo_utils import timeutils
import six
from ceilometer.i18n import _
@ -98,7 +97,6 @@ class LBPoolPollster(BaseLBPollster):
user_id=None,
project_id=pool['tenant_id'],
resource_id=pool['id'],
timestamp=timeutils.utcnow().isoformat(),
resource_metadata=self.extract_metadata(pool)
)
@ -146,7 +144,6 @@ class LBVipPollster(base.BaseServicesPollster):
user_id=None,
project_id=vip['tenant_id'],
resource_id=vip['id'],
timestamp=timeutils.utcnow().isoformat(),
resource_metadata=self.extract_metadata(vip)
)
@ -186,7 +183,6 @@ class LBMemberPollster(BaseLBPollster):
user_id=None,
project_id=member['tenant_id'],
resource_id=member['id'],
timestamp=timeutils.utcnow().isoformat(),
resource_metadata=self.extract_metadata(member)
)
@ -217,7 +213,6 @@ class LBHealthMonitorPollster(base.BaseServicesPollster):
user_id=None,
project_id=probe['tenant_id'],
resource_id=probe['id'],
timestamp=timeutils.utcnow().isoformat(),
resource_metadata=self.extract_metadata(probe)
)
@ -248,7 +243,6 @@ class _LBStatsPollster(base.BaseServicesPollster):
user_id=None,
project_id=pool['tenant_id'],
resource_id=pool['id'],
timestamp=timeutils.utcnow().isoformat(),
resource_metadata=resource_metadata,
)
@ -377,7 +371,6 @@ def make_sample_from_pool(pool, name, type, unit, volume,
user_id=None,
project_id=pool['tenant_id'],
resource_id=pool['id'],
timestamp=timeutils.utcnow().isoformat(),
resource_metadata=resource_metadata,
)
@ -423,7 +416,6 @@ class LBListenerPollster(BaseLBPollster):
user_id=None,
project_id=listener['tenant_id'],
resource_id=listener['id'],
timestamp=timeutils.utcnow().isoformat(),
resource_metadata=self.extract_metadata(listener)
)
@ -468,6 +460,5 @@ class LBLoadBalancerPollster(BaseLBPollster):
user_id=None,
project_id=loadbalancer['tenant_id'],
resource_id=loadbalancer['id'],
timestamp=timeutils.utcnow().isoformat(),
resource_metadata=self.extract_metadata(loadbalancer)
)

3
ceilometer/network/services/vpnaas.py
View File

@ -14,7 +14,6 @@
# under the License.
from oslo_log import log
from oslo_utils import timeutils
from ceilometer.i18n import _
from ceilometer.network.services import base
@ -59,7 +58,6 @@ class VPNServicesPollster(base.BaseServicesPollster):
user_id=None,
project_id=vpn['tenant_id'],
resource_id=vpn['id'],
timestamp=timeutils.utcnow().isoformat(),
resource_metadata=self.extract_metadata(vpn)
)
@ -102,6 +100,5 @@ class IPSecConnectionsPollster(base.BaseServicesPollster):
user_id=None,
project_id=conn['tenant_id'],
resource_id=conn['id'],
timestamp=timeutils.utcnow().isoformat(),
resource_metadata=self.extract_metadata(conn)
)

3
ceilometer/network/statistics/__init__.py
View File

@ -86,7 +86,7 @@ class _Base(plugin_base.PollsterBase):
if not isinstance(data, list):
data = [data]
for (volume, resource_id,
resource_metadata, timestamp) in data:
resource_metadata) in data:
yield sample.Sample(
name=self.meter_name,
@ -96,6 +96,5 @@ class _Base(plugin_base.PollsterBase):
user_id=None,
project_id=None,
resource_id=resource_id,
timestamp=timestamp,
resource_metadata=resource_metadata
)

4
ceilometer/network/statistics/opencontrail/driver.py
View File

@ -14,7 +14,6 @@
import re
from oslo_utils import timeutils
from six.moves.urllib import parse as urlparse
from ceilometer.network.statistics import driver
@ -104,7 +103,6 @@ class OpencontrailDriver(driver.Driver):
fqdn_uuid = params.get('fqdn_uuid', ['*'])[0]
virtual_network = params.get('virtual_network', [None])[0]
timestamp = timeutils.utcnow().isoformat()
statistics = data['o_client'].networks.get_vm_statistics(fqdn_uuid)
if not statistics:
return
@ -113,7 +111,7 @@ class OpencontrailDriver(driver.Driver):
for sample in iter(extractor, value, ports_map,
resource, virtual_network):
if sample is not None:
yield sample + (timestamp, )
yield sample
def _get_iter(self, meter_name):
if meter_name.startswith('switch.port'):

7
ceilometer/network/statistics/opendaylight/driver.py
View File

@ -14,7 +14,6 @@
# under the License.
from oslo_log import log
from oslo_utils import timeutils
import six
from six import moves
from six.moves.urllib import parse as urlparse
@ -161,9 +160,6 @@ class OpenDayLightDriver(driver.Driver):
cs.host_tracker.get_active_hosts(container_name))
container_data['inactive_hosts'] = (
cs.host_tracker.get_inactive_hosts(container_name))
container_data['timestamp'] = timeutils.utcnow().isoformat()
data[container_name] = container_data
except Exception:
LOG.exception(_('Request failed to connect to OpenDaylight'
@ -199,7 +195,6 @@ class OpenDayLightDriver(driver.Driver):
samples = []
for name, value in six.iteritems(data):
timestamp = value['timestamp']
for sample in iter(extractor, value):
if sample is not None:
# set controller name and container name
@ -207,7 +202,7 @@ class OpenDayLightDriver(driver.Driver):
sample[2]['controller'] = 'OpenDaylight'
sample[2]['container'] = name
samples.append(sample + (timestamp, ))
samples.append(sample)
return samples

7
ceilometer/objectstore/rgw.py
View File

@ -18,7 +18,6 @@
from keystoneauth1 import exceptions
from oslo_config import cfg
from oslo_log import log
from oslo_utils import timeutils
import six.moves.urllib.parse as urlparse
from ceilometer.agent import plugin_base
@ -117,7 +116,6 @@ class ContainersObjectsPollster(_Base):
user_id=None,
project_id=tenant,
resource_id=tenant + '/' + it.name,
timestamp=timeutils.utcnow().isoformat(),
resource_metadata=None,
)
@ -137,7 +135,6 @@ class ContainersSizePollster(_Base):
user_id=None,
project_id=tenant,
resource_id=tenant + '/' + it.name,
timestamp=timeutils.utcnow().isoformat(),
resource_metadata=None,
)
@ -156,7 +153,6 @@ class ObjectsSizePollster(_Base):
user_id=None,
project_id=tenant,
resource_id=tenant,
timestamp=timeutils.utcnow().isoformat(),
resource_metadata=None,
)
@ -175,7 +171,6 @@ class ObjectsPollster(_Base):
user_id=None,
project_id=tenant,
resource_id=tenant,
timestamp=timeutils.utcnow().isoformat(),
resource_metadata=None,
)
@ -192,7 +187,6 @@ class ObjectsContainersPollster(_Base):
user_id=None,
project_id=tenant,
resource_id=tenant,
timestamp=timeutils.utcnow().isoformat(),
resource_metadata=None,
)
@ -212,6 +206,5 @@ class UsagePollster(_Base):
user_id=None,
project_id=tenant,
resource_id=tenant,
timestamp=timeutils.utcnow().isoformat(),
resource_metadata=None,
)

6
ceilometer/objectstore/swift.py
View File

@ -20,7 +20,6 @@ from __future__ import absolute_import
from keystoneauth1 import exceptions
from oslo_config import cfg
from oslo_log import log
from oslo_utils import timeutils
import six.moves.urllib.parse as urlparse
from swiftclient import client as swift
@ -117,7 +116,6 @@ class ObjectsPollster(_Base):
user_id=None,
project_id=tenant,
resource_id=tenant,
timestamp=timeutils.utcnow().isoformat(),
resource_metadata=None,
)
@ -136,7 +134,6 @@ class ObjectsSizePollster(_Base):
user_id=None,
project_id=tenant,
resource_id=tenant,
timestamp=timeutils.utcnow().isoformat(),
resource_metadata=None,
)
@ -155,7 +152,6 @@ class ObjectsContainersPollster(_Base):
user_id=None,
project_id=tenant,
resource_id=tenant,
timestamp=timeutils.utcnow().isoformat(),
resource_metadata=None,
)
@ -179,7 +175,6 @@ class ContainersObjectsPollster(_Base):
user_id=None,
project_id=tenant,
resource_id=tenant + '/' + container['name'],
timestamp=timeutils.utcnow().isoformat(),
resource_metadata=None,
)
@ -203,6 +198,5 @@ class ContainersSizePollster(_Base):
user_id=None,
project_id=tenant,
resource_id=tenant + '/' + container['name'],
timestamp=timeutils.utcnow().isoformat(),
resource_metadata=None,
)

6
ceilometer/tests/unit/energy/test_kwapi.py
View File

@ -12,8 +12,6 @@
# License for the specific language governing permissions and limitations
# under the License.
import datetime
from keystoneauth1 import exceptions
import mock
from oslotest import base
@ -107,10 +105,6 @@ class TestEnergyPollster(_BaseTestCase):
samples_by_name = dict((s.resource_id, s) for s in samples)
for name, probe in PROBE_DICT['probes'].items():
sample = samples_by_name[name]
expected = datetime.datetime.fromtimestamp(
probe['timestamp']
).isoformat()
self.assertEqual(expected, sample.timestamp)
self.assertEqual(probe[self.unit], sample.volume)

30
ceilometer/tests/unit/network/statistics/opencontrail/test_driver.py
View File

@ -135,8 +135,7 @@ class TestOpencontrailDriver(base.BaseTestCase):
'network_id': '298a3088-a446-4d5a-bad8-f92ecacd786b',
'project': 'demo',
'project_id': '89271fa581ab4380bf172f868c3615f9',
'resource': 'if_stats_list'},
mock.ANY),
'resource': 'if_stats_list'},),
(13,
'96d49cc3-4e01-40ce-9cac-c0e32642a442',
{'device_owner_id':
@ -145,8 +144,7 @@ class TestOpencontrailDriver(base.BaseTestCase):
'network_id': '298a3088-a446-4d5a-bad8-f92ecacd786b',
'project': 'demo',
'project_id': '89271fa581ab4380bf172f868c3615f9',
'resource': 'if_stats_list'},
mock.ANY)]
'resource': 'if_stats_list'},)]
self._test_meter('switch.port.receive.packets', expected,
self.fake_port_stats_with_node())
@ -159,8 +157,7 @@ class TestOpencontrailDriver(base.BaseTestCase):
'network_id': '298a3088-a446-4d5a-bad8-f92ecacd786b',
'project': 'demo',
'project_id': '89271fa581ab4380bf172f868c3615f9',
'resource': 'if_stats_list'},
mock.ANY)]
'resource': 'if_stats_list'},)]
self._test_meter('switch.port.receive.packets', expected)
def test_switch_port_transmit_packets(self):
@ -172,8 +169,7 @@ class TestOpencontrailDriver(base.BaseTestCase):
'network_id': '298a3088-a446-4d5a-bad8-f92ecacd786b',
'project': 'demo',
'project_id': '89271fa581ab4380bf172f868c3615f9',
'resource': 'if_stats_list'},
mock.ANY)]
'resource': 'if_stats_list'},)]
self._test_meter('switch.port.transmit.packets', expected)
def test_switch_port_receive_bytes(self):
@ -185,8 +181,7 @@ class TestOpencontrailDriver(base.BaseTestCase):
'network_id': '298a3088-a446-4d5a-bad8-f92ecacd786b',
'project': 'demo',
'project_id': '89271fa581ab4380bf172f868c3615f9',
'resource': 'if_stats_list'},
mock.ANY)]
'resource': 'if_stats_list'},)]
self._test_meter('switch.port.receive.bytes', expected)
def test_switch_port_transmit_bytes(self):
@ -198,8 +193,7 @@ class TestOpencontrailDriver(base.BaseTestCase):
'network_id': '298a3088-a446-4d5a-bad8-f92ecacd786b',
'project': 'demo',
'project_id': '89271fa581ab4380bf172f868c3615f9',
'resource': 'if_stats_list'},
mock.ANY)]
'resource': 'if_stats_list'},)]
self._test_meter('switch.port.transmit.bytes', expected)
def test_switch_port_receive_packets_fip(self):
@ -212,8 +206,7 @@ class TestOpencontrailDriver(base.BaseTestCase):
'network_id': '298a3088-a446-4d5a-bad8-f92ecacd786b',
'project': 'demo',
'project_id': '89271fa581ab4380bf172f868c3615f9',
'resource': 'fip_stats_list'},
mock.ANY)]
'resource': 'fip_stats_list'},)]
self._test_meter('switch.port.receive.packets', expected)
def test_switch_port_transmit_packets_fip(self):
@ -226,8 +219,7 @@ class TestOpencontrailDriver(base.BaseTestCase):
'network_id': '298a3088-a446-4d5a-bad8-f92ecacd786b',
'project': 'demo',
'project_id': '89271fa581ab4380bf172f868c3615f9',
'resource': 'fip_stats_list'},
mock.ANY)]
'resource': 'fip_stats_list'},)]
self._test_meter('switch.port.transmit.packets', expected)
def test_switch_port_receive_bytes_fip(self):
@ -240,8 +232,7 @@ class TestOpencontrailDriver(base.BaseTestCase):
'network_id': '298a3088-a446-4d5a-bad8-f92ecacd786b',
'project': 'demo',
'project_id': '89271fa581ab4380bf172f868c3615f9',
'resource': 'fip_stats_list'},
mock.ANY)]
'resource': 'fip_stats_list'},)]
self._test_meter('switch.port.receive.bytes', expected)
def test_switch_port_transmit_bytes_fip(self):
@ -254,8 +245,7 @@ class TestOpencontrailDriver(base.BaseTestCase):
'network_id': '298a3088-a446-4d5a-bad8-f92ecacd786b',
'project': 'demo',
'project_id': '89271fa581ab4380bf172f868c3615f9',
'resource': 'fip_stats_list'},
mock.ANY)]
'resource': 'fip_stats_list'},)]
self._test_meter('switch.port.transmit.bytes', expected)
def test_switch_port_transmit_bytes_non_existing_network(self):

1
ceilometer/tests/unit/network/statistics/opendaylight/test_driver.py
View File

@ -122,7 +122,6 @@ class _Base(base.BaseTestCase):
self.assertEqual(expected[0], sample[0]) # check volume
self.assertEqual(expected[1], sample[1]) # check resource id
self.assertEqual(expected[2], sample[2]) # check resource metadata
self.assertIsNotNone(sample[3]) # timestamp
class TestOpenDayLightDriverSpecial(_Base):

62
ceilometer/tests/unit/network/statistics/test_statistics.py
View File

@ -105,75 +105,65 @@ class TestBaseGetSamples(base.BaseTestCase):
return [v for v in self.pollster.get_samples(self, {}, resources)]
def _assert_sample(self, s, volume, resource_id, resource_metadata,
timestamp):
self.assertEqual('foo', s.name)
self.assertEqual(sample.TYPE_CUMULATIVE, s.type)
self.assertEqual('bar', s.unit)
self.assertEqual(volume, s.volume)
self.assertIsNone(s.user_id)
self.assertIsNone(s.project_id)
self.assertEqual(resource_id, s.resource_id)
self.assertEqual(timestamp, s.timestamp)
self.assertEqual(resource_metadata, s.resource_metadata)
def _assert_sample(self, s, volume, resource_id, resource_metadata):
self.assertEqual('foo', s.name)
self.assertEqual(sample.TYPE_CUMULATIVE, s.type)
self.assertEqual('bar', s.unit)
self.assertEqual(volume, s.volume)
self.assertIsNone(s.user_id)
self.assertIsNone(s.project_id)
self.assertEqual(resource_id, s.resource_id)
self.assertEqual(resource_metadata, s.resource_metadata)
def test_get_samples_one_driver_one_resource(self):
times = self._make_timestamps(2)
fake_driver = self._make_fake_driver((1, 'a', {'spam': 'egg'},
times[0]),
(2, 'b', None, times[1]))
fake_driver = self._make_fake_driver((1, 'a', {'spam': 'egg'},),
(2, 'b', None))
self._setup_ext_mgr(http=fake_driver())
samples = self._get_samples('http://foo')
self.assertEqual(1, len(samples))
self._assert_sample(samples[0], 1, 'a', {'spam': 'egg'}, times[0])
self._assert_sample(samples[0], 1, 'a', {'spam': 'egg'})
def test_get_samples_one_driver_two_resource(self):
times = self._make_timestamps(3)
fake_driver = self._make_fake_driver((1, 'a', {'spam': 'egg'},
times[0]),
(2, 'b', None, times[1]),
(3, 'c', None, times[2]))
fake_driver = self._make_fake_driver((1, 'a', {'spam': 'egg'},),
(2, 'b', None),
(3, 'c', None))
self._setup_ext_mgr(http=fake_driver())
samples = self._get_samples('http://foo', 'http://bar')
self.assertEqual(2, len(samples))
self._assert_sample(samples[0], 1, 'a', {'spam': 'egg'}, times[0])
self._assert_sample(samples[1], 2, 'b', {}, times[1])
self._assert_sample(samples[0], 1, 'a', {'spam': 'egg'})
self._assert_sample(samples[1], 2, 'b', {})
def test_get_samples_two_driver_one_resource(self):
times = self._make_timestamps(4)
fake_driver1 = self._make_fake_driver((1, 'a', {'spam': 'egg'},
times[0]),
(2, 'b', None), times[1])
fake_driver1 = self._make_fake_driver((1, 'a', {'spam': 'egg'},),
(2, 'b', None))
fake_driver2 = self._make_fake_driver((11, 'A', None, times[2]),
(12, 'B', None, times[3]))
fake_driver2 = self._make_fake_driver((11, 'A', None),
(12, 'B', None))
self._setup_ext_mgr(http=fake_driver1(), https=fake_driver2())
samples = self._get_samples('http://foo')
self.assertEqual(1, len(samples))
self._assert_sample(samples[0], 1, 'a', {'spam': 'egg'}, times[0])
self._assert_sample(samples[0], 1, 'a', {'spam': 'egg'})
def test_get_samples_multi_samples(self):
times = self._make_timestamps(2)
fake_driver = self._make_fake_driver([(1, 'a', {'spam': 'egg'},
times[0]),
(2, 'b', None, times[1])])
fake_driver = self._make_fake_driver([(1, 'a', {'spam': 'egg'},),
(2, 'b', None)])
self._setup_ext_mgr(http=fake_driver())
samples = self._get_samples('http://foo')
self.assertEqual(2, len(samples))
self._assert_sample(samples[0], 1, 'a', {'spam': 'egg'}, times[0])
self._assert_sample(samples[1], 2, 'b', {}, times[1])
self._assert_sample(samples[0], 1, 'a', {'spam': 'egg'})
self._assert_sample(samples[1], 2, 'b', {})
def test_get_samples_return_none(self):
fake_driver = self._make_fake_driver(None)

7
releasenotes/notes/unify-timestamp-of-polled-data-fbfcff43cd2d04bc.yaml Normal file
View File

@ -0,0 +1,7 @@
---
fixes:
- >
[`bug 1491509 <https://bugs.launchpad.net/ceilometer/+bug/1491509>`_]
Patch to unify timestamp in samples polled by pollsters. Set the time
point polling starts as timestamp of samples, and drop timetamping in
pollsters.