From 022ecffe56c1e607d76840200023998153ea4b42 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Fri, 29 Jun 2012 15:02:12 -0400 Subject: [PATCH] Convert timestamps to datetime objects before storing Parse the incoming timestamp strings into datetime objects before passing them off to the storage backend. Change-Id: I42d825c9633fa2c120409d79883756d23b045a60 Signed-off-by: Doug Hellmann --- ceilometer/collector/manager.py | 13 ++- ceilometer/compute/libvirt.py | 5 +- ceilometer/openstack/common/timeutils.py | 109 +++++++++++++++++++++++ openstack-common.conf | 2 +- tests/collector/test_manager.py | 20 +++++ 5 files changed, 143 insertions(+), 6 deletions(-) create mode 100644 ceilometer/openstack/common/timeutils.py diff --git a/ceilometer/collector/manager.py b/ceilometer/collector/manager.py index d63ef96a..0ca0f5fb 100644 --- a/ceilometer/collector/manager.py +++ b/ceilometer/collector/manager.py @@ -24,9 +24,10 @@ from ceilometer import log from ceilometer import meter from ceilometer import publish from ceilometer import rpc -from ceilometer.collector import dispatcher from ceilometer import storage +from ceilometer.collector import dispatcher from ceilometer.openstack.common import cfg +from ceilometer.openstack.common import timeutils from ceilometer.openstack.common.rpc import dispatcher as rpc_dispatcher # FIXME(dhellmann): There must be another way to do this. @@ -88,15 +89,23 @@ class CollectorManager(manager.Manager): cast from an agent. """ #LOG.info('metering data: %r', data) - LOG.info('metering data %s for %s: %s', + LOG.info('metering data %s for %s @ %s: %s', data['counter_name'], data['resource_id'], + data.get('timestamp', 'NO TIMESTAMP'), data['counter_volume']) if not meter.verify_signature(data): LOG.warning('message signature invalid, discarding message: %r', data) else: try: + # Convert the timestamp to a datetime instance. + # Storage engines are responsible for converting + # that value to something they can store. + if 'timestamp' in data: + data['timestamp'] = timeutils.parse_isotime( + data['timestamp'], + ) self.storage_conn.record_metering_data(data) except Exception as err: LOG.error('Failed to record metering data: %s', err) diff --git a/ceilometer/compute/libvirt.py b/ceilometer/compute/libvirt.py index 25bdb9b2..ad9c4f3d 100644 --- a/ceilometer/compute/libvirt.py +++ b/ceilometer/compute/libvirt.py @@ -16,8 +16,6 @@ # License for the specific language governing permissions and limitations # under the License. -import datetime - from lxml import etree from nova import flags @@ -27,6 +25,7 @@ from ceilometer import log from ceilometer import counter from ceilometer import plugin from ceilometer.compute import instance as compute_instance +from ceilometer.openstack.common import timeutils FLAGS = flags.FLAGS @@ -42,7 +41,7 @@ def make_counter_from_instance(instance, name, type, volume): user_id=instance.user_id, project_id=instance.project_id, resource_id=instance.uuid, - timestamp=datetime.datetime.utcnow().isoformat(), + timestamp=timeutils.isotime(), duration=None, resource_metadata=compute_instance.get_metadata_from_dbobject( instance), diff --git a/ceilometer/openstack/common/timeutils.py b/ceilometer/openstack/common/timeutils.py new file mode 100644 index 00000000..5eeaf70a --- /dev/null +++ b/ceilometer/openstack/common/timeutils.py @@ -0,0 +1,109 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Time related utilities and helper functions. +""" + +import calendar +import datetime +import time + +import iso8601 + + +TIME_FORMAT = "%Y-%m-%dT%H:%M:%S" +PERFECT_TIME_FORMAT = "%Y-%m-%dT%H:%M:%S.%f" + + +def isotime(at=None): + """Stringify time in ISO 8601 format""" + if not at: + at = utcnow() + str = at.strftime(TIME_FORMAT) + tz = at.tzinfo.tzname(None) if at.tzinfo else 'UTC' + str += ('Z' if tz == 'UTC' else tz) + return str + + +def parse_isotime(timestr): + """Parse time from ISO 8601 format""" + try: + return iso8601.parse_date(timestr) + except iso8601.ParseError as e: + raise ValueError(e.message) + except TypeError as e: + raise ValueError(e.message) + + +def strtime(at=None, fmt=PERFECT_TIME_FORMAT): + """Returns formatted utcnow.""" + if not at: + at = utcnow() + return at.strftime(fmt) + + +def parse_strtime(timestr, fmt=PERFECT_TIME_FORMAT): + """Turn a formatted time back into a datetime.""" + return datetime.datetime.strptime(timestr, fmt) + + +def normalize_time(timestamp): + """Normalize time in arbitrary timezone to UTC""" + offset = timestamp.utcoffset() + return timestamp.replace(tzinfo=None) - offset if offset else timestamp + + +def is_older_than(before, seconds): + """Return True if before is older than seconds.""" + return utcnow() - before > datetime.timedelta(seconds=seconds) + + +def utcnow_ts(): + """Timestamp version of our utcnow function.""" + return calendar.timegm(utcnow().timetuple()) + + +def utcnow(): + """Overridable version of utils.utcnow.""" + if utcnow.override_time: + return utcnow.override_time + return datetime.datetime.utcnow() + + +utcnow.override_time = None + + +def set_time_override(override_time=datetime.datetime.utcnow()): + """Override utils.utcnow to return a constant time.""" + utcnow.override_time = override_time + + +def advance_time_delta(timedelta): + """Advance overriden time using a datetime.timedelta.""" + assert(not utcnow.override_time is None) + utcnow.override_time += timedelta + + +def advance_time_seconds(seconds): + """Advance overriden time by seconds.""" + advance_time_delta(datetime.timedelta(0, seconds)) + + +def clear_time_override(): + """Remove the overridden time.""" + utcnow.override_time = None diff --git a/openstack-common.conf b/openstack-common.conf index 30c96fb3..2d6ccb0c 100644 --- a/openstack-common.conf +++ b/openstack-common.conf @@ -1,3 +1,3 @@ [DEFAULT] -modules=cfg,iniparser,rpc,importutils,excutils,local,jsonutils,gettextutils +modules=cfg,iniparser,rpc,importutils,excutils,local,jsonutils,gettextutils,timeutils base=ceilometer \ No newline at end of file diff --git a/tests/collector/test_manager.py b/tests/collector/test_manager.py index 5806b258..721e2769 100644 --- a/tests/collector/test_manager.py +++ b/tests/collector/test_manager.py @@ -69,3 +69,23 @@ class TestCollectorManager(test.TestCase): assert not self.mgr.storage_conn.called, \ 'Should not have called the storage connection' + + def test_timestamp_conversion(self): + msg = {'counter_name': 'test', + 'resource_id': self.id(), + 'counter_volume': 1, + 'timestamp': '2012-07-02T13:53:40Z', + } + msg['message_signature'] = meter.compute_signature(msg) + + expected = {} + expected.update(msg) + expected['timestamp'] = datetime.datetime(2012, 7, 2, 13, 53, 40) + + self.mgr.storage_conn = self.mox.CreateMock(base.Connection) + self.mgr.storage_conn.record_metering_data(expected) + self.mox.ReplayAll() + + self.mgr.record_metering_data(self.ctx, msg) + self.mox.VerifyAll() +