Split meter publishing from the global config obj

The meter signing and publishing code was tightly coupled
to the global configuration object, making it more difficult
to reuse in an outside project. This change adds arguments
for the configuration settings actually being used in
those libraries and moves use of the global configuration
object higher up the stack to the manager. It also organizes
the definition of the options related to signing and
publishing metering messages so they can more easily by
registered on other configuration objects.

Change-Id: I8b1df0f228d0c2658841c4b7f7f4527414efc9a6
Signed-off-by: Doug Hellmann <doug.hellmann@dreamhost.com>
This commit is contained in:
Doug Hellmann 2012-09-26 17:31:48 -04:00
parent 0d4d97047c
commit 3cdac9f3d0
15 changed files with 158 additions and 91 deletions

View File

@ -20,6 +20,7 @@ import pkg_resources
from nova import manager
from ceilometer.openstack.common import cfg
from ceilometer.openstack.common import log
from ceilometer import publish
@ -65,7 +66,10 @@ class AgentManager(manager.Manager):
LOG.info('polling %s', name)
for c in pollster.get_counters(self, context):
LOG.info('COUNTER: %s', c)
publish.publish_counter(context, c)
publish.publish_counter(context, c,
cfg.CONF.metering_topic,
cfg.CONF.metering_secret,
)
except Exception as err:
LOG.warning('Continuing after error from %s: %s', name, err)
LOG.exception(err)

View File

@ -132,7 +132,7 @@ class CollectorManager(manager.Manager):
data['resource_id'],
data.get('timestamp', 'NO TIMESTAMP'),
data['counter_volume'])
if not meter.verify_signature(data):
if not meter.verify_signature(data, cfg.CONF.metering_secret):
LOG.warning('message signature invalid, discarding message: %r',
data)
else:

View File

@ -29,13 +29,16 @@ METER_OPTS = [
default='change this or be hacked',
help='Secret value for signing metering messages',
),
cfg.StrOpt('metering_topic',
default='metering',
help='the topic ceilometer uses for metering messages',
),
]
cfg.CONF.register_opts(METER_OPTS)
def register_opts(config):
"""Register the options for signing metering messages.
"""
config.register_opts(METER_OPTS)
register_opts(cfg.CONF)
def recursive_keypairs(d):
@ -49,10 +52,10 @@ def recursive_keypairs(d):
yield name, value
def compute_signature(message):
def compute_signature(message, secret):
"""Return the signature for a message dictionary.
"""
digest_maker = hmac.new(cfg.CONF.metering_secret, '', hashlib.sha256)
digest_maker = hmac.new(secret, '', hashlib.sha256)
for name, value in recursive_keypairs(message):
if name == 'message_signature':
# Skip any existing signature value, which would not have
@ -63,16 +66,16 @@ def compute_signature(message):
return digest_maker.hexdigest()
def verify_signature(message):
def verify_signature(message, secret):
"""Check the signature in the message against the value computed
from the rest of the contents.
"""
old_sig = message.get('message_signature')
new_sig = compute_signature(message)
new_sig = compute_signature(message, secret)
return new_sig == old_sig
def meter_message_from_counter(counter):
def meter_message_from_counter(counter, secret):
"""Make a metering message ready to be published or stored.
Returns a dictionary containing a metering message
@ -90,5 +93,5 @@ def meter_message_from_counter(counter):
'resource_metadata': counter.resource_metadata,
'message_id': str(uuid.uuid1()),
}
msg['message_signature'] = compute_signature(msg)
msg['message_signature'] = compute_signature(msg, secret)
return msg

View File

@ -18,15 +18,31 @@
"""Publish a counter using the preferred RPC mechanism.
"""
from ceilometer.openstack.common import cfg
from ceilometer.openstack.common import log
from ceilometer.openstack.common import rpc
from ceilometer.openstack.common import cfg
from ceilometer import meter
LOG = log.getLogger(__name__)
PUBLISH_OPTS = [
cfg.StrOpt('metering_topic',
default='metering',
help='the topic ceilometer uses for metering messages',
),
]
def publish_counter(context, counter):
def register_opts(config):
"""Register the options for publishing metering messages.
"""
config.register_opts(PUBLISH_OPTS)
register_opts(cfg.CONF)
def publish_counter(context, counter, topic, secret):
"""Send a metering message for the data represented by the counter.
:param context: Execution context from the service or RPC call
@ -35,11 +51,9 @@ def publish_counter(context, counter):
msg = {
'method': 'record_metering_data',
'version': '1.0',
'args': {'data': meter.meter_message_from_counter(counter),
'args': {'data': meter.meter_message_from_counter(counter, secret),
},
}
LOG.debug('PUBLISH: %s', str(msg))
rpc.cast(context, cfg.CONF.metering_topic, msg)
rpc.cast(context,
cfg.CONF.metering_topic + '.' + counter.name,
msg)
rpc.cast(context, topic, msg)
rpc.cast(context, topic + '.' + counter.name, msg)

View File

@ -23,6 +23,7 @@ import logging
from ceilometer import counter
from ceilometer import meter
from ceilometer.openstack.common import cfg
from ceilometer.tests import api as tests_api
@ -47,7 +48,9 @@ class TestListEvents(tests_api.TestBase):
'tag': 'self.counter',
}
)
msg = meter.meter_message_from_counter(self.counter1)
msg = meter.meter_message_from_counter(self.counter1,
cfg.CONF.metering_secret,
)
self.conn.record_metering_data(msg)
self.counter2 = counter.Counter(
@ -64,7 +67,9 @@ class TestListEvents(tests_api.TestBase):
'tag': 'self.counter2',
}
)
msg2 = meter.meter_message_from_counter(self.counter2)
msg2 = meter.meter_message_from_counter(self.counter2,
cfg.CONF.metering_secret,
)
self.conn.record_metering_data(msg2)
def test_all(self):

View File

@ -24,6 +24,7 @@ import logging
from ceilometer import counter
from ceilometer import meter
from ceilometer.openstack.common import cfg
from ceilometer.tests import api as tests_api
LOG = logging.getLogger(__name__)
@ -50,7 +51,9 @@ class TestListProjects(tests_api.TestBase):
'tag': 'self.counter',
}
)
msg = meter.meter_message_from_counter(counter1)
msg = meter.meter_message_from_counter(counter1,
cfg.CONF.metering_secret,
)
self.conn.record_metering_data(msg)
counter2 = counter.Counter(
@ -67,7 +70,9 @@ class TestListProjects(tests_api.TestBase):
'tag': 'self.counter2',
}
)
msg2 = meter.meter_message_from_counter(counter2)
msg2 = meter.meter_message_from_counter(counter2,
cfg.CONF.metering_secret,
)
self.conn.record_metering_data(msg2)
data = self.get('/projects')
@ -88,7 +93,9 @@ class TestListProjects(tests_api.TestBase):
'tag': 'self.counter',
}
)
msg = meter.meter_message_from_counter(counter1)
msg = meter.meter_message_from_counter(counter1,
cfg.CONF.metering_secret,
)
self.conn.record_metering_data(msg)
counter2 = counter.Counter(
@ -105,7 +112,9 @@ class TestListProjects(tests_api.TestBase):
'tag': 'self.counter2',
}
)
msg2 = meter.meter_message_from_counter(counter2)
msg2 = meter.meter_message_from_counter(counter2,
cfg.CONF.metering_secret,
)
self.conn.record_metering_data(msg2)
data = self.get('/sources/test_list_users/projects')

View File

@ -23,6 +23,7 @@ import logging
from ceilometer import counter
from ceilometer import meter
from ceilometer.openstack.common import cfg
from ceilometer.tests import api as tests_api
@ -50,7 +51,9 @@ class TestListResources(tests_api.TestBase):
'tag': 'self.counter',
}
)
msg = meter.meter_message_from_counter(counter1)
msg = meter.meter_message_from_counter(counter1,
cfg.CONF.metering_secret,
)
self.conn.record_metering_data(msg)
counter2 = counter.Counter(
@ -67,7 +70,9 @@ class TestListResources(tests_api.TestBase):
'tag': 'self.counter2',
}
)
msg2 = meter.meter_message_from_counter(counter2)
msg2 = meter.meter_message_from_counter(counter2,
cfg.CONF.metering_secret,
)
self.conn.record_metering_data(msg2)
data = self.get('/resources')
@ -88,7 +93,9 @@ class TestListResources(tests_api.TestBase):
'tag': 'self.counter',
}
)
msg = meter.meter_message_from_counter(counter1)
msg = meter.meter_message_from_counter(counter1,
cfg.CONF.metering_secret,
)
self.conn.record_metering_data(msg)
counter2 = counter.Counter(
@ -105,7 +112,9 @@ class TestListResources(tests_api.TestBase):
'tag': 'self.counter2',
}
)
msg2 = meter.meter_message_from_counter(counter2)
msg2 = meter.meter_message_from_counter(counter2,
cfg.CONF.metering_secret,
)
self.conn.record_metering_data(msg2)
data = self.get('/sources/test_list_resources/resources')
@ -127,7 +136,9 @@ class TestListResources(tests_api.TestBase):
'tag': 'self.counter',
}
)
msg = meter.meter_message_from_counter(counter1)
msg = meter.meter_message_from_counter(counter1,
cfg.CONF.metering_secret,
)
self.conn.record_metering_data(msg)
counter2 = counter.Counter(
@ -144,7 +155,9 @@ class TestListResources(tests_api.TestBase):
'tag': 'self.counter2',
}
)
msg2 = meter.meter_message_from_counter(counter2)
msg2 = meter.meter_message_from_counter(counter2,
cfg.CONF.metering_secret,
)
self.conn.record_metering_data(msg2)
data = self.get('/users/user-id/resources')
@ -166,7 +179,9 @@ class TestListResources(tests_api.TestBase):
'tag': 'self.counter',
}
)
msg = meter.meter_message_from_counter(counter1)
msg = meter.meter_message_from_counter(counter1,
cfg.CONF.metering_secret,
)
self.conn.record_metering_data(msg)
counter2 = counter.Counter(
@ -183,7 +198,9 @@ class TestListResources(tests_api.TestBase):
'tag': 'self.counter2',
}
)
msg2 = meter.meter_message_from_counter(counter2)
msg2 = meter.meter_message_from_counter(counter2,
cfg.CONF.metering_secret,
)
self.conn.record_metering_data(msg2)
data = self.get('/projects/project-id/resources')

View File

@ -24,6 +24,7 @@ import logging
from ceilometer import counter
from ceilometer import meter
from ceilometer.openstack.common import cfg
from ceilometer.tests import api as tests_api
LOG = logging.getLogger(__name__)
@ -50,7 +51,9 @@ class TestListUsers(tests_api.TestBase):
'tag': 'self.counter',
}
)
msg = meter.meter_message_from_counter(counter1)
msg = meter.meter_message_from_counter(counter1,
cfg.CONF.metering_secret,
)
self.conn.record_metering_data(msg)
counter2 = counter.Counter(
@ -67,7 +70,9 @@ class TestListUsers(tests_api.TestBase):
'tag': 'self.counter2',
}
)
msg2 = meter.meter_message_from_counter(counter2)
msg2 = meter.meter_message_from_counter(counter2,
cfg.CONF.metering_secret,
)
self.conn.record_metering_data(msg2)
data = self.get('/users')
@ -88,7 +93,9 @@ class TestListUsers(tests_api.TestBase):
'tag': 'self.counter',
}
)
msg = meter.meter_message_from_counter(counter1)
msg = meter.meter_message_from_counter(counter1,
cfg.CONF.metering_secret,
)
self.conn.record_metering_data(msg)
counter2 = counter.Counter(
@ -105,7 +112,9 @@ class TestListUsers(tests_api.TestBase):
'tag': 'self.counter2',
}
)
msg2 = meter.meter_message_from_counter(counter2)
msg2 = meter.meter_message_from_counter(counter2,
cfg.CONF.metering_secret,
)
self.conn.record_metering_data(msg2)
data = self.get('/sources/test_list_users/users')

View File

@ -19,7 +19,6 @@
"""
import datetime
import logging
from ming import mim
from nose.plugins import skip
@ -27,6 +26,7 @@ from nose.plugins import skip
from ceilometer import counter
from ceilometer import meter
from ceilometer.openstack.common import cfg
from ceilometer.tests import api as tests_api
@ -61,7 +61,7 @@ class TestMaxResourceVolume(tests_api.TestBase):
}
)
self.counters.append(c)
msg = meter.meter_message_from_counter(c)
msg = meter.meter_message_from_counter(c, cfg.CONF.metering_secret)
self.conn.record_metering_data(msg)
def test_no_time_bounds(self):

View File

@ -19,7 +19,6 @@
"""
import datetime
import logging
from ming import mim
from nose.plugins import skip
@ -27,6 +26,7 @@ from nose.plugins import skip
from ceilometer import counter
from ceilometer import meter
from ceilometer.openstack.common import cfg
from ceilometer.tests import api as tests_api
@ -61,7 +61,7 @@ class TestSumResourceVolume(tests_api.TestBase):
}
)
self.counters.append(c)
msg = meter.meter_message_from_counter(c)
msg = meter.meter_message_from_counter(c, cfg.CONF.metering_secret)
self.conn.record_metering_data(msg)
def test_no_time_bounds(self):

View File

@ -22,6 +22,7 @@ import datetime
from ceilometer import meter
from ceilometer.collector import manager
from ceilometer.openstack.common import cfg
from ceilometer.storage import base
from ceilometer.openstack.common import rpc
from ceilometer.openstack.common import cfg
@ -96,6 +97,7 @@ class TestCollectorManager(tests_base.TestCase):
super(TestCollectorManager, self).setUp()
self.mgr = manager.CollectorManager()
self.ctx = None
#cfg.CONF.metering_secret = 'not-so-secret'
def test_init_host(self):
self.stubs.Set(rpc, 'create_connection', lambda: StubConnection())
@ -107,7 +109,10 @@ class TestCollectorManager(tests_base.TestCase):
'resource_id': self.id(),
'counter_volume': 1,
}
msg['message_signature'] = meter.compute_signature(msg)
msg['message_signature'] = meter.compute_signature(
msg,
cfg.CONF.metering_secret,
)
self.mgr.storage_conn = self.mox.CreateMock(base.Connection)
self.mgr.storage_conn.record_metering_data(msg)
@ -143,7 +148,10 @@ class TestCollectorManager(tests_base.TestCase):
'counter_volume': 1,
'timestamp': '2012-07-02T13:53:40Z',
}
msg['message_signature'] = meter.compute_signature(msg)
msg['message_signature'] = meter.compute_signature(
msg,
cfg.CONF.metering_secret,
)
expected = {}
expected.update(msg)

View File

@ -115,7 +115,8 @@ class MongoDBEngineTestBase(unittest.TestCase):
'tag': 'self.counter',
}
)
self.msg = meter.meter_message_from_counter(self.counter)
self.msg = meter.meter_message_from_counter(self.counter,
'not-so-secret')
self.conn.record_metering_data(self.msg)
self.counter2 = counter.Counter(
@ -132,7 +133,8 @@ class MongoDBEngineTestBase(unittest.TestCase):
'tag': 'self.counter2',
}
)
self.msg2 = meter.meter_message_from_counter(self.counter2)
self.msg2 = meter.meter_message_from_counter(self.counter2,
'not-so-secret')
self.conn.record_metering_data(self.msg2)
self.counter3 = counter.Counter(
@ -149,7 +151,8 @@ class MongoDBEngineTestBase(unittest.TestCase):
'tag': 'self.counter3',
}
)
self.msg3 = meter.meter_message_from_counter(self.counter3)
self.msg3 = meter.meter_message_from_counter(self.counter3,
'not-so-secret')
self.conn.record_metering_data(self.msg3)
for i in range(2, 4):
@ -167,7 +170,7 @@ class MongoDBEngineTestBase(unittest.TestCase):
'tag': 'counter-%s' % i,
}
)
msg = meter.meter_message_from_counter(c)
msg = meter.meter_message_from_counter(c, 'not-so-secret')
self.conn.record_metering_data(msg)
@ -487,7 +490,7 @@ class TestGetEventInterval(MongoDBEngineTestBase):
resource_metadata={'display_name': 'test-server',
}
)
msg = meter.meter_message_from_counter(c)
msg = meter.meter_message_from_counter(c, 'not-so-secret')
self.conn.record_metering_data(msg)
def test_before_range(self):

View File

@ -22,19 +22,14 @@ import os
import re
import unittest
import mox
from nose.plugins import skip
from sqlalchemy import MetaData, text
from ceilometer import counter
from ceilometer import meter
from ceilometer import storage
from ceilometer.storage import migration
import ceilometer.openstack.common.cfg as cfg
from ceilometer.openstack.common import cfg
from ceilometer.storage import impl_sqlalchemy
from ceilometer.storage.sqlalchemy.models import Meter, Project, Resource
from ceilometer.storage.sqlalchemy.models import Source, User
from ceilometer.storage.sqlalchemy.models import User
LOG = logging.getLogger(__name__)
@ -60,8 +55,8 @@ class SQLAlchemyEngineTestBase(unittest.TestCase):
super(SQLAlchemyEngineTestBase, self).tearDown()
engine_conn = self.session.bind.connect()
if CEILOMETER_TEST_LIVE:
engine_conn.execute(text('drop database %s' % MYSQL_DBNAME))
engine_conn.execute(text('create database %s' % MYSQL_DBNAME))
engine_conn.execute('drop database %s' % MYSQL_DBNAME)
engine_conn.execute('create database %s' % MYSQL_DBNAME)
# needed for sqlite in-memory db to destroy
self.session.close_all()
self.session.bind.dispose()
@ -97,7 +92,9 @@ class SQLAlchemyEngineTestBase(unittest.TestCase):
'tag': 'self.counter',
}
)
self.msg1 = meter.meter_message_from_counter(self.counter)
self.msg1 = meter.meter_message_from_counter(self.counter,
cfg.CONF.metering_secret,
)
self.conn.record_metering_data(self.msg1)
self.counter2 = counter.Counter(
@ -114,7 +111,9 @@ class SQLAlchemyEngineTestBase(unittest.TestCase):
'tag': 'self.counter2',
}
)
self.msg2 = meter.meter_message_from_counter(self.counter2)
self.msg2 = meter.meter_message_from_counter(self.counter2,
cfg.CONF.metering_secret,
)
self.conn.record_metering_data(self.msg2)
self.counter3 = counter.Counter(
@ -131,7 +130,9 @@ class SQLAlchemyEngineTestBase(unittest.TestCase):
'tag': 'self.counter3',
}
)
self.msg3 = meter.meter_message_from_counter(self.counter3)
self.msg3 = meter.meter_message_from_counter(self.counter3,
cfg.CONF.metering_secret,
)
self.conn.record_metering_data(self.msg3)
for i in range(2, 4):
@ -149,7 +150,7 @@ class SQLAlchemyEngineTestBase(unittest.TestCase):
'tag': 'counter-%s' % i,
}
)
msg = meter.meter_message_from_counter(c)
msg = meter.meter_message_from_counter(c, cfg.CONF.metering_secret)
self.conn.record_metering_data(msg)
@ -162,11 +163,9 @@ class UserTest(SQLAlchemyEngineTestBase):
def test_new_user_source(self):
user = self.session.query(User).get('user-id')
assert hasattr(user, 'sources')
sources = user.sources
assert map(lambda x: x.id, user.sources) == ['test-1', 'test-2']
def test_get_users(self):
users = self.conn.get_users()
xpct = set(['user-id', 'user-id-alternate', 'user-id-2', 'user-id-3'])
assert set(self.conn.get_users()) == xpct
@ -413,7 +412,7 @@ class TestGetEventInterval(SQLAlchemyEngineTestBase):
resource_metadata={'display_name': 'test-server',
}
)
msg = meter.meter_message_from_counter(c)
msg = meter.meter_message_from_counter(c, cfg.CONF.metering_secret)
self.conn.record_metering_data(msg)
def test_before_range(self):

View File

@ -20,63 +20,57 @@
from ceilometer import counter
from ceilometer import meter
from ceilometer.openstack.common import cfg
def test_compute_signature_change_key():
sig1 = meter.compute_signature({'a': 'A', 'b': 'B'})
sig2 = meter.compute_signature({'A': 'A', 'b': 'B'})
sig1 = meter.compute_signature({'a': 'A', 'b': 'B'}, 'not-so-secret')
sig2 = meter.compute_signature({'A': 'A', 'b': 'B'}, 'not-so-secret')
assert sig1 != sig2
def test_compute_signature_change_value():
sig1 = meter.compute_signature({'a': 'A', 'b': 'B'})
sig2 = meter.compute_signature({'a': 'a', 'b': 'B'})
sig1 = meter.compute_signature({'a': 'A', 'b': 'B'}, 'not-so-secret')
sig2 = meter.compute_signature({'a': 'a', 'b': 'B'}, 'not-so-secret')
assert sig1 != sig2
def test_compute_signature_same():
sig1 = meter.compute_signature({'a': 'A', 'b': 'B'})
sig2 = meter.compute_signature({'a': 'A', 'b': 'B'})
sig1 = meter.compute_signature({'a': 'A', 'b': 'B'}, 'not-so-secret')
sig2 = meter.compute_signature({'a': 'A', 'b': 'B'}, 'not-so-secret')
assert sig1 == sig2
def test_compute_signature_signed():
data = {'a': 'A', 'b': 'B'}
sig1 = meter.compute_signature(data)
sig1 = meter.compute_signature(data, 'not-so-secret')
data['message_signature'] = sig1
sig2 = meter.compute_signature(data)
sig2 = meter.compute_signature(data, 'not-so-secret')
assert sig1 == sig2
def test_compute_signature_use_configured_secret():
data = {'a': 'A', 'b': 'B'}
sig1 = meter.compute_signature(data)
old_secret = cfg.CONF.metering_secret
try:
cfg.CONF.metering_secret = 'not the default value'
sig2 = meter.compute_signature(data)
finally:
cfg.CONF.metering_secret = old_secret
sig1 = meter.compute_signature(data, 'not-so-secret')
sig2 = meter.compute_signature(data, 'different-value')
assert sig1 != sig2
def test_verify_signature_signed():
data = {'a': 'A', 'b': 'B'}
sig1 = meter.compute_signature(data)
sig1 = meter.compute_signature(data, 'not-so-secret')
data['message_signature'] = sig1
assert meter.verify_signature(data)
assert meter.verify_signature(data, 'not-so-secret')
def test_verify_signature_unsigned():
data = {'a': 'A', 'b': 'B'}
assert not meter.verify_signature(data)
assert not meter.verify_signature(data, 'not-so-secret')
def test_verify_signature_incorrect():
data = {'a': 'A', 'b': 'B',
'message_signature': 'Not the same'}
assert not meter.verify_signature(data)
assert not meter.verify_signature(data, 'not-so-secret')
def test_recursive_keypairs():
@ -101,8 +95,8 @@ def test_verify_signature_nested():
'b': 'B',
},
}
data['message_signature'] = meter.compute_signature(data)
assert meter.verify_signature(data)
data['message_signature'] = meter.compute_signature(data, 'not-so-secret')
assert meter.verify_signature(data, 'not-so-secret')
TEST_COUNTER = counter.Counter(source='src',
@ -156,14 +150,14 @@ TEST_NOTICE = {
def test_meter_message_from_counter_signed():
msg = meter.meter_message_from_counter(TEST_COUNTER)
msg = meter.meter_message_from_counter(TEST_COUNTER, 'not-so-secret')
assert 'message_signature' in msg
def test_meter_message_from_counter_field():
def compare(f, c, msg_f, msg):
assert msg == c
msg = meter.meter_message_from_counter(TEST_COUNTER)
msg = meter.meter_message_from_counter(TEST_COUNTER, 'not-so-secret')
name_map = {'name': 'counter_name',
'type': 'counter_type',
'volume': 'counter_volume',

View File

@ -19,7 +19,6 @@
"""
import datetime
import unittest
from ceilometer.openstack.common import rpc
from ceilometer.tests import base
@ -51,7 +50,10 @@ class TestPublish(base.TestCase):
super(TestPublish, self).setUp()
self.notifications = []
self.stubs.Set(rpc, 'cast', self.faux_notify)
publish.publish_counter(None, self.test_data)
publish.publish_counter(None,
self.test_data,
'metering',
'not-so-secret')
def test_notify(self):
assert len(self.notifications) == 2