openstack/os-collect-config
Code Issues Proposed changes

Add support for zaqar transport

Browse Source 
This adds a new zaqar collector able to read configuration from a
specific zaqar queue.

blueprint software-config-zaqar
Change-Id: Ie38af7b59e7a1aa370ac7760bb7819e37c2165c3
This commit is contained in:
Thomas Herve 2015-04-22 10:37:04 +02:00 committed by Thomas Herve
parent ca089dfcc2
commit b13122918c
6 changed files with 271 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
os_collect_config/collect.py
View File

@ -35,8 +35,11 @@ from os_collect_config import local
from os_collect_config.openstack.common import log
from os_collect_config import request
from os_collect_config import version
from os_collect_config import zaqar
DEFAULT_COLLECTORS = ['heat_local', 'ec2', 'cfn', 'heat', 'request', 'local',
'zaqar']
DEFAULT_COLLECTORS = ['heat_local', 'ec2', 'cfn', 'heat', 'request', 'local']
opts = [
cfg.StrOpt('command', short='c',
help='Command to run on metadata changes. If specified,'
@ -92,7 +95,8 @@ COLLECTORS = {ec2.name: ec2,
heat.name: heat,
heat_local.name: heat_local,
local.name: local,
request.name: request}
request.name: request,
zaqar.name: zaqar}
def setup_conf():
@ -111,6 +115,9 @@ def setup_conf():
heat_group = cfg.OptGroup(name='heat',
title='Heat Metadata options')
zaqar_group = cfg.OptGroup(name='zaqar',
title='Zaqar queue options')
request_group = cfg.OptGroup(name='request',
title='Request Metadata options')
@ -124,6 +131,7 @@ def setup_conf():
CONF.register_group(heat_group)
CONF.register_group(request_group)
CONF.register_group(keystone_group)
CONF.register_group(zaqar_group)
CONF.register_cli_opts(ec2.opts, group='ec2')
CONF.register_cli_opts(cfn.opts, group='cfn')
CONF.register_cli_opts(heat_local.opts, group='heat_local')
@ -131,6 +139,7 @@ def setup_conf():
CONF.register_cli_opts(heat.opts, group='heat')
CONF.register_cli_opts(request.opts, group='request')
CONF.register_cli_opts(keystone.opts, group='keystone')
CONF.register_cli_opts(zaqar.opts, group='zaqar')
CONF.register_cli_opts(opts)

8
os_collect_config/exc.py
View File

@ -58,5 +58,13 @@ class RequestMetadataNotConfigured(SourceNotAvailable):
"""The request metadata is not fully configured."""
class ZaqarMetadataNotConfigured(SourceNotConfigured):
"""The zaqar metadata service is not fully configured."""
class ZaqarMetadataNotAvailable(SourceNotAvailable):
"""The Zaqar metadata is not available."""
class InvalidArguments(ValueError):
"""Invalid arguments."""

26
os_collect_config/tests/test_collect.py
View File

@ -37,6 +37,7 @@ from os_collect_config.tests import test_heat
from os_collect_config.tests import test_heat_local
from os_collect_config.tests import test_local
from os_collect_config.tests import test_request
from os_collect_config.tests import test_zaqar
def _setup_heat_local_metadata(test_case):
@ -76,6 +77,10 @@ class TestCollect(testtools.TestCase):
'heatclient': test_heat.FakeHeatClient(self)
},
'request': {'requests_impl': test_request.FakeRequests},
'zaqar': {
'keystoneclient': test_zaqar.FakeKeystoneClient(self),
'zaqarclient': test_zaqar.FakeZaqarClient(self)
},
}
return collect.__main__(args=fake_args,
collector_kwargs_map=collector_kwargs_map)
@ -352,6 +357,11 @@ class TestCollectAll(testtools.TestCase):
cfg.CONF.heat.resource_name = 'server'
cfg.CONF.local.path = [_setup_local_metadata(self)]
cfg.CONF.request.metadata_url = 'http://127.0.0.1:8000/my_metadata/'
cfg.CONF.zaqar.auth_url = 'http://127.0.0.1:5000/v3'
cfg.CONF.zaqar.user_id = '0123456789ABCDEF'
cfg.CONF.zaqar.password = 'FEDCBA9876543210'
cfg.CONF.zaqar.project_id = '9f6b09df-4d7f-4a33-8ec3-9924d8f46f10'
cfg.CONF.zaqar.queue_id = '4f3f46d3-09f1-42a7-8c13-f91a5457192c'
@mock.patch.object(ks_discover.Discover, '__init__')
@mock.patch.object(ks_discover.Discover, 'url_for')
@ -368,6 +378,10 @@ class TestCollectAll(testtools.TestCase):
'heatclient': test_heat.FakeHeatClient(self)
},
'request': {'requests_impl': test_request.FakeRequests},
'zaqar': {
'keystoneclient': test_zaqar.FakeKeystoneClient(self),
'zaqarclient': test_zaqar.FakeZaqarClient(self)
},
}
if collectors is None:
collectors = cfg.CONF.collectors
@ -382,7 +396,7 @@ class TestCollectAll(testtools.TestCase):
store=True, collector_kwargs_map=collector_kwargs_map)
if expected_changed is None:
expected_changed = set(['heat_local', 'cfn', 'ec2',
'heat', 'local', 'request'])
'heat', 'local', 'request', 'zaqar'])
self.assertEqual(expected_changed, changed_keys)
self.assertThat(paths, matchers.IsInstance(list))
for path in paths:
@ -402,10 +416,14 @@ class TestCollectAll(testtools.TestCase):
'heatclient': test_heat.FakeHeatClient(self)
},
'request': {'requests_impl': test_request.FakeRequests},
'zaqar': {
'keystoneclient': test_zaqar.FakeKeystoneClient(self),
'zaqarclient': test_zaqar.FakeZaqarClient(self)
},
}
expected_changed = set((
'heat_local', 'ec2', 'cfn', 'heat', 'local', 'request',
'dep-name1', 'dep-name2', 'dep-name3'))
'dep-name1', 'dep-name2', 'dep-name3', 'zaqar'))
self._test_collect_all_store(collector_kwargs_map=soft_config_map,
expected_changed=expected_changed)
@ -441,6 +459,10 @@ class TestCollectAll(testtools.TestCase):
'heatclient': test_heat.FakeHeatClient(self)
},
'request': {'requests_impl': test_request.FakeRequests},
'zaqar': {
'keystoneclient': test_zaqar.FakeKeystoneClient(self),
'zaqarclient': test_zaqar.FakeZaqarClient(self)
},
}
(changed_keys, paths) = self._call_collect_all(
store=True, collector_kwargs_map=soft_config_map)

133
os_collect_config/tests/test_zaqar.py Normal file
View File

@ -0,0 +1,133 @@
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import fixtures
from keystoneclient import discover as ks_discover
import mock
from oslo_config import cfg
import testtools
from testtools import matchers
from zaqarclient.queues.v1 import message
from os_collect_config import collect
from os_collect_config import exc
from os_collect_config.tests import test_heat
from os_collect_config import zaqar
class FakeKeystoneClient(test_heat.FakeKeystoneClient):
def url_for(self, service_type, endpoint_type):
self._test.assertEqual('messaging', service_type)
self._test.assertEqual('publicURL', endpoint_type)
return 'http://127.0.0.1:8888/'
class FakeZaqarClient(object):
def __init__(self, testcase):
self._test = testcase
def Client(self, endpoint, conf, version):
self._test.assertEqual(1.1, version)
self._test.assertEqual('http://127.0.0.1:8888/', endpoint)
return self
def queue(self, queue_id):
self._test.assertEqual(
'4f3f46d3-09f1-42a7-8c13-f91a5457192c', queue_id)
return FakeQueue()
class FakeQueue(object):
def pop(self):
return iter([message.Message(
queue=self, ttl=10, age=10, body=test_heat.META_DATA, href='')])
class TestZaqar(testtools.TestCase):
def setUp(self):
super(TestZaqar, self).setUp()
self.log = self.useFixture(fixtures.FakeLogger())
self.useFixture(fixtures.NestedTempfile())
collect.setup_conf()
cfg.CONF.zaqar.auth_url = 'http://127.0.0.1:5000/v3'
cfg.CONF.zaqar.user_id = '0123456789ABCDEF'
cfg.CONF.zaqar.password = 'FEDCBA9876543210'
cfg.CONF.zaqar.project_id = '9f6b09df-4d7f-4a33-8ec3-9924d8f46f10'
cfg.CONF.zaqar.queue_id = '4f3f46d3-09f1-42a7-8c13-f91a5457192c'
@mock.patch.object(ks_discover.Discover, '__init__')
@mock.patch.object(ks_discover.Discover, 'url_for')
def test_collect_zaqar(self, mock_url_for, mock___init__):
mock___init__.return_value = None
mock_url_for.return_value = cfg.CONF.zaqar.auth_url
zaqar_md = zaqar.Collector(
keystoneclient=FakeKeystoneClient(self, cfg.CONF.zaqar),
zaqarclient=FakeZaqarClient(self)).collect()
self.assertThat(zaqar_md, matchers.IsInstance(list))
self.assertEqual('zaqar', zaqar_md[0][0])
zaqar_md = zaqar_md[0][1]
for k in ('int1', 'strfoo', 'map_ab'):
self.assertIn(k, zaqar_md)
self.assertEqual(zaqar_md[k], test_heat.META_DATA[k])
@mock.patch.object(ks_discover.Discover, '__init__')
@mock.patch.object(ks_discover.Discover, 'url_for')
def test_collect_zaqar_fail(self, mock_url_for, mock___init__):
mock___init__.return_value = None
mock_url_for.return_value = cfg.CONF.zaqar.auth_url
zaqar_collect = zaqar.Collector(
keystoneclient=test_heat.FakeFailKeystoneClient(
self, cfg.CONF.zaqar),
zaqarclient=FakeZaqarClient(self))
self.assertRaises(exc.ZaqarMetadataNotAvailable, zaqar_collect.collect)
self.assertIn('Forbidden', self.log.output)
def test_collect_zaqar_no_auth_url(self):
cfg.CONF.zaqar.auth_url = None
zaqar_collect = zaqar.Collector()
self.assertRaises(
exc.ZaqarMetadataNotConfigured, zaqar_collect.collect)
self.assertIn('No auth_url configured', self.log.output)
def test_collect_zaqar_no_password(self):
cfg.CONF.zaqar.password = None
zaqar_collect = zaqar.Collector()
self.assertRaises(
exc.ZaqarMetadataNotConfigured, zaqar_collect.collect)
self.assertIn('No password configured', self.log.output)
def test_collect_zaqar_no_project_id(self):
cfg.CONF.zaqar.project_id = None
zaqar_collect = zaqar.Collector()
self.assertRaises(
exc.ZaqarMetadataNotConfigured, zaqar_collect.collect)
self.assertIn('No project_id configured', self.log.output)
def test_collect_zaqar_no_user_id(self):
cfg.CONF.zaqar.user_id = None
zaqar_collect = zaqar.Collector()
self.assertRaises(
exc.ZaqarMetadataNotConfigured, zaqar_collect.collect)
self.assertIn('No user_id configured', self.log.output)
def test_collect_zaqar_no_queue_id(self):
cfg.CONF.zaqar.queue_id = None
zaqar_collect = zaqar.Collector()
self.assertRaises(
exc.ZaqarMetadataNotConfigured, zaqar_collect.collect)
self.assertIn('No queue_id configured', self.log.output)

94
os_collect_config/zaqar.py Normal file
View File

@ -0,0 +1,94 @@
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from keystoneclient.v3 import client as keystoneclient
from oslo_config import cfg
import six
from zaqarclient.queues.v1 import client as zaqarclient
from os_collect_config import exc
from os_collect_config import keystone
from os_collect_config.openstack.common import log
CONF = cfg.CONF
logger = log.getLogger(__name__)
opts = [
cfg.StrOpt('user-id',
help='User ID for API authentication'),
cfg.StrOpt('password',
help='Password for API authentication'),
cfg.StrOpt('project-id',
help='ID of project for API authentication'),
cfg.StrOpt('auth-url',
help='URL for API authentication'),
cfg.StrOpt('queue-id',
help='ID of the queue to be checked'),
]
name = 'zaqar'
class Collector(object):
def __init__(self,
keystoneclient=keystoneclient,
zaqarclient=zaqarclient):
self.keystoneclient = keystoneclient
self.zaqarclient = zaqarclient
def collect(self):
if CONF.zaqar.auth_url is None:
logger.warn('No auth_url configured.')
raise exc.ZaqarMetadataNotConfigured()
if CONF.zaqar.password is None:
logger.warn('No password configured.')
raise exc.ZaqarMetadataNotConfigured()
if CONF.zaqar.project_id is None:
logger.warn('No project_id configured.')
raise exc.ZaqarMetadataNotConfigured()
if CONF.zaqar.user_id is None:
logger.warn('No user_id configured.')
raise exc.ZaqarMetadataNotConfigured()
if CONF.zaqar.queue_id is None:
logger.warn('No queue_id configured.')
raise exc.ZaqarMetadataNotConfigured()
try:
ks = keystone.Keystone(
auth_url=CONF.zaqar.auth_url,
user_id=CONF.zaqar.user_id,
password=CONF.zaqar.password,
project_id=CONF.zaqar.project_id,
keystoneclient=self.keystoneclient).client
endpoint = ks.service_catalog.url_for(
service_type='messaging', endpoint_type='publicURL')
logger.debug('Fetching metadata from %s' % endpoint)
conf = {
'auth_opts': {
'backend': 'keystone',
'options': {
'os_auth_token': ks.auth_token,
'os_project_id': CONF.zaqar.project_id
}
}
}
zaqar = self.zaqarclient.Client(endpoint, conf=conf, version=1.1)
queue = zaqar.queue(CONF.zaqar.queue_id)
r = six.next(queue.pop())
return [('zaqar', r.body)]
except Exception as e:
logger.warn(str(e))
raise exc.ZaqarMetadataNotAvailable()

1
requirements.txt
View File

@ -8,6 +8,7 @@ argparse
eventlet>=0.17.3
python-keystoneclient>=1.6.0
python-heatclient>=0.3.0
python-zaqarclient>=0.1.1
requests>=2.5.2
iso8601>=0.1.9
lxml>=2.3