openstack/os-collect-config
Code Issues Proposed changes

Collector which does a simple HTTP GET

Browse Source 
This change implements a collector which does an HTTP GET via
python requests to fetch the metadata.

It should work with any GET-able URL, however it is designed to
work with Swift TempURLs.

Swift objects are not consistent, so the Last-Modified header is
checked for each poll and metadata is not fetched if the last
modified is not newer than the previous successful poll.

This collector will be enabled for OS::Nova::Server
software_config_transport: POLL_TEMP_URL which is available
in the Juno release of Heat. Using POLL_TEMP_URL will result
in no metadata polling load on heat, which has historically been
an issue with tripleo scalability.

Change-Id: I22155c22bdcc3c81a5e945ca5436a8f29f196528
This commit is contained in:
Steve Baker 2014-10-13 13:18:48 +13:00
parent fa3d4e34e8
commit ad475ee927
5 changed files with 271 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
os_collect_config/collect.py
View File

@ -31,10 +31,11 @@ from os_collect_config import heat_local
from os_collect_config import keystone
from os_collect_config import local
from os_collect_config.openstack.common import log
from os_collect_config import request
from os_collect_config import version
from oslo.config import cfg
DEFAULT_COLLECTORS = ['heat_local', 'ec2', 'cfn', 'heat']
DEFAULT_COLLECTORS = ['heat_local', 'ec2', 'cfn', 'heat', 'request']
opts = [
cfg.StrOpt('command', short='c',
help='Command to run on metadata changes. If specified,'
@ -83,7 +84,8 @@ COLLECTORS = {ec2.name: ec2,
cfn.name: cfn,
heat.name: heat,
heat_local.name: heat_local,
local.name: local}
local.name: local,
request.name: request}
def setup_conf():
@ -102,6 +104,9 @@ def setup_conf():
heat_group = cfg.OptGroup(name='heat',
title='Heat Metadata options')
request_group = cfg.OptGroup(name='request',
title='Request Metadata options')
keystone_group = cfg.OptGroup(name='keystone',
title='Keystone auth options')
@ -110,12 +115,14 @@ def setup_conf():
CONF.register_group(heat_local_group)
CONF.register_group(local_group)
CONF.register_group(heat_group)
CONF.register_group(request_group)
CONF.register_group(keystone_group)
CONF.register_cli_opts(ec2.opts, group='ec2')
CONF.register_cli_opts(cfn.opts, group='cfn')
CONF.register_cli_opts(heat_local.opts, group='heat_local')
CONF.register_cli_opts(local.opts, group='local')
CONF.register_cli_opts(heat.opts, group='heat')
CONF.register_cli_opts(request.opts, group='request')
CONF.register_cli_opts(keystone.opts, group='keystone')
CONF.register_cli_opts(opts)

8
os_collect_config/exc.py
View File

@ -50,5 +50,13 @@ class LocalMetadataNotAvailable(SourceNotAvailable):
"""The local metadata is not available."""
class RequestMetadataNotAvailable(SourceNotAvailable):
"""The request metadata is not available."""
class RequestMetadataNotConfigured(SourceNotAvailable):
"""The request metadata is not fully configured."""
class InvalidArguments(ValueError):
"""Invalid arguments."""

93
os_collect_config/request.py Normal file
View File

@ -0,0 +1,93 @@
# Copyright (c) 2013 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import calendar
import json
import time
from oslo.config import cfg
from os_collect_config import common
from os_collect_config import exc
from os_collect_config.openstack.common import log
CONF = cfg.CONF
logger = log.getLogger(__name__)
opts = [
cfg.StrOpt('metadata-url',
help='URL to query for metadata'),
]
name = 'request'
class Collector(object):
def __init__(self, requests_impl=common.requests):
self._requests_impl = requests_impl
self._session = requests_impl.Session()
self.last_modified = None
def check_fetch_content(self, headers):
'''Raises RequestMetadataNotAvailable if metadata should not be
fetched.
'''
# no last-modified header, so fetch
lm = headers.get('last-modified')
if not lm:
return
last_modified = calendar.timegm(
time.strptime(lm, '%a, %d %b %Y %H:%M:%S %Z'))
# first run, so fetch
if not self.last_modified:
return last_modified
if last_modified < self.last_modified:
logger.warn(
'Last-Modified is older than previous collection')
if last_modified <= self.last_modified:
raise exc.RequestMetadataNotAvailable
return last_modified
def collect(self):
if CONF.request.metadata_url is None:
logger.warn('No metadata_url configured.')
raise exc.RequestMetadataNotConfigured
url = CONF.request.metadata_url
final_content = {}
try:
head = self._session.head(url)
last_modified = self.check_fetch_content(head.headers)
content = self._session.get(url)
content.raise_for_status()
self.last_modified = last_modified
except self._requests_impl.exceptions.RequestException as e:
logger.warn(e)
raise exc.RequestMetadataNotAvailable
try:
value = json.loads(content.text)
except ValueError as e:
logger.warn(
'Failed to parse as json. (%s)' % e)
raise exc.RequestMetadataNotAvailable
final_content.update(value)
return [('request', final_content)]

19
os_collect_config/tests/test_collect.py
View File

@ -35,6 +35,7 @@ from os_collect_config.tests import test_cfn
from os_collect_config.tests import test_ec2
from os_collect_config.tests import test_heat
from os_collect_config.tests import test_heat_local
from os_collect_config.tests import test_request
def _setup_local_metadata(test_case):
@ -63,7 +64,8 @@ class TestCollect(testtools.TestCase):
'heat': {
'keystoneclient': test_heat.FakeKeystoneClient(self),
'heatclient': test_heat.FakeHeatClient(self)
}
},
'request': {'requests_impl': test_request.FakeRequests},
}
return collect.__main__(args=fake_args,
collector_kwargs_map=collector_kwargs_map)
@ -338,6 +340,7 @@ class TestCollectAll(testtools.TestCase):
cfg.CONF.heat.project_id = '9f6b09df-4d7f-4a33-8ec3-9924d8f46f10'
cfg.CONF.heat.stack_id = 'a/c482680f-7238-403d-8f76-36acf0c8e0aa'
cfg.CONF.heat.resource_name = 'server'
cfg.CONF.request.metadata_url = 'http://127.0.0.1:8000/my_metadata/'
@mock.patch.object(ks_discover.Discover, '__init__')
@mock.patch.object(ks_discover.Discover, 'url_for')
@ -352,7 +355,8 @@ class TestCollectAll(testtools.TestCase):
'heat': {
'keystoneclient': test_heat.FakeKeystoneClient(self),
'heatclient': test_heat.FakeHeatClient(self)
}
},
'request': {'requests_impl': test_request.FakeRequests},
}
if collectors is None:
collectors = cfg.CONF.collectors
@ -366,7 +370,8 @@ class TestCollectAll(testtools.TestCase):
(changed_keys, paths) = self._call_collect_all(
store=True, collector_kwargs_map=collector_kwargs_map)
if expected_changed is None:
expected_changed = set(['heat_local', 'cfn', 'ec2', 'heat'])
expected_changed = set(
['heat_local', 'cfn', 'ec2', 'heat', 'request'])
self.assertEqual(expected_changed, changed_keys)
self.assertThat(paths, matchers.IsInstance(list))
for path in paths:
@ -384,10 +389,11 @@ class TestCollectAll(testtools.TestCase):
'heat': {
'keystoneclient': test_heat.FakeKeystoneClient(self),
'heatclient': test_heat.FakeHeatClient(self)
}
},
'request': {'requests_impl': test_request.FakeRequests},
}
expected_changed = set((
'heat_local', 'ec2', 'cfn', 'heat',
'heat_local', 'ec2', 'cfn', 'heat', 'request',
'dep-name1', 'dep-name2', 'dep-name3'))
self._test_collect_all_store(collector_kwargs_map=soft_config_map,
expected_changed=expected_changed)
@ -422,7 +428,8 @@ class TestCollectAll(testtools.TestCase):
'heat': {
'keystoneclient': test_heat.FakeKeystoneClient(self),
'heatclient': test_heat.FakeHeatClient(self)
}
},
'request': {'requests_impl': test_request.FakeRequests},
}
(changed_keys, paths) = self._call_collect_all(
store=True, collector_kwargs_map=soft_config_map)

148
os_collect_config/tests/test_request.py Normal file
View File

@ -0,0 +1,148 @@
# Copyright (c) 2013 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import calendar
import json
import time
import fixtures
from oslo.config import cfg
import requests
import testtools
from testtools import matchers
from os_collect_config import collect
from os_collect_config import exc
from os_collect_config import request
META_DATA = {u'int1': 1,
u'strfoo': u'foo',
u'map_ab': {
u'a': 'apple',
u'b': 'banana',
}}
class FakeResponse(dict):
def __init__(self, text, headers=None):
self.text = text
self.headers = headers
def raise_for_status(self):
pass
class FakeRequests(object):
exceptions = requests.exceptions
class Session(object):
def get(self, url):
return FakeResponse(json.dumps(META_DATA))
def head(self, url):
return FakeResponse('', headers={
'last-modified': time.strftime(
"%a, %d %b %Y %H:%M:%S %Z", time.gmtime())})
class FakeFailRequests(object):
exceptions = requests.exceptions
class Session(object):
def get(self, url):
raise requests.exceptions.HTTPError(403, 'Forbidden')
def head(self, url):
raise requests.exceptions.HTTPError(403, 'Forbidden')
class TestRequestBase(testtools.TestCase):
def setUp(self):
super(TestRequestBase, self).setUp()
self.log = self.useFixture(fixtures.FakeLogger())
collect.setup_conf()
cfg.CONF.request.metadata_url = 'http://127.0.0.1:8000/my_metadata'
class TestRequest(TestRequestBase):
def test_collect_request(self):
req_collect = request.Collector(requests_impl=FakeRequests)
self.assertIsNone(req_collect.last_modified)
req_md = req_collect.collect()
self.assertIsNotNone(req_collect.last_modified)
self.assertThat(req_md, matchers.IsInstance(list))
self.assertEqual('request', req_md[0][0])
req_md = req_md[0][1]
for k in ('int1', 'strfoo', 'map_ab'):
self.assertIn(k, req_md)
self.assertEqual(req_md[k], META_DATA[k])
self.assertEqual('', self.log.output)
def test_collect_request_fail(self):
req_collect = request.Collector(requests_impl=FakeFailRequests)
self.assertRaises(exc.RequestMetadataNotAvailable, req_collect.collect)
self.assertIn('Forbidden', self.log.output)
def test_collect_request_no_metadata_url(self):
cfg.CONF.request.metadata_url = None
req_collect = request.Collector(requests_impl=FakeRequests)
self.assertRaises(exc.RequestMetadataNotConfigured,
req_collect.collect)
self.assertIn('No metadata_url configured', self.log.output)
def test_check_fetch_content(self):
req_collect = request.Collector()
now_secs = calendar.timegm(time.gmtime())
now_str = time.strftime("%a, %d %b %Y %H:%M:%S %Z",
time.gmtime(now_secs))
future_secs = calendar.timegm(time.gmtime()) + 10
future_str = time.strftime("%a, %d %b %Y %H:%M:%S %Z",
time.gmtime(future_secs))
past_secs = calendar.timegm(time.gmtime()) - 10
past_str = time.strftime("%a, %d %b %Y %H:%M:%S %Z",
time.gmtime(past_secs))
self.assertIsNone(req_collect.last_modified)
# first run always collects
self.assertEqual(
now_secs,
req_collect.check_fetch_content({'last-modified': now_str}))
# second run unmodified, does not collect
req_collect.last_modified = now_secs
self.assertRaises(exc.RequestMetadataNotAvailable,
req_collect.check_fetch_content,
{'last-modified': now_str})
# run with later date, collects
self.assertEqual(
future_secs,
req_collect.check_fetch_content({'last-modified': future_str}))
# run with earlier date, does not collect
self.assertRaises(exc.RequestMetadataNotAvailable,
req_collect.check_fetch_content,
{'last-modified': past_str})
# run no last-modified header, collects
self.assertIsNone(req_collect.check_fetch_content({}))