openstack/ironic-inspector
Code Issues Proposed changes

Fix edeploy plugin puts too much data in Ironic extra column

Browse Source 
The edeploy plugin stores all of the facts it collects in the extra
column of the Ironic db. When using real hardware, edeploy collects a
large amount of facts and can lead to overflowing that column.

This patch fixes this by storing the collected data in Swift instead.
This makes it usable more generically as well. Anything stored on the
'data' key in the dictionary returned by the ramdisk is stored as a
JSON encoded string in a Swift object. The object is named
'extra_hardware-<node uuid>' and is stored in the 'ironic-inspector'
container.

Change-Id: Ie9e017df735a95350991ce419fa3b64249819d70
Closes-Bug: 1461252
This commit is contained in:
John Trowbridge 2015-06-08 16:05:06 -04:00
parent 3039aec2d5
commit 912c30830a
8 changed files with 307 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
example.conf
View File

@ -359,3 +359,33 @@
# (boolean value)
# Deprecated group/name - [discoverd]/always_store_ramdisk_logs
#always_store_ramdisk_logs = false
[swift]
#
# From ironic_inspector.common.swift
#
# Maximum number of times to retry a Swift request, before failing.
# (integer value)
#max_retries = 2
# Number of seconds that the Swift object will last before being
# deleted. (set to 0 to never delete the object). (integer value)
#delete_after = 0
# User name for accessing Swift API. (string value)
#username =
# Password for accessing Swift API. (string value)
#password =
# Tenant name for accessing Swift API. (string value)
#tenant_name =
# Keystone authentication API version (string value)
#os_auth_version = 2
# Keystone authentication URL (string value)
#os_auth_url =

130
ironic_inspector/common/swift.py Normal file
View File

@ -0,0 +1,130 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Mostly copied from ironic/common/swift.py
import logging
from oslo_config import cfg
from swiftclient import client as swift_client
from swiftclient import exceptions as swift_exceptions
from ironic_inspector.common.i18n import _
from ironic_inspector import utils
CONF = cfg.CONF
LOG = logging.getLogger('ironic_inspector.common.swift')
SWIFT_OPTS = [
cfg.IntOpt('max_retries',
default=2,
help='Maximum number of times to retry a Swift request, '
'before failing.'),
cfg.IntOpt('delete_after',
default=0,
help='Number of seconds that the Swift object will last before '
'being deleted. (set to 0 to never delete the object).'),
cfg.StrOpt('container',
default='ironic-inspector',
help='Default Swift container to use when creating objects.'),
cfg.StrOpt('username',
default='',
help='User name for accessing Swift API.'),
cfg.StrOpt('password',
default='',
help='Password for accessing Swift API.',
secret=True),
cfg.StrOpt('tenant_name',
default='',
help='Tenant name for accessing Swift API.'),
cfg.StrOpt('os_auth_version',
default='2',
help='Keystone authentication API version'),
cfg.StrOpt('os_auth_url',
default='',
help='Keystone authentication URL'),
]
def list_opts():
return [
('swift', SWIFT_OPTS)
]
CONF.register_opts(SWIFT_OPTS, group='swift')
class SwiftAPI(object):
"""API for communicating with Swift."""
def __init__(self,
user=CONF.swift.username,
tenant_name=CONF.swift.tenant_name,
key=CONF.swift.password,
auth_url=CONF.swift.os_auth_url,
auth_version=CONF.swift.os_auth_version):
"""Constructor for creating a SwiftAPI object.
:param user: the name of the user for Swift account
:param tenant_name: the name of the tenant for Swift account
:param key: the 'password' or key to authenticate with
:param auth_url: the url for authentication
:param auth_version: the version of api to use for authentication
"""
params = {'retries': CONF.swift.max_retries,
'user': user,
'tenant_name': tenant_name,
'key': key,
'authurl': auth_url,
'auth_version': auth_version}
self.connection = swift_client.Connection(**params)
def create_object(self, object, data, container=CONF.swift.container,
headers=None):
"""Uploads a given string to Swift.
:param object: The name of the object in Swift
:param data: string data to put in the object
:param container: The name of the container for the object.
:param headers: the headers for the object to pass to Swift
:returns: The Swift UUID of the object
:raises: utils.Error, if any operation with Swift fails.
"""
try:
self.connection.put_container(container)
except swift_exceptions.ClientException as e:
err_msg = (_('Swift failed to create container %(container)s. '
'Error was: %(error)s') %
{'container': container, 'error': e})
raise utils.Error(err_msg)
if CONF.swift.delete_after > 0:
headers = headers or {}
headers['X-Delete-After'] = CONF.swift.delete_after
try:
obj_uuid = self.connection.put_object(container,
object,
data,
headers=headers)
except swift_exceptions.ClientException as e:
err_msg = (_('Swift failed to create object %(object)s in '
'container %(container)s. Error was: %(error)s') %
{'object': object, 'container': container, 'error': e})
raise utils.Error(err_msg)
return obj_uuid

37
ironic_inspector/plugins/edeploy.py
View File

@ -17,27 +17,48 @@ See https://blueprints.launchpad.net/ironic-inspector/+spec/edeploy for
details on how to use it. Note that this plugin requires a special ramdisk.
"""
import json
import logging
from oslo_config import cfg
from ironic_inspector.common.i18n import _LW
from ironic_inspector.common import swift
from ironic_inspector.plugins import base
CONF = cfg.CONF
LOG = logging.getLogger('ironic_inspector.plugins.edeploy')
class eDeployHook(base.ProcessingHook):
"""Processing hook for saving additional data from eDeploy ramdisk."""
def _store_extra_hardware(self, name, data):
"""Handles storing the extra hardware data from the ramdisk"""
swift_api = swift.SwiftAPI()
swift_api.create_object(name, data)
def before_update(self, introspection_data, node_info, node_patches,
ports_patches, **kwargs):
"""Store the hardware data from what has been discovered."""
"""Stores the 'data' key from introspection_data in Swift.
If the 'data' key exists, updates Ironic extra column
'hardware_swift_object' key to the name of the Swift object, and stores
the data in the 'inspector' container in Swift.
Otherwise, it does nothing.
"""
if 'data' not in introspection_data:
LOG.warning(_LW('No eDeploy data was received from the ramdisk'))
LOG.warning(_LW('No extra hardware information was received from '
'the ramdisk'))
return
# (trown) it is useful for the edeploy report tooling to have the node
# uuid stored with the other edeploy_facts
introspection_data['data'].append(['system', 'product',
'ironic_uuid', node_info.uuid])
name = 'extra_hardware-%s' % node_info.uuid
self._store_extra_hardware(name,
json.dumps(introspection_data['data']))
node_patches.append({'op': 'add',
'path': '/extra/edeploy_facts',
'value': introspection_data['data']})
'path': '/extra/hardware_swift_object',
'value': name})

25
ironic_inspector/test/test_plugins_edeploy.py
View File

@ -11,10 +11,17 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import json
try:
from unittest import mock
except ImportError:
import mock
from ironic_inspector.plugins import edeploy
from ironic_inspector.test import base as test_base
@mock.patch.object(edeploy.swift, 'SwiftAPI', autospec=True)
class TestEdeploy(test_base.NodeTest):
def setUp(self):
@ -29,7 +36,7 @@ class TestEdeploy(test_base.NodeTest):
self.assertFalse(ports_patches)
return node_patches
def test_data_recieved(self):
def test_data_recieved(self, swift_mock):
introspection_data = {
'data': [['memory', 'total', 'size', '4294967296'],
['cpu', 'physical', 'number', '1'],
@ -37,19 +44,21 @@ class TestEdeploy(test_base.NodeTest):
self.hook.before_processing(introspection_data)
node_patches = self._before_update(introspection_data)
expected_value = [['memory', 'total', 'size', '4294967296'],
['cpu', 'physical', 'number', '1'],
['cpu', 'logical', 'number', '1'],
['system', 'product', 'ironic_uuid', self.node.uuid]]
swift_conn = swift_mock.return_value
name = 'extra_hardware-%s' % self.uuid
data = json.dumps(introspection_data['data'])
swift_conn.create_object.assert_called_once_with(name, data)
self.assertEqual('add',
node_patches[0]['op'])
self.assertEqual('/extra/edeploy_facts',
self.assertEqual('/extra/hardware_swift_object',
node_patches[0]['path'])
self.assertEqual(expected_value,
self.assertEqual(name,
node_patches[0]['value'])
def test_no_data_recieved(self):
def test_no_data_recieved(self, swift_mock):
introspection_data = {'cats': 'meow'}
swift_conn = swift_mock.return_value
self.hook.before_processing(introspection_data)
node_patches = self._before_update(introspection_data)
self.assertFalse(node_patches)
self.assertFalse(swift_conn.create_object.called)

97
ironic_inspector/test/test_swift.py Normal file
View File

@ -0,0 +1,97 @@
# Copyright 2013 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# Mostly copied from ironic/tests/test_swift.py
import sys
try:
from unittest import mock
except ImportError:
import mock
from oslo_config import cfg
from six.moves import reload_module
from swiftclient import client as swift_client
from swiftclient import exceptions as swift_exception
from ironic_inspector.common import swift
from ironic_inspector.test import base
from ironic_inspector import utils
CONF = cfg.CONF
@mock.patch.object(swift_client, 'Connection', autospec=True)
class SwiftTestCase(base.BaseTest):
def setUp(self):
super(SwiftTestCase, self).setUp()
self.swift_exception = swift_exception.ClientException('', '')
CONF.set_override('username', 'swift', 'swift')
CONF.set_override('tenant_name', 'tenant', 'swift')
CONF.set_override('password', 'password', 'swift')
CONF.set_override('os_auth_url', 'http://authurl/v2.0', 'swift')
CONF.set_override('os_auth_version', '2', 'swift')
CONF.set_override('max_retries', 2, 'swift')
# The constructor of SwiftAPI accepts arguments whose
# default values are values of some config options above. So reload
# the module to make sure the required values are set.
reload_module(sys.modules['ironic_inspector.common.swift'])
def test___init__(self, connection_mock):
swift.SwiftAPI()
params = {'retries': 2,
'user': 'swift',
'tenant_name': 'tenant',
'key': 'password',
'authurl': 'http://authurl/v2.0',
'auth_version': '2'}
connection_mock.assert_called_once_with(**params)
def test_create_object(self, connection_mock):
swiftapi = swift.SwiftAPI()
connection_obj_mock = connection_mock.return_value
connection_obj_mock.put_object.return_value = 'object-uuid'
object_uuid = swiftapi.create_object('object', 'some-string-data')
connection_obj_mock.put_container.assert_called_once_with('ironic-'
'inspector')
connection_obj_mock.put_object.assert_called_once_with(
'ironic-inspector', 'object', 'some-string-data', headers=None)
self.assertEqual('object-uuid', object_uuid)
def test_create_object_create_container_fails(self, connection_mock):
swiftapi = swift.SwiftAPI()
connection_obj_mock = connection_mock.return_value
connection_obj_mock.put_container.side_effect = self.swift_exception
self.assertRaises(utils.Error, swiftapi.create_object, 'object',
'some-string-data')
connection_obj_mock.put_container.assert_called_once_with('ironic-'
'inspector')
self.assertFalse(connection_obj_mock.put_object.called)
def test_create_object_put_object_fails(self, connection_mock):
swiftapi = swift.SwiftAPI()
connection_obj_mock = connection_mock.return_value
connection_obj_mock.put_object.side_effect = self.swift_exception
self.assertRaises(utils.Error, swiftapi.create_object, 'object',
'some-string-data')
connection_obj_mock.put_container.assert_called_once_with('ironic-'
'inspector')
connection_obj_mock.put_object.assert_called_once_with(
'ironic-inspector', 'object', 'some-string-data', headers=None)

2
plugin-requirements.txt
View File

@ -0,0 +1,2 @@
# required for edeploy plugin
python-swiftclient>=2.2.0

1
setup.py
View File

@ -51,6 +51,7 @@ setup(
],
'oslo.config.opts': [
"ironic_inspector = ironic_inspector.conf:list_opts",
"ironic_inspector.common.swift = ironic_inspector.common.swift:list_opts"
],
},
classifiers = [

1
tox.ini
View File

@ -44,3 +44,4 @@ commands =
--output-file example.conf \
--namespace ironic_inspector \
--namespace keystonemiddleware.auth_token
--namespace ironic_inspector.common.swift