Support new ironic "enroll" state

This state was introduced in Liberty as a new state for freshly
enrolled node. Transition from it is done by the same verb "manage",
but now involves validation of power credentials.

This commit also

 - makes utils.wait_for_provision_state raise appropriate exceptions on
   error rather than returning False
 - adds a utils.nodes_in_states function to list baremetal nodes in a
   given set of states.
 - adds logic to baremetal/fakes.py to track states of fake nodes; this
   is much simpler and less brittle than precisely mocking the results of
   API calls in exactly the order the library code makes them.
 - consistently mocks bulk introspection tests at the client layer,
   rather than at a variety of layers.
 - adds tests for timeout and power-credential error during bulk
   introspection.
 - doesn't set nodes to "available" if they fail introspection or power
   credentials

Change-Id: I4da61491f60f7ebd42ca1f8fe45c3d4df6e49887
This commit is contained in:
Dmitry Tantsur 2015-10-15 10:25:13 +02:00 committed by Miles Gould
parent 7433110cd0
commit 5ece838dc0
6 changed files with 271 additions and 87 deletions

View File

@ -52,3 +52,7 @@ class InvalidConfiguration(ValueError):
class IntrospectionError(RuntimeError):
"""Introspection failed"""
class StateTransitionFailed(Exception):
"""Ironic node state transition failed"""

View File

@ -319,12 +319,9 @@ class TestWaitForIntrospection(TestCase):
baremetal_client = mock.Mock()
baremetal_client.node.get.return_value = mock.Mock(
provision_state="available")
provision_state="available", last_error=None)
result = utils.wait_for_provision_state(baremetal_client, 'UUID',
"available")
self.assertEqual(result, True)
utils.wait_for_provision_state(baremetal_client, 'UUID', "available")
def test_wait_for_provision_state_not_found(self):
@ -332,23 +329,30 @@ class TestWaitForIntrospection(TestCase):
baremetal_client.node.get.return_value = None
result = utils.wait_for_provision_state(baremetal_client, 'UUID',
"available")
utils.wait_for_provision_state(baremetal_client, 'UUID', "available")
self.assertEqual(result, True)
def test_wait_for_provision_state_timeout(self):
baremetal_client = mock.Mock()
baremetal_client.node.get.return_value = mock.Mock(
provision_state="not what we want", last_error=None)
with self.assertRaises(exceptions.Timeout):
utils.wait_for_provision_state(baremetal_client, 'UUID',
"available", loops=1, sleep=0.01)
def test_wait_for_provision_state_fail(self):
baremetal_client = mock.Mock()
baremetal_client.node.get.return_value = mock.Mock(
provision_state="not what we want")
provision_state="enroll",
last_error="node on fire; returning to previous state.")
result = utils.wait_for_provision_state(baremetal_client, 'UUID',
"available", loops=1,
sleep=0.01)
self.assertEqual(result, False)
with self.assertRaises(exceptions.StateTransitionFailed):
utils.wait_for_provision_state(baremetal_client, 'UUID',
"available", loops=1, sleep=0.01)
@mock.patch('subprocess.check_call')
@mock.patch('os.path.exists')

View File

@ -17,6 +17,56 @@ import mock
from openstackclient.tests import utils
class FakeBaremetalNodeClient(object):
def __init__(self, states={}, transitions={}, transition_errors={}):
"""Create a new test double for the "baremetal node" command.
:param states: dictionary of nodes' initial states. Keys are uuids and
values are states, eg {"ABC: "available"}.
:param transitions: dictionary of expected state transitions.
Keys are (uuid, transition) pairs, and values are
the states nodes end up in after that transition,
eg {("ABC", "manage"): "manageable"}.
Updates which occur are stored in "updates" for
later inspection.
:param transition_errors: dict of errors caused by state transitions.
Keys are (uuid, transition) pairs, and values
are the value of node.last_error after that
transition,
eg {("ABC", "manage"): "Node on fire."}.
"""
self.states = states
self.transitions = transitions
self.transition_errors = transition_errors
self.last_errors = {}
self.updates = [] # inspect this to see which transitions occurred
def set_provision_state(self, node_uuid, transition):
key = (node_uuid, transition)
new_state = self.transitions[key]
self.states[node_uuid] = new_state
self.last_errors[node_uuid] = self.transition_errors.get(key, None)
self.updates.append(key)
def _get(self, uuid, detail=False, **kwargs):
mock_node = mock.Mock(uuid=uuid, provision_state=self.states[uuid])
if detail:
mock_node.last_error = self.last_errors.get(uuid, None)
else:
mock_node.mock_add_spec(
('instance_uuid', 'maintenance', 'power_state',
'provision_state', 'uuid', 'name'),
spec_set=True)
return mock_node
def get(self, uuid):
return self._get(uuid, detail=True)
def list(self, *args, **kwargs):
return [self._get(uuid, **kwargs)
for uuid in (sorted(self.states.keys()))]
class FakeClientWrapper(object):
def __init__(self):

View File

@ -19,6 +19,8 @@ import json
import mock
import os
import fixtures
from tripleoclient import exceptions
from tripleoclient.tests.v1.baremetal import fakes
from tripleoclient.v1 import baremetal
@ -453,11 +455,14 @@ class TestStartBaremetalIntrospectionBulk(fakes.TestBaremetal):
@mock.patch.object(baremetal.inspector_client, 'get_status', autospec=True)
@mock.patch.object(baremetal.inspector_client, 'introspect', autospec=True)
def test_introspect_bulk_one(self, inspection_mock, get_status_mock):
client = self.app.client_manager.tripleoclient.baremetal
client.node.list.return_value = [
mock.Mock(uuid="ABCDEFGH", provision_state="manageable")
]
client.node = fakes.FakeBaremetalNodeClient(
states={"ABCDEFGH": "available"},
transitions={
("ABCDEFGH", "manage"): "manageable",
("ABCDEFGH", "provide"): "available",
}
)
get_status_mock.return_value = {'finished': True, 'error': None}
parsed_args = self.check_parser(self.cmd, [], [])
@ -465,74 +470,159 @@ class TestStartBaremetalIntrospectionBulk(fakes.TestBaremetal):
inspection_mock.assert_called_once_with(
'ABCDEFGH', base_url=None, auth_token='TOKEN')
self.assertEqual(client.node.updates, [
('ABCDEFGH', 'manage'),
('ABCDEFGH', 'provide')
])
@mock.patch.object(baremetal.inspector_client, 'get_status', autospec=True)
@mock.patch.object(baremetal.inspector_client, 'introspect', autospec=True)
def test_introspect_bulk_failed(self, inspection_mock, get_status_mock):
def test_introspect_bulk_failed(self, introspect_mock, get_status_mock):
client = self.app.client_manager.tripleoclient.baremetal
client.node.list.return_value = [
mock.Mock(uuid="ABCDEFGH", provision_state="manageable")
]
get_status_mock.return_value = {'finished': True,
'error': 'fake error'}
client.node = fakes.FakeBaremetalNodeClient(
states={"ABCDEFGH": "available", "IJKLMNOP": "available"},
transitions={
("ABCDEFGH", "manage"): "manageable",
("IJKLMNOP", "manage"): "manageable",
("ABCDEFGH", "provide"): "available",
}
)
status_returns = {
"ABCDEFGH": {'finished': True, 'error': None},
"IJKLMNOP": {'finished': True, 'error': 'fake error'}
}
get_status_mock.side_effect = \
lambda uuid, *args, **kwargs: status_returns[uuid]
parsed_args = self.check_parser(self.cmd, [], [])
self.assertRaisesRegexp(exceptions.IntrospectionError,
'ABCDEFGH: fake error',
'IJKLMNOP: fake error',
self.cmd.take_action, parsed_args)
inspection_mock.assert_called_once_with(
'ABCDEFGH', base_url=None, auth_token='TOKEN')
introspect_mock.assert_has_calls([
mock.call('ABCDEFGH', base_url=None, auth_token='TOKEN'),
mock.call('IJKLMNOP', base_url=None, auth_token='TOKEN')
])
self.assertEqual({'ABCDEFGH': 'available', 'IJKLMNOP': 'manageable'},
client.node.states)
@mock.patch('tripleoclient.utils.wait_for_node_introspection',
autospec=True)
@mock.patch('tripleoclient.utils.wait_for_provision_state',
autospec=True)
@mock.patch.object(baremetal.inspector_client, 'get_status', autospec=True)
@mock.patch.object(baremetal.inspector_client, 'introspect', autospec=True)
def test_introspect_bulk(self, introspect_mock, get_status_mock,
wait_for_state_mock, wait_for_inspect_mock):
wait_for_inspect_mock.return_value = []
wait_for_state_mock.return_value = True
def test_introspect_bulk(self, introspect_mock, get_status_mock):
client = self.app.client_manager.tripleoclient.baremetal
client.node = fakes.FakeBaremetalNodeClient(
states={
"ABC": "available",
"DEF": "enroll",
"GHI": "manageable",
"JKL": "clean_wait"
},
transitions={
("ABC", "manage"): "manageable",
("DEF", "manage"): "manageable",
("ABC", "provide"): "available",
("DEF", "provide"): "available",
("GHI", "provide"): "available"
}
)
get_status_mock.return_value = {'finished': True, 'error': None}
client = self.app.client_manager.tripleoclient.baremetal
client.node.list.return_value = [
mock.Mock(uuid="ABCDEFGH", provision_state="available"),
mock.Mock(uuid="IJKLMNOP", provision_state="manageable"),
mock.Mock(uuid="QRSTUVWX", provision_state="available"),
]
arglist = []
verifylist = []
parsed_args = self.check_parser(self.cmd, arglist, verifylist)
parsed_args = self.check_parser(self.cmd, [], [])
self.cmd.take_action(parsed_args)
# The nodes that are available are set to "manageable" state.
client.node.set_provision_state.assert_has_calls([
mock.call('ABCDEFGH', 'manage'),
mock.call('QRSTUVWX', 'manage'),
# Then all manageable nodes are set to "available".
self.assertEqual(client.node.updates, [
('ABC', 'manage'),
('DEF', 'manage'),
('ABC', 'provide'),
('DEF', 'provide'),
('GHI', 'provide')
])
# Since everything is mocked, the node states doesn't change.
# Therefore only the node originally in manageable state is
# Nodes which start in "enroll", "available" or "manageable" states are
# introspected:
introspect_mock.assert_has_calls([
mock.call('IJKLMNOP', base_url=None, auth_token='TOKEN'),
mock.call('ABC', base_url=None, auth_token='TOKEN'),
mock.call('DEF', base_url=None, auth_token='TOKEN'),
mock.call('GHI', base_url=None, auth_token='TOKEN')
])
wait_for_inspect_mock.assert_called_once_with(
baremetal.inspector_client, 'TOKEN', None,
['IJKLMNOP'])
get_status_mock.assert_has_calls([
mock.call('ABC', base_url=None, auth_token='TOKEN'),
mock.call('DEF', base_url=None, auth_token='TOKEN'),
mock.call('GHI', base_url=None, auth_token='TOKEN')
], any_order=True)
# And lastly it will be set to available:
client.node.set_provision_state.assert_has_calls([
mock.call('IJKLMNOP', 'provide'),
])
@mock.patch.object(baremetal.inspector_client, 'get_status', autospec=True)
@mock.patch.object(baremetal.inspector_client, 'introspect', autospec=True)
def test_introspect_bulk_timeout(self, introspect_mock, get_status_mock):
client = self.app.client_manager.tripleoclient.baremetal
client.node = fakes.FakeBaremetalNodeClient(
states={
"ABC": "available",
"DEF": "enroll",
},
transitions={
("ABC", "manage"): "available", # transition times out
("DEF", "manage"): "manageable",
("DEF", "provide"): "available"
}
)
get_status_mock.return_value = {'finished': True, 'error': None}
log_fixture = self.useFixture(fixtures.FakeLogger())
parsed_args = self.check_parser(self.cmd, [], [])
self.cmd.take_action(parsed_args)
self.assertIn("FAIL: Timeout waiting for Node ABC", log_fixture.output)
# Nodes that don't timeout are introspected
introspect_mock.assert_called_once_with(
'DEF', base_url=None, auth_token='TOKEN')
get_status_mock.assert_called_once_with(
'DEF', base_url=None, auth_token='TOKEN')
# Nodes that were successfully introspected are made available
self.assertEqual(
[("ABC", "manage"), ("DEF", "manage"), ("DEF", "provide")],
client.node.updates)
@mock.patch.object(baremetal.inspector_client, 'get_status', autospec=True)
@mock.patch.object(baremetal.inspector_client, 'introspect', autospec=True)
def test_introspect_bulk_transition_fails(self, introspect_mock,
get_status_mock):
client = self.app.client_manager.tripleoclient.baremetal
client.node = fakes.FakeBaremetalNodeClient(
states={
"ABC": "available",
"DEF": "enroll",
},
transitions={
("ABC", "manage"): "manageable",
("DEF", "manage"): "enroll", # state transition fails
("ABC", "provide"): "available"
},
transition_errors={
("DEF", "manage"): "power credential verification failed"
}
)
get_status_mock.return_value = {'finished': True, 'error': None}
log_fixture = self.useFixture(fixtures.FakeLogger())
parsed_args = self.check_parser(self.cmd, [], [])
self.cmd.take_action(parsed_args)
self.assertIn("FAIL: State transition failed for Node DEF",
log_fixture.output)
# Nodes that successfully transition are introspected
introspect_mock.assert_called_once_with(
'ABC', base_url=None, auth_token='TOKEN')
get_status_mock.assert_called_once_with(
'ABC', base_url=None, auth_token='TOKEN')
# Nodes that were successfully introspected are made available
self.assertEqual(
[("ABC", "manage"), ("DEF", "manage"), ("ABC", "provide")],
client.node.updates)
class TestStatusBaremetalIntrospectionBulk(fakes.TestBaremetal):

View File

@ -13,6 +13,7 @@
# under the License.
#
from __future__ import print_function
import base64
import hashlib
import json
@ -22,7 +23,6 @@ import passlib.utils as passutils
import six
import struct
import subprocess
import sys
import time
from heatclient.common import event_utils
@ -248,6 +248,12 @@ def event_log_formatter(events):
return "\n".join(event_log)
def nodes_in_states(baremetal_client, states):
"""List the introspectable nodes with the right provision_states."""
nodes = baremetal_client.node.list(maintenance=False, associated=False)
return [node for node in nodes if node.provision_state in states]
def wait_for_provision_state(baremetal_client, node_uuid, provision_state,
loops=10, sleep=1):
"""Wait for a given Provisioning state in Ironic
@ -269,6 +275,8 @@ def wait_for_provision_state(baremetal_client, node_uuid, provision_state,
:param sleep: How long to sleep between loops
:type sleep: int
:raises exceptions.StateTransitionFailed: if node.last_error is set
"""
for _ in range(0, loops):
@ -278,14 +286,32 @@ def wait_for_provision_state(baremetal_client, node_uuid, provision_state,
if node is None:
# The node can't be found in ironic, so we don't need to wait for
# the provision state
return True
return
if node.provision_state == provision_state:
return True
return
# node.last_error should be None after any successful operation
if node.last_error:
raise exceptions.StateTransitionFailed(
"Error transitioning node %(uuid)s to provision state "
"%(state)s: %(error)s. Now in state %(actual)s." % {
'uuid': node_uuid,
'state': provision_state,
'error': node.last_error,
'actual': node.provision_state
}
)
time.sleep(sleep)
return False
raise exceptions.Timeout(
"Node %(uuid)s did not reach provision state %(state)s. "
"Now in state %(actual)s." % {
'uuid': node_uuid,
'state': provision_state,
'actual': node.provision_state
}
)
def wait_for_node_introspection(inspector_client, auth_token, inspector_url,
@ -313,7 +339,6 @@ def wait_for_node_introspection(inspector_client, auth_token, inspector_url,
for _ in range(0, loops):
for node_uuid in node_uuids:
status = inspector_client.get_status(
node_uuid,
base_url=inspector_url,
@ -398,6 +423,17 @@ def set_nodes_state(baremetal_client, nodes, transition, target_state,
are already deployed and the state can't always be
changed.
:type skipped_states: iterable of strings
:param error_states: Node states treated as error for this transition
:type error_states: collection of strings
:param error_message: Optional message to append to an error message
:param error_message: str
:raises exceptions.StateTransitionFailed: if a node enters any of the
states in error_states
:raises exceptions.Timeout: if a node takes too long to reach target state
"""
log = logging.getLogger(__name__ + ".set_nodes_state")
@ -412,13 +448,15 @@ def set_nodes_state(baremetal_client, nodes, transition, target_state,
.format(node.provision_state, transition, node.uuid))
baremetal_client.node.set_provision_state(node.uuid, transition)
if not wait_for_provision_state(baremetal_client, node.uuid,
target_state):
print("FAIL: State not updated for Node {0}".format(
node.uuid, file=sys.stderr))
else:
yield node.uuid
try:
wait_for_provision_state(baremetal_client, node.uuid, target_state)
except exceptions.StateTransitionFailed as e:
log.error("FAIL: State transition failed for Node {0}. {1}"
.format(node.uuid, e))
except exceptions.Timeout as e:
log.error("FAIL: Timeout waiting for Node {0}. {1}"
.format(node.uuid, e))
yield node.uuid
def get_hiera_key(key_name):

View File

@ -210,19 +210,16 @@ class StartBaremetalIntrospectionBulk(IntrospectionParser, command.Command):
auth_token = self.app.client_manager.auth_ref.auth_token
node_uuids = []
print("Setting available nodes to manageable...")
self.log.debug("Moving available nodes to manageable state.")
available_nodes = [node for node in client.node.list(maintenance=False,
associated=False)
if node.provision_state == "available"]
print("Setting nodes for introspection to manageable...")
self.log.debug("Moving available/enroll nodes to manageable state.")
available_nodes = utils.nodes_in_states(client,
("available", "enroll"))
for uuid in utils.set_nodes_state(client, available_nodes, 'manage',
'manageable'):
self.log.debug("Node {0} has been set to manageable.".format(uuid))
for node in client.node.list(maintenance=False, associated=False):
if node.provision_state != "manageable":
continue
manageable_nodes = utils.nodes_in_states(client, ("manageable",))
for node in manageable_nodes:
node_uuids.append(node.uuid)
print("Starting introspection of node: {0}".format(node.uuid))
@ -239,12 +236,14 @@ class StartBaremetalIntrospectionBulk(IntrospectionParser, command.Command):
print("Waiting for introspection to finish...")
errors = []
successful_node_uuids = set()
for uuid, status in utils.wait_for_node_introspection(
inspector_client, auth_token, parsed_args.inspector_url,
node_uuids):
if status['error'] is None:
print("Introspection for UUID {0} finished successfully."
.format(uuid))
successful_node_uuids.add(uuid)
else:
print("Introspection for UUID {0} finished with error: {1}"
.format(uuid, status['error']))
@ -253,11 +252,10 @@ class StartBaremetalIntrospectionBulk(IntrospectionParser, command.Command):
print("Setting manageable nodes to available...")
self.log.debug("Moving manageable nodes to available state.")
available_nodes = [node for node in client.node.list(maintenance=False,
associated=False)
if node.provision_state == "manageable"]
successful_nodes = [n for n in manageable_nodes
if n.uuid in successful_node_uuids]
for uuid in utils.set_nodes_state(
client, available_nodes, 'provide',
client, successful_nodes, 'provide',
'available', skipped_states=("available", "active")):
print("Node {0} has been set to available.".format(uuid))