518 lines
21 KiB
Python
518 lines
21 KiB
Python
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
from unittest import mock
|
|
|
|
from ironic_lib import utils
|
|
from oslo_concurrency import processutils
|
|
from tooz import coordination
|
|
|
|
from ironic_python_agent import burnin
|
|
from ironic_python_agent import errors
|
|
from ironic_python_agent import hardware
|
|
from ironic_python_agent.tests.unit import base
|
|
|
|
|
|
SMART_OUTPUT_JSON_COMPLETED = ("""
|
|
{
|
|
"ata_smart_data": {
|
|
"self_test": {
|
|
"status": {
|
|
"value": 0,
|
|
"string": "completed without error",
|
|
"passed": true
|
|
},
|
|
"polling_minutes": {
|
|
"short": 1,
|
|
"extended": 2,
|
|
"conveyance": 2
|
|
}
|
|
}
|
|
}
|
|
}
|
|
""")
|
|
|
|
SMART_OUTPUT_JSON_MISSING = ("""
|
|
{
|
|
"ata_smart_data": {
|
|
"self_test": {
|
|
"status": {
|
|
"value": 0,
|
|
"passed": true
|
|
}
|
|
}
|
|
}
|
|
}
|
|
""")
|
|
|
|
|
|
@mock.patch.object(utils, 'execute', autospec=True)
|
|
class TestBurnin(base.IronicAgentTest):
|
|
|
|
def test_stress_ng_cpu_default(self, mock_execute):
|
|
|
|
node = {'driver_info': {}}
|
|
mock_execute.return_value = (['out', 'err'])
|
|
|
|
burnin.stress_ng_cpu(node)
|
|
|
|
mock_execute.assert_called_once_with(
|
|
'stress-ng', '--cpu', 0, '--timeout', 86400, '--metrics-brief')
|
|
|
|
def test_stress_ng_cpu_non_default(self, mock_execute):
|
|
|
|
node = {'driver_info': {
|
|
'agent_burnin_cpu_cpu': 3,
|
|
'agent_burnin_cpu_timeout': 2911,
|
|
'agent_burnin_cpu_outputfile': '/var/log/burnin.cpu'}}
|
|
mock_execute.return_value = (['out', 'err'])
|
|
|
|
burnin.stress_ng_cpu(node)
|
|
|
|
mock_execute.assert_called_once_with(
|
|
'stress-ng', '--cpu', 3, '--timeout', 2911, '--metrics-brief',
|
|
'--log-file', '/var/log/burnin.cpu')
|
|
|
|
def test_stress_ng_cpu_no_stress_ng(self, mock_execute):
|
|
|
|
node = {'driver_info': {}}
|
|
mock_execute.side_effect = (['out', 'err'],
|
|
processutils.ProcessExecutionError())
|
|
|
|
burnin.stress_ng_cpu(node)
|
|
|
|
self.assertRaises(errors.CommandExecutionError,
|
|
burnin.stress_ng_cpu, node)
|
|
|
|
def test_stress_ng_vm_default(self, mock_execute):
|
|
|
|
node = {'driver_info': {}}
|
|
mock_execute.return_value = (['out', 'err'])
|
|
|
|
burnin.stress_ng_vm(node)
|
|
|
|
mock_execute.assert_called_once_with(
|
|
|
|
'stress-ng', '--vm', 0, '--vm-bytes', '98%',
|
|
'--timeout', 86400, '--metrics-brief')
|
|
|
|
def test_stress_ng_vm_non_default(self, mock_execute):
|
|
|
|
node = {'driver_info': {
|
|
'agent_burnin_vm_vm': 2,
|
|
'agent_burnin_vm_vm-bytes': '25%',
|
|
'agent_burnin_vm_timeout': 120,
|
|
'agent_burnin_vm_outputfile': '/var/log/burnin.vm'}}
|
|
mock_execute.return_value = (['out', 'err'])
|
|
|
|
burnin.stress_ng_vm(node)
|
|
|
|
mock_execute.assert_called_once_with(
|
|
'stress-ng', '--vm', 2, '--vm-bytes', '25%',
|
|
'--timeout', 120, '--metrics-brief',
|
|
'--log-file', '/var/log/burnin.vm')
|
|
|
|
def test_stress_ng_vm_no_stress_ng(self, mock_execute):
|
|
|
|
node = {'driver_info': {}}
|
|
mock_execute.side_effect = (['out', 'err'],
|
|
processutils.ProcessExecutionError())
|
|
|
|
burnin.stress_ng_vm(node)
|
|
|
|
self.assertRaises(errors.CommandExecutionError,
|
|
burnin.stress_ng_vm, node)
|
|
|
|
@mock.patch.object(hardware, 'list_all_block_devices', autospec=True)
|
|
def test_fio_disk_default(self, mock_list, mock_execute):
|
|
|
|
node = {'driver_info': {}}
|
|
|
|
mock_list.return_value = [
|
|
hardware.BlockDevice('/dev/sdj', 'big', 1073741824, True),
|
|
hardware.BlockDevice('/dev/hdaa', 'small', 65535, False),
|
|
]
|
|
mock_execute.return_value = (['out', 'err'])
|
|
|
|
burnin.fio_disk(node)
|
|
|
|
mock_execute.assert_called_once_with(
|
|
'fio', '--rw', 'readwrite', '--bs', '4k', '--direct', 1,
|
|
'--ioengine', 'libaio', '--iodepth', '32', '--verify',
|
|
'crc32c', '--verify_dump', 1, '--continue_on_error', 'verify',
|
|
'--loops', 4, '--runtime', 0, '--time_based', '--name',
|
|
'/dev/sdj', '--name', '/dev/hdaa')
|
|
|
|
@mock.patch.object(hardware, 'list_all_block_devices', autospec=True)
|
|
def test_fio_disk_no_default(self, mock_list, mock_execute):
|
|
|
|
node = {'driver_info': {
|
|
'agent_burnin_fio_disk_runtime': 600,
|
|
'agent_burnin_fio_disk_loops': 5,
|
|
'agent_burnin_fio_disk_outputfile': '/var/log/burnin.disk'}}
|
|
|
|
mock_list.return_value = [
|
|
hardware.BlockDevice('/dev/sdj', 'big', 1073741824, True),
|
|
hardware.BlockDevice('/dev/hdaa', 'small', 65535, False),
|
|
]
|
|
mock_execute.return_value = (['out', 'err'])
|
|
|
|
burnin.fio_disk(node)
|
|
|
|
mock_execute.assert_called_once_with(
|
|
'fio', '--rw', 'readwrite', '--bs', '4k', '--direct', 1,
|
|
'--ioengine', 'libaio', '--iodepth', '32', '--verify',
|
|
'crc32c', '--verify_dump', 1, '--continue_on_error', 'verify',
|
|
'--loops', 5, '--runtime', 600, '--time_based', '--output-format',
|
|
'json', '--output', '/var/log/burnin.disk', '--name', '/dev/sdj',
|
|
'--name', '/dev/hdaa', )
|
|
|
|
def test__smart_test_status(self, mock_execute):
|
|
device = hardware.BlockDevice('/dev/sdj', 'big', 1073741824, True)
|
|
mock_execute.return_value = ([SMART_OUTPUT_JSON_COMPLETED, 'err'])
|
|
|
|
status = burnin._smart_test_status(device)
|
|
|
|
mock_execute.assert_called_once_with('smartctl', '-ja', '/dev/sdj')
|
|
self.assertEqual(status, "completed without error")
|
|
|
|
def test__smart_test_status_missing(self, mock_execute):
|
|
device = hardware.BlockDevice('/dev/sdj', 'big', 1073741824, True)
|
|
mock_execute.return_value = ([SMART_OUTPUT_JSON_MISSING, 'err'])
|
|
|
|
status = burnin._smart_test_status(device)
|
|
|
|
mock_execute.assert_called_once_with('smartctl', '-ja', '/dev/sdj')
|
|
self.assertIsNone(status)
|
|
|
|
@mock.patch.object(burnin, '_smart_test_status', autospec=True)
|
|
@mock.patch.object(hardware, 'list_all_block_devices', autospec=True)
|
|
def test_fio_disk_smart_test(self, mock_list, mock_status, mock_execute):
|
|
|
|
node = {'driver_info': {'agent_burnin_fio_disk_smart_test': True}}
|
|
|
|
mock_list.return_value = [
|
|
hardware.BlockDevice('/dev/sdj', 'big', 1073741824, True),
|
|
hardware.BlockDevice('/dev/hdaa', 'small', 65535, False),
|
|
]
|
|
mock_status.return_value = "completed without error"
|
|
mock_execute.return_value = (['out', 'err'])
|
|
|
|
burnin.fio_disk(node)
|
|
|
|
expected_calls = [
|
|
mock.call('fio', '--rw', 'readwrite', '--bs', '4k', '--direct', 1,
|
|
'--ioengine', 'libaio', '--iodepth', '32', '--verify',
|
|
'crc32c', '--verify_dump', 1, '--continue_on_error',
|
|
'verify', '--loops', 4, '--runtime', 0, '--time_based',
|
|
'--name', '/dev/sdj', '--name', '/dev/hdaa'),
|
|
mock.call('smartctl', '-t', 'long', '/dev/sdj'),
|
|
mock.call('smartctl', '-t', 'long', '/dev/hdaa')
|
|
]
|
|
mock_execute.assert_has_calls(expected_calls)
|
|
|
|
@mock.patch.object(hardware, 'list_all_block_devices', autospec=True)
|
|
def test_fio_disk_no_fio(self, mock_list, mock_execute):
|
|
|
|
node = {'driver_info': {}}
|
|
mock_execute.side_effect = (['out', 'err'],
|
|
processutils.ProcessExecutionError())
|
|
|
|
burnin.fio_disk(node)
|
|
|
|
self.assertRaises(errors.CommandExecutionError,
|
|
burnin.fio_disk, node)
|
|
|
|
def test_fio_network_reader(self, mock_execute):
|
|
|
|
node = {'driver_info': {'agent_burnin_fio_network_runtime': 600,
|
|
'agent_burnin_fio_network_config':
|
|
{'partner': 'host-002',
|
|
'role': 'reader'}}}
|
|
mock_execute.return_value = (['out', 'err'])
|
|
|
|
burnin.fio_network(node)
|
|
|
|
expected_calls = [
|
|
mock.call('fio', '--ioengine', 'net', '--port', '9000',
|
|
'--fill_device', 1, '--group_reporting',
|
|
'--gtod_reduce', 1, '--numjobs', 16, '--name',
|
|
'reader', '--rw', 'read', '--hostname', 'host-002'),
|
|
mock.call('fio', '--ioengine', 'net', '--port', '9000',
|
|
'--fill_device', 1, '--group_reporting',
|
|
'--gtod_reduce', 1, '--numjobs', 16, '--name', 'writer',
|
|
'--rw', 'write', '--runtime', 600, '--time_based',
|
|
'--listen')]
|
|
mock_execute.assert_has_calls(expected_calls)
|
|
|
|
def test_fio_network_reader_w_logfile(self, mock_execute):
|
|
|
|
node = {'driver_info': {
|
|
'agent_burnin_fio_network_runtime': 600,
|
|
'agent_burnin_fio_network_config':
|
|
{'partner': 'host-002',
|
|
'role': 'reader'},
|
|
'agent_burnin_fio_network_outputfile': '/var/log/burnin.network'}}
|
|
mock_execute.return_value = (['out', 'err'])
|
|
|
|
burnin.fio_network(node)
|
|
|
|
expected_calls = [
|
|
mock.call('fio', '--ioengine', 'net', '--port', '9000',
|
|
'--fill_device', 1, '--group_reporting',
|
|
'--gtod_reduce', 1, '--numjobs', 16, '--name',
|
|
'reader', '--rw', 'read', '--hostname', 'host-002',
|
|
'--output-format', 'json', '--output',
|
|
'/var/log/burnin.network.reader'),
|
|
mock.call('fio', '--ioengine', 'net', '--port', '9000',
|
|
'--fill_device', 1, '--group_reporting',
|
|
'--gtod_reduce', 1, '--numjobs', 16, '--name', 'writer',
|
|
'--rw', 'write', '--runtime', 600, '--time_based',
|
|
'--listen', '--output-format', 'json', '--output',
|
|
'/var/log/burnin.network.writer')]
|
|
mock_execute.assert_has_calls(expected_calls)
|
|
|
|
def test_fio_network_writer(self, mock_execute):
|
|
|
|
node = {'driver_info': {'agent_burnin_fio_network_runtime': 600,
|
|
'agent_burnin_fio_network_config':
|
|
{'partner': 'host-001',
|
|
'role': 'writer'}}}
|
|
mock_execute.return_value = (['out', 'err'])
|
|
|
|
burnin.fio_network(node)
|
|
|
|
expected_calls = [
|
|
mock.call('fio', '--ioengine', 'net', '--port', '9000',
|
|
'--fill_device', 1, '--group_reporting',
|
|
'--gtod_reduce', 1, '--numjobs', 16, '--name', 'writer',
|
|
'--rw', 'write', '--runtime', 600, '--time_based',
|
|
'--listen'),
|
|
mock.call('fio', '--ioengine', 'net', '--port', '9000',
|
|
'--fill_device', 1, '--group_reporting',
|
|
'--gtod_reduce', 1, '--numjobs', 16, '--name',
|
|
'reader', '--rw', 'read', '--hostname', 'host-001')]
|
|
mock_execute.assert_has_calls(expected_calls)
|
|
|
|
def test_fio_network_writer_w_logfile(self, mock_execute):
|
|
|
|
node = {'driver_info': {
|
|
'agent_burnin_fio_network_runtime': 600,
|
|
'agent_burnin_fio_network_config':
|
|
{'partner': 'host-001',
|
|
'role': 'writer'},
|
|
'agent_burnin_fio_network_outputfile': '/var/log/burnin.network'}}
|
|
mock_execute.return_value = (['out', 'err'])
|
|
|
|
burnin.fio_network(node)
|
|
|
|
expected_calls = [
|
|
mock.call('fio', '--ioengine', 'net', '--port', '9000',
|
|
'--fill_device', 1, '--group_reporting',
|
|
'--gtod_reduce', 1, '--numjobs', 16, '--name', 'writer',
|
|
'--rw', 'write', '--runtime', 600, '--time_based',
|
|
'--listen', '--output-format', 'json', '--output',
|
|
'/var/log/burnin.network.writer'),
|
|
mock.call('fio', '--ioengine', 'net', '--port', '9000',
|
|
'--fill_device', 1, '--group_reporting',
|
|
'--gtod_reduce', 1, '--numjobs', 16, '--name',
|
|
'reader', '--rw', 'read', '--hostname', 'host-001',
|
|
'--output-format', 'json', '--output',
|
|
'/var/log/burnin.network.reader')]
|
|
mock_execute.assert_has_calls(expected_calls)
|
|
|
|
def test_fio_network_no_fio(self, mock_execute):
|
|
|
|
node = {'driver_info': {'agent_burnin_fio_network_config':
|
|
{'partner': 'host-003', 'role': 'reader'}}}
|
|
mock_execute.side_effect = processutils.ProcessExecutionError('boom')
|
|
|
|
self.assertRaises(errors.CommandExecutionError,
|
|
burnin.fio_network, node)
|
|
|
|
def test_fio_network_unknown_role(self, mock_execute):
|
|
|
|
node = {'driver_info': {'agent_burnin_fio_network_config':
|
|
{'partner': 'host-003', 'role': 'read'}}}
|
|
|
|
self.assertRaises(errors.CleaningError, burnin.fio_network, node)
|
|
|
|
def test_fio_network_no_role(self, mock_execute):
|
|
|
|
node = {'driver_info': {'agent_burnin_fio_network_config':
|
|
{'partner': 'host-003'}}}
|
|
|
|
self.assertRaises(errors.CleaningError, burnin.fio_network, node)
|
|
|
|
def test_fio_network_no_partner(self, mock_execute):
|
|
|
|
node = {'driver_info': {'agent_burnin_fio_network_config':
|
|
{'role': 'reader'}}}
|
|
|
|
self.assertRaises(errors.CleaningError, burnin.fio_network, node)
|
|
|
|
@mock.patch('time.sleep', autospec=True)
|
|
def test_fio_network_reader_loop(self, mock_time, mock_execute):
|
|
|
|
node = {'driver_info': {'agent_burnin_fio_network_config':
|
|
{'partner': 'host-004', 'role': 'reader'}}}
|
|
# mock the infinite loop
|
|
mock_execute.side_effect = (processutils.ProcessExecutionError(
|
|
'Connection timeout', exit_code=16),
|
|
processutils.ProcessExecutionError(
|
|
'Connection timeout', exit_code=16),
|
|
processutils.ProcessExecutionError(
|
|
'Connection refused', exit_code=16),
|
|
['out', 'err'], # connected!
|
|
['out', 'err']) # reversed roles
|
|
|
|
burnin.fio_network(node)
|
|
|
|
# we loop 3 times, then do the 2 fio calls
|
|
self.assertEqual(5, mock_execute.call_count)
|
|
self.assertEqual(3, mock_time.call_count)
|
|
|
|
def test_fio_network_dynamic_pairing_raise_missing_config(self,
|
|
mock_execute):
|
|
node = {'driver_info': {}}
|
|
self.assertRaises(errors.CleaningError, burnin.fio_network, node)
|
|
|
|
def test_fio_network_dynamic_pairing_raise_wrong_config(self,
|
|
mock_execute):
|
|
node = {'driver_info': {
|
|
'backend_url': 'zookeeper://zookeeper-host-01:2181',
|
|
'group_name': 'ironic.dynamic-network-burnin',
|
|
'timeout': 600}}
|
|
self.assertRaises(errors.CleaningError, burnin.fio_network, node)
|
|
|
|
@mock.patch.object(burnin, '_find_network_burnin_partner_and_role',
|
|
autospec=True)
|
|
def test_fio_network_dynamic_pairing_defaults(self, mock_find,
|
|
mock_execute):
|
|
node = {'driver_info': {
|
|
'agent_burnin_fio_network_pairing_backend_url':
|
|
'zookeeper://zookeeper-host-01:2181'}}
|
|
mock_find.return_value = ['partner-host', 'reader']
|
|
mock_execute.return_value = (['out', 'err'])
|
|
|
|
burnin.fio_network(node)
|
|
|
|
mock_find.assert_called_once_with(
|
|
backend_url='zookeeper://zookeeper-host-01:2181',
|
|
group_name='ironic.network-burnin',
|
|
timeout=900)
|
|
|
|
@mock.patch.object(burnin, '_find_network_burnin_partner_and_role',
|
|
autospec=True)
|
|
def test_fio_network_dynamic_pairing_no_defaults(self, mock_find,
|
|
mock_execute):
|
|
node = {'driver_info': {
|
|
'agent_burnin_fio_network_pairing_backend_url':
|
|
'zookeeper://zookeeper-host-01:2181',
|
|
'agent_burnin_fio_network_pairing_group_name':
|
|
'ironic.special-group',
|
|
'agent_burnin_fio_network_pairing_timeout': 600}}
|
|
mock_find.return_value = ['partner-host', 'reader']
|
|
mock_execute.return_value = (['out', 'err'])
|
|
|
|
burnin.fio_network(node)
|
|
|
|
mock_find.assert_called_once_with(
|
|
backend_url='zookeeper://zookeeper-host-01:2181',
|
|
group_name='ironic.special-group',
|
|
timeout=600)
|
|
|
|
@mock.patch.object(coordination, 'get_coordinator', autospec=True)
|
|
def test_fio_network_dynamic_find_timeout(self, mock_get_coordinator,
|
|
mock_execute):
|
|
mock_coordinator = mock.MagicMock()
|
|
mock_get_coordinator.return_value = mock_coordinator
|
|
|
|
# timeout since no other node is joining
|
|
self.assertRaises(errors.CleaningError,
|
|
burnin._find_network_burnin_partner_and_role,
|
|
"zk://xyz", 'group', 2)
|
|
|
|
# group did not exist, so we created it
|
|
mock_coordinator.create_group.assert_called_once_with('group')
|
|
mock_coordinator.join_group.assert_called_once()
|
|
# get_members is called initially, then every second
|
|
# up to the timeout
|
|
self.assertEqual(3, mock_coordinator.get_members.call_count)
|
|
|
|
@mock.patch.object(coordination, 'get_coordinator', autospec=True)
|
|
def test_fio_network_dynamic_find_pair_1st(self, mock_get_coordinator,
|
|
mock_execute):
|
|
mock_coordinator = mock.MagicMock()
|
|
mock_get_coordinator.return_value = mock_coordinator
|
|
|
|
class Members:
|
|
def __init__(self, members=[]):
|
|
self.members = members
|
|
|
|
def get(self):
|
|
return self.members
|
|
|
|
# we are the first node to enter, so no other host
|
|
# initially until the second one appears after some
|
|
# interations
|
|
mock_coordinator.get_members.side_effect = \
|
|
[Members(), Members([b'host1']), Members([b'host1']),
|
|
Members([b'host1']), Members([b'host1', b'host2'])]
|
|
|
|
(partner, role) = \
|
|
burnin._find_network_burnin_partner_and_role("zk://xyz",
|
|
"group", 10)
|
|
|
|
# ... so we will leave first and be the writer
|
|
self.assertEqual((partner, role), ("host2", "writer"))
|
|
|
|
# group did not exist, so we created it
|
|
mock_coordinator.create_group.assert_called_once_with('group')
|
|
mock_coordinator.join_group.assert_called_once()
|
|
# get_members is called initially, then every second
|
|
# up to the timeout
|
|
self.assertEqual(5, mock_coordinator.get_members.call_count)
|
|
|
|
@mock.patch.object(coordination, 'get_coordinator', autospec=True)
|
|
def test_fio_network_dynamic_find_pair_2nd(self, mock_get_coordinator,
|
|
mock_execute):
|
|
mock_coordinator = mock.MagicMock()
|
|
mock_get_coordinator.return_value = mock_coordinator
|
|
|
|
class Members:
|
|
def __init__(self, members=[]):
|
|
self.members = members
|
|
|
|
def get(self):
|
|
return self.members
|
|
|
|
# we are the second node to enter, host1 is there before us ...
|
|
mock_coordinator.get_members.side_effect = \
|
|
[Members([b'host1']),
|
|
Members([b'host1', b'host2']),
|
|
Members([b'host2'])]
|
|
|
|
(partner, role) = \
|
|
burnin._find_network_burnin_partner_and_role("zk://xyz",
|
|
"group", 10)
|
|
|
|
# ... so we will leave second and be the reader
|
|
self.assertEqual((partner, role), ("host1", "reader"))
|
|
|
|
# group did not exist, so we created it
|
|
mock_coordinator.create_group.assert_called_once_with('group')
|
|
mock_coordinator.join_group.assert_called_once()
|
|
# get_members is called initially, then every second until the
|
|
# other node appears
|
|
self.assertEqual(3, mock_coordinator.get_members.call_count)
|