36e5993a04
This fixes several spelling issues identified by codepsell. In some cases, I may have manually modified a line to make the output more clear or to correct grammatical issues which were obvious in the codespell output. Later changes in this chain will provide the codespell config used to generate this, as well as adding this commit's SHA, once landed, to a .git-blame-ignore-revs file to ensure it will not pollute git historys for modern clients. Related-Bug: 2047654 Change-Id: I240cf8484865c9b748ceb51f3c7b9fd973cb5ada
518 lines
21 KiB
Python
518 lines
21 KiB
Python
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
from unittest import mock
|
|
|
|
from ironic_lib import utils
|
|
from oslo_concurrency import processutils
|
|
from tooz import coordination
|
|
|
|
from ironic_python_agent import burnin
|
|
from ironic_python_agent import errors
|
|
from ironic_python_agent import hardware
|
|
from ironic_python_agent.tests.unit import base
|
|
|
|
|
|
SMART_OUTPUT_JSON_COMPLETED = ("""
|
|
{
|
|
"ata_smart_data": {
|
|
"self_test": {
|
|
"status": {
|
|
"value": 0,
|
|
"string": "completed without error",
|
|
"passed": true
|
|
},
|
|
"polling_minutes": {
|
|
"short": 1,
|
|
"extended": 2,
|
|
"conveyance": 2
|
|
}
|
|
}
|
|
}
|
|
}
|
|
""")
|
|
|
|
SMART_OUTPUT_JSON_MISSING = ("""
|
|
{
|
|
"ata_smart_data": {
|
|
"self_test": {
|
|
"status": {
|
|
"value": 0,
|
|
"passed": true
|
|
}
|
|
}
|
|
}
|
|
}
|
|
""")
|
|
|
|
|
|
@mock.patch.object(utils, 'execute', autospec=True)
|
|
class TestBurnin(base.IronicAgentTest):
|
|
|
|
def test_stress_ng_cpu_default(self, mock_execute):
|
|
|
|
node = {'driver_info': {}}
|
|
mock_execute.return_value = (['out', 'err'])
|
|
|
|
burnin.stress_ng_cpu(node)
|
|
|
|
mock_execute.assert_called_once_with(
|
|
'stress-ng', '--cpu', 0, '--timeout', 86400, '--metrics-brief')
|
|
|
|
def test_stress_ng_cpu_non_default(self, mock_execute):
|
|
|
|
node = {'driver_info': {
|
|
'agent_burnin_cpu_cpu': 3,
|
|
'agent_burnin_cpu_timeout': 2911,
|
|
'agent_burnin_cpu_outputfile': '/var/log/burnin.cpu'}}
|
|
mock_execute.return_value = (['out', 'err'])
|
|
|
|
burnin.stress_ng_cpu(node)
|
|
|
|
mock_execute.assert_called_once_with(
|
|
'stress-ng', '--cpu', 3, '--timeout', 2911, '--metrics-brief',
|
|
'--log-file', '/var/log/burnin.cpu')
|
|
|
|
def test_stress_ng_cpu_no_stress_ng(self, mock_execute):
|
|
|
|
node = {'driver_info': {}}
|
|
mock_execute.side_effect = (['out', 'err'],
|
|
processutils.ProcessExecutionError())
|
|
|
|
burnin.stress_ng_cpu(node)
|
|
|
|
self.assertRaises(errors.CommandExecutionError,
|
|
burnin.stress_ng_cpu, node)
|
|
|
|
def test_stress_ng_vm_default(self, mock_execute):
|
|
|
|
node = {'driver_info': {}}
|
|
mock_execute.return_value = (['out', 'err'])
|
|
|
|
burnin.stress_ng_vm(node)
|
|
|
|
mock_execute.assert_called_once_with(
|
|
|
|
'stress-ng', '--vm', 0, '--vm-bytes', '98%',
|
|
'--timeout', 86400, '--metrics-brief')
|
|
|
|
def test_stress_ng_vm_non_default(self, mock_execute):
|
|
|
|
node = {'driver_info': {
|
|
'agent_burnin_vm_vm': 2,
|
|
'agent_burnin_vm_vm-bytes': '25%',
|
|
'agent_burnin_vm_timeout': 120,
|
|
'agent_burnin_vm_outputfile': '/var/log/burnin.vm'}}
|
|
mock_execute.return_value = (['out', 'err'])
|
|
|
|
burnin.stress_ng_vm(node)
|
|
|
|
mock_execute.assert_called_once_with(
|
|
'stress-ng', '--vm', 2, '--vm-bytes', '25%',
|
|
'--timeout', 120, '--metrics-brief',
|
|
'--log-file', '/var/log/burnin.vm')
|
|
|
|
def test_stress_ng_vm_no_stress_ng(self, mock_execute):
|
|
|
|
node = {'driver_info': {}}
|
|
mock_execute.side_effect = (['out', 'err'],
|
|
processutils.ProcessExecutionError())
|
|
|
|
burnin.stress_ng_vm(node)
|
|
|
|
self.assertRaises(errors.CommandExecutionError,
|
|
burnin.stress_ng_vm, node)
|
|
|
|
@mock.patch.object(hardware, 'list_all_block_devices', autospec=True)
|
|
def test_fio_disk_default(self, mock_list, mock_execute):
|
|
|
|
node = {'driver_info': {}}
|
|
|
|
mock_list.return_value = [
|
|
hardware.BlockDevice('/dev/sdj', 'big', 1073741824, True),
|
|
hardware.BlockDevice('/dev/hdaa', 'small', 65535, False),
|
|
]
|
|
mock_execute.return_value = (['out', 'err'])
|
|
|
|
burnin.fio_disk(node)
|
|
|
|
mock_execute.assert_called_once_with(
|
|
'fio', '--rw', 'readwrite', '--bs', '4k', '--direct', 1,
|
|
'--ioengine', 'libaio', '--iodepth', '32', '--verify',
|
|
'crc32c', '--verify_dump', 1, '--continue_on_error', 'verify',
|
|
'--loops', 4, '--runtime', 0, '--time_based', '--name',
|
|
'/dev/sdj', '--name', '/dev/hdaa')
|
|
|
|
@mock.patch.object(hardware, 'list_all_block_devices', autospec=True)
|
|
def test_fio_disk_no_default(self, mock_list, mock_execute):
|
|
|
|
node = {'driver_info': {
|
|
'agent_burnin_fio_disk_runtime': 600,
|
|
'agent_burnin_fio_disk_loops': 5,
|
|
'agent_burnin_fio_disk_outputfile': '/var/log/burnin.disk'}}
|
|
|
|
mock_list.return_value = [
|
|
hardware.BlockDevice('/dev/sdj', 'big', 1073741824, True),
|
|
hardware.BlockDevice('/dev/hdaa', 'small', 65535, False),
|
|
]
|
|
mock_execute.return_value = (['out', 'err'])
|
|
|
|
burnin.fio_disk(node)
|
|
|
|
mock_execute.assert_called_once_with(
|
|
'fio', '--rw', 'readwrite', '--bs', '4k', '--direct', 1,
|
|
'--ioengine', 'libaio', '--iodepth', '32', '--verify',
|
|
'crc32c', '--verify_dump', 1, '--continue_on_error', 'verify',
|
|
'--loops', 5, '--runtime', 600, '--time_based', '--output-format',
|
|
'json', '--output', '/var/log/burnin.disk', '--name', '/dev/sdj',
|
|
'--name', '/dev/hdaa', )
|
|
|
|
def test__smart_test_status(self, mock_execute):
|
|
device = hardware.BlockDevice('/dev/sdj', 'big', 1073741824, True)
|
|
mock_execute.return_value = ([SMART_OUTPUT_JSON_COMPLETED, 'err'])
|
|
|
|
status = burnin._smart_test_status(device)
|
|
|
|
mock_execute.assert_called_once_with('smartctl', '-ja', '/dev/sdj')
|
|
self.assertEqual(status, "completed without error")
|
|
|
|
def test__smart_test_status_missing(self, mock_execute):
|
|
device = hardware.BlockDevice('/dev/sdj', 'big', 1073741824, True)
|
|
mock_execute.return_value = ([SMART_OUTPUT_JSON_MISSING, 'err'])
|
|
|
|
status = burnin._smart_test_status(device)
|
|
|
|
mock_execute.assert_called_once_with('smartctl', '-ja', '/dev/sdj')
|
|
self.assertIsNone(status)
|
|
|
|
@mock.patch.object(burnin, '_smart_test_status', autospec=True)
|
|
@mock.patch.object(hardware, 'list_all_block_devices', autospec=True)
|
|
def test_fio_disk_smart_test(self, mock_list, mock_status, mock_execute):
|
|
|
|
node = {'driver_info': {'agent_burnin_fio_disk_smart_test': True}}
|
|
|
|
mock_list.return_value = [
|
|
hardware.BlockDevice('/dev/sdj', 'big', 1073741824, True),
|
|
hardware.BlockDevice('/dev/hdaa', 'small', 65535, False),
|
|
]
|
|
mock_status.return_value = "completed without error"
|
|
mock_execute.return_value = (['out', 'err'])
|
|
|
|
burnin.fio_disk(node)
|
|
|
|
expected_calls = [
|
|
mock.call('fio', '--rw', 'readwrite', '--bs', '4k', '--direct', 1,
|
|
'--ioengine', 'libaio', '--iodepth', '32', '--verify',
|
|
'crc32c', '--verify_dump', 1, '--continue_on_error',
|
|
'verify', '--loops', 4, '--runtime', 0, '--time_based',
|
|
'--name', '/dev/sdj', '--name', '/dev/hdaa'),
|
|
mock.call('smartctl', '-t', 'long', '/dev/sdj'),
|
|
mock.call('smartctl', '-t', 'long', '/dev/hdaa')
|
|
]
|
|
mock_execute.assert_has_calls(expected_calls)
|
|
|
|
@mock.patch.object(hardware, 'list_all_block_devices', autospec=True)
|
|
def test_fio_disk_no_fio(self, mock_list, mock_execute):
|
|
|
|
node = {'driver_info': {}}
|
|
mock_execute.side_effect = (['out', 'err'],
|
|
processutils.ProcessExecutionError())
|
|
|
|
burnin.fio_disk(node)
|
|
|
|
self.assertRaises(errors.CommandExecutionError,
|
|
burnin.fio_disk, node)
|
|
|
|
def test_fio_network_reader(self, mock_execute):
|
|
|
|
node = {'driver_info': {'agent_burnin_fio_network_runtime': 600,
|
|
'agent_burnin_fio_network_config':
|
|
{'partner': 'host-002',
|
|
'role': 'reader'}}}
|
|
mock_execute.return_value = (['out', 'err'])
|
|
|
|
burnin.fio_network(node)
|
|
|
|
expected_calls = [
|
|
mock.call('fio', '--ioengine', 'net', '--port', '9000',
|
|
'--fill_device', 1, '--group_reporting',
|
|
'--gtod_reduce', 1, '--numjobs', 16, '--name',
|
|
'reader', '--rw', 'read', '--hostname', 'host-002'),
|
|
mock.call('fio', '--ioengine', 'net', '--port', '9000',
|
|
'--fill_device', 1, '--group_reporting',
|
|
'--gtod_reduce', 1, '--numjobs', 16, '--name', 'writer',
|
|
'--rw', 'write', '--runtime', 600, '--time_based',
|
|
'--listen')]
|
|
mock_execute.assert_has_calls(expected_calls)
|
|
|
|
def test_fio_network_reader_w_logfile(self, mock_execute):
|
|
|
|
node = {'driver_info': {
|
|
'agent_burnin_fio_network_runtime': 600,
|
|
'agent_burnin_fio_network_config':
|
|
{'partner': 'host-002',
|
|
'role': 'reader'},
|
|
'agent_burnin_fio_network_outputfile': '/var/log/burnin.network'}}
|
|
mock_execute.return_value = (['out', 'err'])
|
|
|
|
burnin.fio_network(node)
|
|
|
|
expected_calls = [
|
|
mock.call('fio', '--ioengine', 'net', '--port', '9000',
|
|
'--fill_device', 1, '--group_reporting',
|
|
'--gtod_reduce', 1, '--numjobs', 16, '--name',
|
|
'reader', '--rw', 'read', '--hostname', 'host-002',
|
|
'--output-format', 'json', '--output',
|
|
'/var/log/burnin.network.reader'),
|
|
mock.call('fio', '--ioengine', 'net', '--port', '9000',
|
|
'--fill_device', 1, '--group_reporting',
|
|
'--gtod_reduce', 1, '--numjobs', 16, '--name', 'writer',
|
|
'--rw', 'write', '--runtime', 600, '--time_based',
|
|
'--listen', '--output-format', 'json', '--output',
|
|
'/var/log/burnin.network.writer')]
|
|
mock_execute.assert_has_calls(expected_calls)
|
|
|
|
def test_fio_network_writer(self, mock_execute):
|
|
|
|
node = {'driver_info': {'agent_burnin_fio_network_runtime': 600,
|
|
'agent_burnin_fio_network_config':
|
|
{'partner': 'host-001',
|
|
'role': 'writer'}}}
|
|
mock_execute.return_value = (['out', 'err'])
|
|
|
|
burnin.fio_network(node)
|
|
|
|
expected_calls = [
|
|
mock.call('fio', '--ioengine', 'net', '--port', '9000',
|
|
'--fill_device', 1, '--group_reporting',
|
|
'--gtod_reduce', 1, '--numjobs', 16, '--name', 'writer',
|
|
'--rw', 'write', '--runtime', 600, '--time_based',
|
|
'--listen'),
|
|
mock.call('fio', '--ioengine', 'net', '--port', '9000',
|
|
'--fill_device', 1, '--group_reporting',
|
|
'--gtod_reduce', 1, '--numjobs', 16, '--name',
|
|
'reader', '--rw', 'read', '--hostname', 'host-001')]
|
|
mock_execute.assert_has_calls(expected_calls)
|
|
|
|
def test_fio_network_writer_w_logfile(self, mock_execute):
|
|
|
|
node = {'driver_info': {
|
|
'agent_burnin_fio_network_runtime': 600,
|
|
'agent_burnin_fio_network_config':
|
|
{'partner': 'host-001',
|
|
'role': 'writer'},
|
|
'agent_burnin_fio_network_outputfile': '/var/log/burnin.network'}}
|
|
mock_execute.return_value = (['out', 'err'])
|
|
|
|
burnin.fio_network(node)
|
|
|
|
expected_calls = [
|
|
mock.call('fio', '--ioengine', 'net', '--port', '9000',
|
|
'--fill_device', 1, '--group_reporting',
|
|
'--gtod_reduce', 1, '--numjobs', 16, '--name', 'writer',
|
|
'--rw', 'write', '--runtime', 600, '--time_based',
|
|
'--listen', '--output-format', 'json', '--output',
|
|
'/var/log/burnin.network.writer'),
|
|
mock.call('fio', '--ioengine', 'net', '--port', '9000',
|
|
'--fill_device', 1, '--group_reporting',
|
|
'--gtod_reduce', 1, '--numjobs', 16, '--name',
|
|
'reader', '--rw', 'read', '--hostname', 'host-001',
|
|
'--output-format', 'json', '--output',
|
|
'/var/log/burnin.network.reader')]
|
|
mock_execute.assert_has_calls(expected_calls)
|
|
|
|
def test_fio_network_no_fio(self, mock_execute):
|
|
|
|
node = {'driver_info': {'agent_burnin_fio_network_config':
|
|
{'partner': 'host-003', 'role': 'reader'}}}
|
|
mock_execute.side_effect = processutils.ProcessExecutionError('boom')
|
|
|
|
self.assertRaises(errors.CommandExecutionError,
|
|
burnin.fio_network, node)
|
|
|
|
def test_fio_network_unknown_role(self, mock_execute):
|
|
|
|
node = {'driver_info': {'agent_burnin_fio_network_config':
|
|
{'partner': 'host-003', 'role': 'read'}}}
|
|
|
|
self.assertRaises(errors.CleaningError, burnin.fio_network, node)
|
|
|
|
def test_fio_network_no_role(self, mock_execute):
|
|
|
|
node = {'driver_info': {'agent_burnin_fio_network_config':
|
|
{'partner': 'host-003'}}}
|
|
|
|
self.assertRaises(errors.CleaningError, burnin.fio_network, node)
|
|
|
|
def test_fio_network_no_partner(self, mock_execute):
|
|
|
|
node = {'driver_info': {'agent_burnin_fio_network_config':
|
|
{'role': 'reader'}}}
|
|
|
|
self.assertRaises(errors.CleaningError, burnin.fio_network, node)
|
|
|
|
@mock.patch('time.sleep', autospec=True)
|
|
def test_fio_network_reader_loop(self, mock_time, mock_execute):
|
|
|
|
node = {'driver_info': {'agent_burnin_fio_network_config':
|
|
{'partner': 'host-004', 'role': 'reader'}}}
|
|
# mock the infinite loop
|
|
mock_execute.side_effect = (processutils.ProcessExecutionError(
|
|
'Connection timeout', exit_code=16),
|
|
processutils.ProcessExecutionError(
|
|
'Connection timeout', exit_code=16),
|
|
processutils.ProcessExecutionError(
|
|
'Connection refused', exit_code=16),
|
|
['out', 'err'], # connected!
|
|
['out', 'err']) # reversed roles
|
|
|
|
burnin.fio_network(node)
|
|
|
|
# we loop 3 times, then do the 2 fio calls
|
|
self.assertEqual(5, mock_execute.call_count)
|
|
self.assertEqual(3, mock_time.call_count)
|
|
|
|
def test_fio_network_dynamic_pairing_raise_missing_config(self,
|
|
mock_execute):
|
|
node = {'driver_info': {}}
|
|
self.assertRaises(errors.CleaningError, burnin.fio_network, node)
|
|
|
|
def test_fio_network_dynamic_pairing_raise_wrong_config(self,
|
|
mock_execute):
|
|
node = {'driver_info': {
|
|
'backend_url': 'zookeeper://zookeeper-host-01:2181',
|
|
'group_name': 'ironic.dynamic-network-burnin',
|
|
'timeout': 600}}
|
|
self.assertRaises(errors.CleaningError, burnin.fio_network, node)
|
|
|
|
@mock.patch.object(burnin, '_find_network_burnin_partner_and_role',
|
|
autospec=True)
|
|
def test_fio_network_dynamic_pairing_defaults(self, mock_find,
|
|
mock_execute):
|
|
node = {'driver_info': {
|
|
'agent_burnin_fio_network_pairing_backend_url':
|
|
'zookeeper://zookeeper-host-01:2181'}}
|
|
mock_find.return_value = ['partner-host', 'reader']
|
|
mock_execute.return_value = (['out', 'err'])
|
|
|
|
burnin.fio_network(node)
|
|
|
|
mock_find.assert_called_once_with(
|
|
backend_url='zookeeper://zookeeper-host-01:2181',
|
|
group_name='ironic.network-burnin',
|
|
timeout=900)
|
|
|
|
@mock.patch.object(burnin, '_find_network_burnin_partner_and_role',
|
|
autospec=True)
|
|
def test_fio_network_dynamic_pairing_no_defaults(self, mock_find,
|
|
mock_execute):
|
|
node = {'driver_info': {
|
|
'agent_burnin_fio_network_pairing_backend_url':
|
|
'zookeeper://zookeeper-host-01:2181',
|
|
'agent_burnin_fio_network_pairing_group_name':
|
|
'ironic.special-group',
|
|
'agent_burnin_fio_network_pairing_timeout': 600}}
|
|
mock_find.return_value = ['partner-host', 'reader']
|
|
mock_execute.return_value = (['out', 'err'])
|
|
|
|
burnin.fio_network(node)
|
|
|
|
mock_find.assert_called_once_with(
|
|
backend_url='zookeeper://zookeeper-host-01:2181',
|
|
group_name='ironic.special-group',
|
|
timeout=600)
|
|
|
|
@mock.patch.object(coordination, 'get_coordinator', autospec=True)
|
|
def test_fio_network_dynamic_find_timeout(self, mock_get_coordinator,
|
|
mock_execute):
|
|
mock_coordinator = mock.MagicMock()
|
|
mock_get_coordinator.return_value = mock_coordinator
|
|
|
|
# timeout since no other node is joining
|
|
self.assertRaises(errors.CleaningError,
|
|
burnin._find_network_burnin_partner_and_role,
|
|
"zk://xyz", 'group', 2)
|
|
|
|
# group did not exist, so we created it
|
|
mock_coordinator.create_group.assert_called_once_with('group')
|
|
mock_coordinator.join_group.assert_called_once()
|
|
# get_members is called initially, then every second
|
|
# up to the timeout
|
|
self.assertEqual(3, mock_coordinator.get_members.call_count)
|
|
|
|
@mock.patch.object(coordination, 'get_coordinator', autospec=True)
|
|
def test_fio_network_dynamic_find_pair_1st(self, mock_get_coordinator,
|
|
mock_execute):
|
|
mock_coordinator = mock.MagicMock()
|
|
mock_get_coordinator.return_value = mock_coordinator
|
|
|
|
class Members:
|
|
def __init__(self, members=[]):
|
|
self.members = members
|
|
|
|
def get(self):
|
|
return self.members
|
|
|
|
# we are the first node to enter, so no other host
|
|
# initially until the second one appears after some
|
|
# iterations
|
|
mock_coordinator.get_members.side_effect = \
|
|
[Members(), Members([b'host1']), Members([b'host1']),
|
|
Members([b'host1']), Members([b'host1', b'host2'])]
|
|
|
|
(partner, role) = \
|
|
burnin._find_network_burnin_partner_and_role("zk://xyz",
|
|
"group", 10)
|
|
|
|
# ... so we will leave first and be the writer
|
|
self.assertEqual((partner, role), ("host2", "writer"))
|
|
|
|
# group did not exist, so we created it
|
|
mock_coordinator.create_group.assert_called_once_with('group')
|
|
mock_coordinator.join_group.assert_called_once()
|
|
# get_members is called initially, then every second
|
|
# up to the timeout
|
|
self.assertEqual(5, mock_coordinator.get_members.call_count)
|
|
|
|
@mock.patch.object(coordination, 'get_coordinator', autospec=True)
|
|
def test_fio_network_dynamic_find_pair_2nd(self, mock_get_coordinator,
|
|
mock_execute):
|
|
mock_coordinator = mock.MagicMock()
|
|
mock_get_coordinator.return_value = mock_coordinator
|
|
|
|
class Members:
|
|
def __init__(self, members=[]):
|
|
self.members = members
|
|
|
|
def get(self):
|
|
return self.members
|
|
|
|
# we are the second node to enter, host1 is there before us ...
|
|
mock_coordinator.get_members.side_effect = \
|
|
[Members([b'host1']),
|
|
Members([b'host1', b'host2']),
|
|
Members([b'host2'])]
|
|
|
|
(partner, role) = \
|
|
burnin._find_network_burnin_partner_and_role("zk://xyz",
|
|
"group", 10)
|
|
|
|
# ... so we will leave second and be the reader
|
|
self.assertEqual((partner, role), ("host1", "reader"))
|
|
|
|
# group did not exist, so we created it
|
|
mock_coordinator.create_group.assert_called_once_with('group')
|
|
mock_coordinator.join_group.assert_called_once()
|
|
# get_members is called initially, then every second until the
|
|
# other node appears
|
|
self.assertEqual(3, mock_coordinator.get_members.call_count)
|