From 393b20204b3adab12e69e6e1d24a221e2751e6ec Mon Sep 17 00:00:00 2001 From: Steve Baker Date: Thu, 13 Oct 2022 10:33:14 +1300 Subject: [PATCH] Add configurable delays to the fake drivers Simulating workloads with the fake driver currently misses the reality that some operations take time to complete, rather than occuring instantly. This makes it difficult to mock real workloads for performance and functional testing of ironic itself. This change adds configurable random wait times for fake drivers in a new ironic.conf [fake] section. Each supported driver having one configuration option controlling the delay. These delays are applied to operations which typically block in other drivers. The default value of zero continues the existing behaviour of no delay. A single integer value will result in a constant delay in seconds. Two values separated by a comma will result in a triangular distribution weighted by the first value, specifically in python[1]: random.triangular(a, b, a) Change-Id: I7cb1b50d035939e6c4538b3373002a309bfedea4 [1] https://docs.python.org/3/library/random.html#random.triangular --- ironic/conf/__init__.py | 2 + ironic/conf/fake.py | 85 +++++++++++++++++++ ironic/drivers/modules/fake.py | 63 ++++++++++++++ .../tests/unit/drivers/test_fake_hardware.py | 29 +++++++ .../notes/fakedelay-7eac23ad8881a736.yaml | 8 ++ 5 files changed, 187 insertions(+) create mode 100644 ironic/conf/fake.py create mode 100644 releasenotes/notes/fakedelay-7eac23ad8881a736.yaml diff --git a/ironic/conf/__init__.py b/ironic/conf/__init__.py index ad1ba227c7..ca7ce74d45 100644 --- a/ironic/conf/__init__.py +++ b/ironic/conf/__init__.py @@ -29,6 +29,7 @@ from ironic.conf import deploy from ironic.conf import dhcp from ironic.conf import dnsmasq from ironic.conf import drac +from ironic.conf import fake from ironic.conf import glance from ironic.conf import healthcheck from ironic.conf import ibmc @@ -64,6 +65,7 @@ deploy.register_opts(CONF) drac.register_opts(CONF) dhcp.register_opts(CONF) dnsmasq.register_opts(CONF) +fake.register_opts(CONF) glance.register_opts(CONF) healthcheck.register_opts(CONF) ibmc.register_opts(CONF) diff --git a/ironic/conf/fake.py b/ironic/conf/fake.py new file mode 100644 index 0000000000..8f6d75ee3e --- /dev/null +++ b/ironic/conf/fake.py @@ -0,0 +1,85 @@ +# +# Copyright 2022 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_config import cfg + +from ironic.common.i18n import _ + +opts = [ + cfg.StrOpt('power_delay', + default='0', + help=_('Delay in seconds for operations with the fake ' + 'power driver. Two comma-delimited values will ' + 'result in a delay with a triangular random ' + 'distribution, weighted on the first value.')), + cfg.StrOpt('boot_delay', + default='0', + help=_('Delay in seconds for operations with the fake ' + 'boot driver. Two comma-delimited values will ' + 'result in a delay with a triangular random ' + 'distribution, weighted on the first value.')), + cfg.StrOpt('deploy_delay', + default='0', + help=_('Delay in seconds for operations with the fake ' + 'deploy driver. Two comma-delimited values will ' + 'result in a delay with a triangular random ' + 'distribution, weighted on the first value.')), + cfg.StrOpt('vendor_delay', + default='0', + help=_('Delay in seconds for operations with the fake ' + 'vendor driver. Two comma-delimited values will ' + 'result in a delay with a triangular random ' + 'distribution, weighted on the first value.')), + cfg.StrOpt('management_delay', + default='0', + help=_('Delay in seconds for operations with the fake ' + 'management driver. Two comma-delimited values will ' + 'result in a delay with a triangular random ' + 'distribution, weighted on the first value.')), + cfg.StrOpt('inspect_delay', + default='0', + help=_('Delay in seconds for operations with the fake ' + 'inspect driver. Two comma-delimited values will ' + 'result in a delay with a triangular random ' + 'distribution, weighted on the first value.')), + cfg.StrOpt('raid_delay', + default='0', + help=_('Delay in seconds for operations with the fake ' + 'raid driver. Two comma-delimited values will ' + 'result in a delay with a triangular random ' + 'distribution, weighted on the first value.')), + cfg.StrOpt('bios_delay', + default='0', + help=_('Delay in seconds for operations with the fake ' + 'bios driver. Two comma-delimited values will ' + 'result in a delay with a triangular random ' + 'distribution, weighted on the first value.')), + cfg.StrOpt('storage_delay', + default='0', + help=_('Delay in seconds for operations with the fake ' + 'storage driver. Two comma-delimited values will ' + 'result in a delay with a triangular random ' + 'distribution, weighted on the first value.')), + cfg.StrOpt('rescue_delay', + default='0', + help=_('Delay in seconds for operations with the fake ' + 'rescue driver. Two comma-delimited values will ' + 'result in a delay with a triangular random ' + 'distribution, weighted on the first value.')), +] + + +def register_opts(conf): + conf.register_opts(opts, group='fake') diff --git a/ironic/drivers/modules/fake.py b/ironic/drivers/modules/fake.py index dffd9065df..0a26efb4ce 100644 --- a/ironic/drivers/modules/fake.py +++ b/ironic/drivers/modules/fake.py @@ -24,6 +24,9 @@ functionality between a power interface and a deploy interface, when both rely on separate vendor_passthru methods. """ +import random +import time + from oslo_log import log from ironic.common import boot_devices @@ -32,6 +35,7 @@ from ironic.common import exception from ironic.common.i18n import _ from ironic.common import indicator_states from ironic.common import states +from ironic.conf import CONF from ironic.drivers import base from ironic import objects @@ -39,6 +43,34 @@ from ironic import objects LOG = log.getLogger(__name__) +def parse_sleep_range(sleep_range): + if not sleep_range: + return 0, 0 + + sleep_split = sleep_range.split(',') + if len(sleep_split) == 1: + a = sleep_split[0] + b = sleep_split[0] + else: + a = sleep_split[0] + b = sleep_split[1] + return int(a), int(b) + + +def sleep(sleep_range): + earliest, latest = parse_sleep_range(sleep_range) + if earliest == 0 and latest == 0: + # no sleep + return + if earliest == latest: + # constant sleep + sleep = earliest + else: + # triangular random sleep, weighted towards the earliest + sleep = random.triangular(earliest, latest, earliest) + time.sleep(sleep) + + class FakePower(base.PowerInterface): """Example implementation of a simple power interface.""" @@ -49,12 +81,15 @@ class FakePower(base.PowerInterface): pass def get_power_state(self, task): + sleep(CONF.fake.power_delay) return task.node.power_state def reboot(self, task, timeout=None): + sleep(CONF.fake.power_delay) pass def set_power_state(self, task, power_state, timeout=None): + sleep(CONF.fake.power_delay) if power_state not in [states.POWER_ON, states.POWER_OFF, states.SOFT_REBOOT, states.SOFT_POWER_OFF]: raise exception.InvalidParameterValue( @@ -81,15 +116,19 @@ class FakeBoot(base.BootInterface): pass def prepare_ramdisk(self, task, ramdisk_params, mode='deploy'): + sleep(CONF.fake.boot_delay) pass def clean_up_ramdisk(self, task, mode='deploy'): + sleep(CONF.fake.boot_delay) pass def prepare_instance(self, task): + sleep(CONF.fake.boot_delay) pass def clean_up_instance(self, task): + sleep(CONF.fake.boot_delay) pass @@ -108,18 +147,23 @@ class FakeDeploy(base.DeployInterface): @base.deploy_step(priority=100) def deploy(self, task): + sleep(CONF.fake.deploy_delay) return None def tear_down(self, task): + sleep(CONF.fake.deploy_delay) return states.DELETED def prepare(self, task): + sleep(CONF.fake.deploy_delay) pass def clean_up(self, task): + sleep(CONF.fake.deploy_delay) pass def take_over(self, task): + sleep(CONF.fake.deploy_delay) pass @@ -140,6 +184,7 @@ class FakeVendorA(base.VendorInterface): @base.passthru(['POST'], description=_("Test if the value of bar is baz")) def first_method(self, task, http_method, bar): + sleep(CONF.fake.vendor_delay) return True if bar == 'baz' else False @@ -161,16 +206,19 @@ class FakeVendorB(base.VendorInterface): @base.passthru(['POST'], description=_("Test if the value of bar is kazoo")) def second_method(self, task, http_method, bar): + sleep(CONF.fake.vendor_delay) return True if bar == 'kazoo' else False @base.passthru(['POST'], async_call=False, description=_("Test if the value of bar is meow")) def third_method_sync(self, task, http_method, bar): + sleep(CONF.fake.vendor_delay) return True if bar == 'meow' else False @base.passthru(['POST'], require_exclusive_lock=False, description=_("Test if the value of bar is woof")) def fourth_method_shared_lock(self, task, http_method, bar): + sleep(CONF.fake.vendor_delay) return True if bar == 'woof' else False @@ -211,17 +259,21 @@ class FakeManagement(base.ManagementInterface): return [boot_devices.PXE] def set_boot_device(self, task, device, persistent=False): + sleep(CONF.fake.management_delay) if device not in self.get_supported_boot_devices(task): raise exception.InvalidParameterValue(_( "Invalid boot device %s specified.") % device) def get_boot_device(self, task): + sleep(CONF.fake.management_delay) return {'boot_device': boot_devices.PXE, 'persistent': False} def get_sensors_data(self, task): + sleep(CONF.fake.management_delay) return {} def get_supported_indicators(self, task, component=None): + sleep(CONF.fake.management_delay) indicators = { components.CHASSIS: { 'led-0': { @@ -248,6 +300,7 @@ class FakeManagement(base.ManagementInterface): if not component or component == c} def get_indicator_state(self, task, component, indicator): + sleep(CONF.fake.management_delay) indicators = self.get_supported_indicators(task) if component not in indicators: raise exception.InvalidParameterValue(_( @@ -271,6 +324,7 @@ class FakeInspect(base.InspectInterface): pass def inspect_hardware(self, task): + sleep(CONF.fake.inspect_delay) return states.MANAGEABLE @@ -282,9 +336,11 @@ class FakeRAID(base.RAIDInterface): def create_configuration(self, task, create_root_volume=True, create_nonroot_volumes=True): + sleep(CONF.fake.raid_delay) pass def delete_configuration(self, task): + sleep(CONF.fake.raid_delay) pass @@ -302,6 +358,7 @@ class FakeBIOS(base.BIOSInterface): 'to contain a dictionary with name/value pairs'), 'required': True}}) def apply_configuration(self, task, settings): + sleep(CONF.fake.bios_delay) # Note: the implementation of apply_configuration in fake interface # is just for testing purpose, for real driver implementation, please # refer to develop doc at https://docs.openstack.org/ironic/latest/ @@ -328,6 +385,7 @@ class FakeBIOS(base.BIOSInterface): @base.clean_step(priority=0) def factory_reset(self, task): + sleep(CONF.fake.bios_delay) # Note: the implementation of factory_reset in fake interface is # just for testing purpose, for real driver implementation, please # refer to develop doc at https://docs.openstack.org/ironic/latest/ @@ -340,6 +398,7 @@ class FakeBIOS(base.BIOSInterface): @base.clean_step(priority=0) def cache_bios_settings(self, task): + sleep(CONF.fake.bios_delay) # Note: the implementation of cache_bios_settings in fake interface # is just for testing purpose, for real driver implementation, please # refer to develop doc at https://docs.openstack.org/ironic/latest/ @@ -357,9 +416,11 @@ class FakeStorage(base.StorageInterface): return {} def attach_volumes(self, task): + sleep(CONF.fake.storage_delay) pass def detach_volumes(self, task): + sleep(CONF.fake.storage_delay) pass def should_write_image(self, task): @@ -376,7 +437,9 @@ class FakeRescue(base.RescueInterface): pass def rescue(self, task): + sleep(CONF.fake.rescue_delay) return states.RESCUE def unrescue(self, task): + sleep(CONF.fake.rescue_delay) return states.ACTIVE diff --git a/ironic/tests/unit/drivers/test_fake_hardware.py b/ironic/tests/unit/drivers/test_fake_hardware.py index 70460a6a4d..637f52bf95 100644 --- a/ironic/tests/unit/drivers/test_fake_hardware.py +++ b/ironic/tests/unit/drivers/test_fake_hardware.py @@ -17,6 +17,8 @@ """Test class for Fake driver.""" +import time +from unittest import mock from ironic.common import boot_devices from ironic.common import boot_modes @@ -26,6 +28,7 @@ from ironic.common import indicator_states from ironic.common import states from ironic.conductor import task_manager from ironic.drivers import base as driver_base +from ironic.drivers.modules import fake from ironic.tests.unit.db import base as db_base from ironic.tests.unit.db import utils as db_utils @@ -164,3 +167,29 @@ class FakeHardwareTestCase(db_base.DbTestCase): self.assertEqual({}, self.driver.inspect.get_properties()) self.driver.inspect.validate(self.task) self.driver.inspect.inspect_hardware(self.task) + + def test_parse_sleep_range(self): + self.assertEqual((0, 0), fake.parse_sleep_range('0')) + self.assertEqual((0, 0), fake.parse_sleep_range('')) + self.assertEqual((1, 1), fake.parse_sleep_range('1')) + self.assertEqual((1, 10), fake.parse_sleep_range('1,10')) + self.assertEqual((10, 20), fake.parse_sleep_range('10, 20')) + + @mock.patch.object(time, 'sleep', autospec=True) + def test_sleep_zero(self, mock_sleep): + fake.sleep("0") + mock_sleep.assert_not_called() + + @mock.patch.object(time, 'sleep', autospec=True) + def test_sleep_one(self, mock_sleep): + fake.sleep("1") + mock_sleep.assert_called_once_with(1) + + @mock.patch.object(time, 'sleep', autospec=True) + def test_sleep_range(self, mock_sleep): + for i in range(100): + fake.sleep("1,10") + for call in mock_sleep.call_args_list: + v = call[0][0] + self.assertGreaterEqual(v, 1) + self.assertLessEqual(v, 10) diff --git a/releasenotes/notes/fakedelay-7eac23ad8881a736.yaml b/releasenotes/notes/fakedelay-7eac23ad8881a736.yaml new file mode 100644 index 0000000000..fe02d33ffe --- /dev/null +++ b/releasenotes/notes/fakedelay-7eac23ad8881a736.yaml @@ -0,0 +1,8 @@ +--- +features: + - | + There are now configurable random wait times for fake drivers in a new + ironic.conf [fake] section. Each supported driver having one configuration + option controlling the delay. These delays are applied to operations which + typically block in other drivers. This allows more realistic scenarios to + be arranged for performance and functional testing of ironic itself.