Additional condition is comparison when doing config check out of date
This will fix below scenario: Controller-0 is installed and some configs were applied. Config f5d83753-c485-4221-b4aa-e986f5421d6d is the target config and requests boot. After controller-0 is unlocked and reboot.another request with uuid 29c89b7e-51fe-40cb-8700-e22c83ed5fa9 is coming,then the config_target is set to a9c89b7e-51fe-40cb-8700-e22c83ed5fa9; Once 29c89b7e-51fe-40cb-8700-e22c83ed5fa9 was applied,the config should be cleared, but because 29c89b7e-51fe-40cb-8700-e22c83ed5fa9 is not same as a9c89b7e-51fe-40cb-8700-e22c83ed5fa9,the alarm can not be cleared. In this fix, tracking list for reboot config and clear it once config is applied. the out of date config will be clear if reboot config tracking is empty and config_applied w/ reboot config flag is equal config_target Closes-Bug: 1851874 Change-Id: Iabeab338bc3fb4615cefcff9e4ae9402e4216321 Signed-off-by: Sun Austin <austin.sun@intel.com>
This commit is contained in:
parent
89e2975eb8
commit
238b196d3a
|
@ -192,6 +192,9 @@ class ConductorManager(service.PeriodicService):
|
||||||
# Timeouts for adding & removing operations
|
# Timeouts for adding & removing operations
|
||||||
self._pv_op_timeouts = {}
|
self._pv_op_timeouts = {}
|
||||||
self._stor_bck_op_timeouts = {}
|
self._stor_bck_op_timeouts = {}
|
||||||
|
# struct {'host_uuid':[config_uuid_0,config_uuid_1]}
|
||||||
|
# this will track the config w/ reboot request to apply
|
||||||
|
self._host_reboot_config_uuid = {}
|
||||||
|
|
||||||
def start(self):
|
def start(self):
|
||||||
self._start()
|
self._start()
|
||||||
|
@ -7892,9 +7895,38 @@ class ConductorManager(service.PeriodicService):
|
||||||
# We avoid re-raising this as it may brake critical operations after this one
|
# We avoid re-raising this as it may brake critical operations after this one
|
||||||
return constants.CINDER_RESIZE_FAILURE
|
return constants.CINDER_RESIZE_FAILURE
|
||||||
|
|
||||||
|
def _remove_config_from_reboot_config_list(self, ihost_uuid, config_uuid):
|
||||||
|
LOG.info("_remove_config_from_reboot_config_list host: %s,config_uuid: %s" %
|
||||||
|
(ihost_uuid, config_uuid))
|
||||||
|
if ihost_uuid in self._host_reboot_config_uuid:
|
||||||
|
try:
|
||||||
|
self._host_reboot_config_uuid[ihost_uuid].remove(config_uuid)
|
||||||
|
except ValueError:
|
||||||
|
LOG.info("_remove_config_from_reboot_config_list fail"
|
||||||
|
" host:%s", ihost_uuid)
|
||||||
|
pass
|
||||||
|
|
||||||
|
def _clear_config_from_reboot_config_list(self, ihost_uuid):
|
||||||
|
LOG.info("_clear_config_from_reboot_config_list host:%s", ihost_uuid)
|
||||||
|
if ihost_uuid in self._host_reboot_config_uuid:
|
||||||
|
try:
|
||||||
|
del self._host_reboot_config_uuid[ihost_uuid][:]
|
||||||
|
except ValueError:
|
||||||
|
LOG.info("_clear_config_from_reboot_config_list fail"
|
||||||
|
" host: %s", ihost_uuid)
|
||||||
|
pass
|
||||||
|
|
||||||
def _config_out_of_date(self, ihost_obj):
|
def _config_out_of_date(self, ihost_obj):
|
||||||
target = ihost_obj.config_target
|
target = ihost_obj.config_target
|
||||||
applied = ihost_obj.config_applied
|
applied = ihost_obj.config_applied
|
||||||
|
applied_reboot = None
|
||||||
|
if applied is not None:
|
||||||
|
try:
|
||||||
|
applied_reboot = self._config_set_reboot_required(applied)
|
||||||
|
except ValueError:
|
||||||
|
# for worker node, the applied might be 'install'
|
||||||
|
applied_reboot = applied
|
||||||
|
pass
|
||||||
hostname = ihost_obj.hostname
|
hostname = ihost_obj.hostname
|
||||||
|
|
||||||
if not hostname:
|
if not hostname:
|
||||||
|
@ -7905,6 +7937,7 @@ class ConductorManager(service.PeriodicService):
|
||||||
(hostname, applied))
|
(hostname, applied))
|
||||||
return False
|
return False
|
||||||
elif target == applied:
|
elif target == applied:
|
||||||
|
self._clear_config_from_reboot_config_list(ihost_obj.uuid)
|
||||||
if ihost_obj.personality == constants.CONTROLLER:
|
if ihost_obj.personality == constants.CONTROLLER:
|
||||||
|
|
||||||
controller_fs_list = self.dbapi.controller_fs_get_list()
|
controller_fs_list = self.dbapi.controller_fs_get_list()
|
||||||
|
@ -7921,6 +7954,13 @@ class ConductorManager(service.PeriodicService):
|
||||||
LOG.info("%s: iconfig up to date: target %s, applied %s " %
|
LOG.info("%s: iconfig up to date: target %s, applied %s " %
|
||||||
(hostname, target, applied))
|
(hostname, target, applied))
|
||||||
return False
|
return False
|
||||||
|
elif target == applied_reboot:
|
||||||
|
if ihost_obj.uuid in self._host_reboot_config_uuid:
|
||||||
|
if len(self._host_reboot_config_uuid[ihost_obj.uuid]) == 0:
|
||||||
|
return False
|
||||||
|
return True
|
||||||
|
else:
|
||||||
|
return False
|
||||||
else:
|
else:
|
||||||
LOG.warn("%s: iconfig out of date: target %s, applied %s " %
|
LOG.warn("%s: iconfig out of date: target %s, applied %s " %
|
||||||
(hostname, target, applied))
|
(hostname, target, applied))
|
||||||
|
@ -8112,6 +8152,8 @@ class ConductorManager(service.PeriodicService):
|
||||||
@cutils.synchronized(lock_name, external=False)
|
@cutils.synchronized(lock_name, external=False)
|
||||||
def _sync_update_host_config_applied(self,
|
def _sync_update_host_config_applied(self,
|
||||||
context, ihost_obj, config_uuid):
|
context, ihost_obj, config_uuid):
|
||||||
|
self._remove_config_from_reboot_config_list(ihost_obj.uuid,
|
||||||
|
config_uuid)
|
||||||
if ihost_obj.config_applied != config_uuid:
|
if ihost_obj.config_applied != config_uuid:
|
||||||
ihost_obj.config_applied = config_uuid
|
ihost_obj.config_applied = config_uuid
|
||||||
ihost_obj.save(context)
|
ihost_obj.save(context)
|
||||||
|
@ -8159,6 +8201,12 @@ class ConductorManager(service.PeriodicService):
|
||||||
|
|
||||||
for host in hosts:
|
for host in hosts:
|
||||||
if host.personality and host.personality in personalities:
|
if host.personality and host.personality in personalities:
|
||||||
|
if reboot:
|
||||||
|
if host.uuid in self._host_reboot_config_uuid:
|
||||||
|
self._host_reboot_config_uuid[host.uuid].append(config_uuid)
|
||||||
|
else:
|
||||||
|
self._host_reboot_config_uuid[host.uuid] = []
|
||||||
|
self._host_reboot_config_uuid[host.uuid].append(config_uuid)
|
||||||
self._update_host_config_target(context, host, config_uuid)
|
self._update_host_config_target(context, host, config_uuid)
|
||||||
|
|
||||||
LOG.info("_config_update_hosts config_uuid=%s" % config_uuid)
|
LOG.info("_config_update_hosts config_uuid=%s" % config_uuid)
|
||||||
|
|
|
@ -23,11 +23,13 @@
|
||||||
"""Test class for Sysinv ManagerService."""
|
"""Test class for Sysinv ManagerService."""
|
||||||
|
|
||||||
import mock
|
import mock
|
||||||
|
import os.path
|
||||||
import uuid
|
import uuid
|
||||||
|
|
||||||
from sysinv.common import constants
|
from sysinv.common import constants
|
||||||
from sysinv.common import exception
|
from sysinv.common import exception
|
||||||
from sysinv.common import kubernetes
|
from sysinv.common import kubernetes
|
||||||
|
from sysinv.common import utils as cutils
|
||||||
from sysinv.conductor import manager
|
from sysinv.conductor import manager
|
||||||
from sysinv.db import api as dbapi
|
from sysinv.db import api as dbapi
|
||||||
from sysinv.openstack.common import context
|
from sysinv.openstack.common import context
|
||||||
|
@ -97,16 +99,14 @@ class ManagerTestCase(base.DbTestCase):
|
||||||
self.upgrade_downgrade_kube_components_patcher.start()
|
self.upgrade_downgrade_kube_components_patcher.start()
|
||||||
self.addCleanup(self.mock_upgrade_downgrade_kube_components.stop)
|
self.addCleanup(self.mock_upgrade_downgrade_kube_components.stop)
|
||||||
|
|
||||||
self.do_update_alarm_status_patcher = mock.patch.object(
|
self.service.fm_api = mock.Mock()
|
||||||
manager.ConductorManager, '_do_update_alarm_status')
|
self.service.fm_api.set_fault.side_effect = self._raise_alarm
|
||||||
self.mock_do_update_alarm_status = \
|
self.service.fm_api.clear_fault.side_effect = self._clear_alarm
|
||||||
self.do_update_alarm_status_patcher.start()
|
|
||||||
self.addCleanup(self.mock_do_update_alarm_status.stop)
|
|
||||||
|
|
||||||
self.fail_config_apply_runtime_manifest = False
|
self.fail_config_apply_runtime_manifest = False
|
||||||
|
|
||||||
def mock_config_apply_runtime_manifest(obj, context, config_uuid,
|
def mock_config_apply_runtime_manifest(obj, context, config_uuid,
|
||||||
config_dict):
|
config_dict, force=False):
|
||||||
if not self.fail_config_apply_runtime_manifest:
|
if not self.fail_config_apply_runtime_manifest:
|
||||||
# Pretend the config was applied
|
# Pretend the config was applied
|
||||||
if 'host_uuids' in config_dict:
|
if 'host_uuids' in config_dict:
|
||||||
|
@ -184,6 +184,16 @@ class ManagerTestCase(base.DbTestCase):
|
||||||
self.mocked_get_kube_versions.start()
|
self.mocked_get_kube_versions.start()
|
||||||
self.addCleanup(self.mocked_get_kube_versions.stop)
|
self.addCleanup(self.mocked_get_kube_versions.stop)
|
||||||
|
|
||||||
|
self.service._puppet = mock.Mock()
|
||||||
|
self.service._allocate_addresses_for_host = mock.Mock()
|
||||||
|
self.service._update_pxe_config = mock.Mock()
|
||||||
|
self.service._ceph_mon_create = mock.Mock()
|
||||||
|
self.alarm_raised = False
|
||||||
|
|
||||||
|
def tearDown(self):
|
||||||
|
super(ManagerTestCase, self).tearDown()
|
||||||
|
self.upgrade_downgrade_kube_components_patcher.stop()
|
||||||
|
|
||||||
def _create_test_ihost(self, **kwargs):
|
def _create_test_ihost(self, **kwargs):
|
||||||
# ensure the system ID for proper association
|
# ensure the system ID for proper association
|
||||||
kwargs['forisystemid'] = self.system['id']
|
kwargs['forisystemid'] = self.system['id']
|
||||||
|
@ -1012,3 +1022,47 @@ class ManagerTestCase(base.DbTestCase):
|
||||||
# Verify that the host upgrade status was cleared
|
# Verify that the host upgrade status was cleared
|
||||||
updated_host_upgrade = self.dbapi.kube_host_upgrade_get(1)
|
updated_host_upgrade = self.dbapi.kube_host_upgrade_get(1)
|
||||||
self.assertIsNotNone(updated_host_upgrade.status)
|
self.assertIsNotNone(updated_host_upgrade.status)
|
||||||
|
|
||||||
|
def test_configure_out_of_date(self):
|
||||||
|
config_applied = self.service._config_set_reboot_required(uuid.uuid4())
|
||||||
|
config_target = self.service._config_set_reboot_required(uuid.uuid4())
|
||||||
|
ihost = self._create_test_ihost(config_applied=config_applied,
|
||||||
|
config_target=config_target)
|
||||||
|
os.path.isfile = mock.Mock(return_value=True)
|
||||||
|
cutils.is_aio_system = mock.Mock(return_value=True)
|
||||||
|
ihost['mgmt_mac'] = '00:11:22:33:44:55'
|
||||||
|
ihost['mgmt_ip'] = '1.2.3.42'
|
||||||
|
ihost['hostname'] = 'controller-0'
|
||||||
|
ihost['invprovision'] = 'provisioned'
|
||||||
|
ihost['personality'] = 'controller'
|
||||||
|
ihost['administrative'] = 'unlocked'
|
||||||
|
ihost['operational'] = 'available'
|
||||||
|
ihost['availability'] = 'online'
|
||||||
|
ihost['serialid'] = '1234567890abc'
|
||||||
|
ihost['boot_device'] = 'sda'
|
||||||
|
ihost['rootfs_device'] = 'sda'
|
||||||
|
ihost['install_output'] = 'text'
|
||||||
|
ihost['console'] = 'ttyS0,115200'
|
||||||
|
self.service.configure_ihost(self.context, ihost)
|
||||||
|
res = self.dbapi.ihost_get(ihost['uuid'])
|
||||||
|
imsg_dict = {'config_applied': res['config_target']}
|
||||||
|
self.service.iconfig_update_by_ihost(self.context, ihost['uuid'], imsg_dict)
|
||||||
|
self.assertEqual(self.alarm_raised, False)
|
||||||
|
|
||||||
|
personalities = [constants.CONTROLLER]
|
||||||
|
self.service._config_update_hosts(self.context, personalities, reboot=True)
|
||||||
|
res = self.dbapi.ihost_get(ihost['uuid'])
|
||||||
|
|
||||||
|
personalities = [constants.CONTROLLER]
|
||||||
|
self.service._config_update_hosts(self.context, personalities, reboot=False)
|
||||||
|
res = self.dbapi.ihost_get(ihost['uuid'])
|
||||||
|
config_uuid = self.service._config_clear_reboot_required(res['config_target'])
|
||||||
|
imsg_dict = {'config_applied': config_uuid}
|
||||||
|
self.service.iconfig_update_by_ihost(self.context, ihost['uuid'], imsg_dict)
|
||||||
|
self.assertEqual(self.alarm_raised, True)
|
||||||
|
|
||||||
|
def _raise_alarm(self, fault):
|
||||||
|
self.alarm_raised = True
|
||||||
|
|
||||||
|
def _clear_alarm(self, fm_id, fm_instance):
|
||||||
|
self.alarm_raised = False
|
||||||
|
|
Loading…
Reference in New Issue