Enable config_update_puppet retry for deferred config
A deferred config typically occurs shortly after initialization, when the sysinv agent is ready to handle config requests. In rare cases, an exception may occur when attempting to generate hieradata, which depends on external services (e.g. k8s, ceph, etc). This updates the puppet hieradata update to handle potential exception and retry in the case of deferred runtime config. TestPlan: PASSED: bootstrap and host-unlock with these changes PASSED: duplex system swact with these changes PASSED: runtime manifest apply and file update apply PASSED: deferred runtime manifest retries on hieradata update exception PASSED: deferred runtime file retries on hieradata update exception Closes-Bug: 2011838 Signed-off-by: John Kung <john.kung@windriver.com> Change-Id: I77b2fd084165645abb7d68d436696c183a91ec8a
This commit is contained in:
parent
6e832b4707
commit
3db1864d1c
|
@ -11383,13 +11383,9 @@ class ConductorManager(service.PeriodicService):
|
|||
self._host_deferred_runtime_config)
|
||||
return False
|
||||
|
||||
# the config will be processed so remove from deferred list if it is a
|
||||
# deferred one.
|
||||
if deferred_config:
|
||||
self._host_deferred_runtime_config.remove(deferred_config)
|
||||
|
||||
# Ensure hiera data is updated prior to active apply.
|
||||
self._config_update_puppet(config_uuid, config_dict)
|
||||
if not self._try_config_update_puppet(
|
||||
config_uuid, config_dict, deferred_config):
|
||||
return False
|
||||
|
||||
rpcapi = agent_rpcapi.AgentAPI()
|
||||
try:
|
||||
|
@ -11488,6 +11484,41 @@ class ConductorManager(service.PeriodicService):
|
|||
break
|
||||
return config
|
||||
|
||||
def _try_config_update_puppet(
|
||||
self, config_uuid, config_dict,
|
||||
deferred_config=None, host_uuids=None, force=False):
|
||||
"""Attempt the config puppet hierdata update.
|
||||
|
||||
In the case of a deferred config, the puppet update can be
|
||||
asynchronously retried on exception.
|
||||
In the case of synchronous (non-deferred) config, exceptions
|
||||
are raised.
|
||||
"""
|
||||
|
||||
# the config will be processed so remove from deferred list if it is a
|
||||
# deferred one.
|
||||
if deferred_config:
|
||||
self._host_deferred_runtime_config.remove(deferred_config)
|
||||
|
||||
# Update hiera data for all hosts prior to runtime apply if host_uuid
|
||||
# is not set. If host_uuids is set only update hiera data for those hosts.
|
||||
try:
|
||||
self._config_update_puppet(config_uuid,
|
||||
config_dict,
|
||||
host_uuids=host_uuids,
|
||||
force=force)
|
||||
except Exception as e:
|
||||
LOG.exception("_config_update_puppet %s" % e)
|
||||
if deferred_config:
|
||||
self._host_deferred_runtime_config.append(deferred_config)
|
||||
LOG.warn("deferred update runtime config %s exception. Retry." %
|
||||
deferred_config)
|
||||
return False
|
||||
else:
|
||||
raise
|
||||
|
||||
return True
|
||||
|
||||
def _config_apply_runtime_manifest(self,
|
||||
context,
|
||||
config_uuid,
|
||||
|
@ -11557,17 +11588,9 @@ class ConductorManager(service.PeriodicService):
|
|||
self._host_deferred_runtime_config)
|
||||
return
|
||||
|
||||
# the config will be processed so remove from deferred list if it is a
|
||||
# deferred one.
|
||||
if deferred_config:
|
||||
self._host_deferred_runtime_config.remove(deferred_config)
|
||||
|
||||
# Update hiera data for all hosts prior to runtime apply if host_uuid
|
||||
# is not set. If host_uuids is set only update hiera data for those hosts.
|
||||
self._config_update_puppet(config_uuid,
|
||||
config_dict,
|
||||
host_uuids=host_uuids,
|
||||
force=force)
|
||||
if not self._try_config_update_puppet(
|
||||
config_uuid, config_dict, deferred_config, host_uuids, force):
|
||||
return
|
||||
|
||||
self.evaluate_apps_reapply(context, trigger={'type': constants.APP_EVALUATE_REAPPLY_TYPE_RUNTIME_APPLY_PUPPET})
|
||||
|
||||
|
|
Loading…
Reference in New Issue