Enable config_update_puppet retry for deferred config

A deferred config typically occurs shortly after initialization, when
the sysinv agent is ready to handle config requests.  In rare cases,
an exception may occur when attempting to generate hieradata, which
depends on external services (e.g. k8s, ceph, etc).

This updates the puppet hieradata update to handle potential exception
and retry in the case of deferred runtime config.

TestPlan:

PASSED: bootstrap and host-unlock with these changes
PASSED: duplex system swact with these changes
PASSED: runtime manifest apply and file update apply
PASSED: deferred runtime manifest retries on hieradata update exception
PASSED: deferred runtime file retries on hieradata update exception

Closes-Bug: 2011838
Signed-off-by: John Kung <john.kung@windriver.com>
Change-Id: I77b2fd084165645abb7d68d436696c183a91ec8a
This commit is contained in:
John Kung 2023-03-16 10:49:43 -04:00
parent 6e832b4707
commit 3db1864d1c
1 changed files with 41 additions and 18 deletions

View File

@ -11383,13 +11383,9 @@ class ConductorManager(service.PeriodicService):
self._host_deferred_runtime_config)
return False
# the config will be processed so remove from deferred list if it is a
# deferred one.
if deferred_config:
self._host_deferred_runtime_config.remove(deferred_config)
# Ensure hiera data is updated prior to active apply.
self._config_update_puppet(config_uuid, config_dict)
if not self._try_config_update_puppet(
config_uuid, config_dict, deferred_config):
return False
rpcapi = agent_rpcapi.AgentAPI()
try:
@ -11488,6 +11484,41 @@ class ConductorManager(service.PeriodicService):
break
return config
def _try_config_update_puppet(
self, config_uuid, config_dict,
deferred_config=None, host_uuids=None, force=False):
"""Attempt the config puppet hierdata update.
In the case of a deferred config, the puppet update can be
asynchronously retried on exception.
In the case of synchronous (non-deferred) config, exceptions
are raised.
"""
# the config will be processed so remove from deferred list if it is a
# deferred one.
if deferred_config:
self._host_deferred_runtime_config.remove(deferred_config)
# Update hiera data for all hosts prior to runtime apply if host_uuid
# is not set. If host_uuids is set only update hiera data for those hosts.
try:
self._config_update_puppet(config_uuid,
config_dict,
host_uuids=host_uuids,
force=force)
except Exception as e:
LOG.exception("_config_update_puppet %s" % e)
if deferred_config:
self._host_deferred_runtime_config.append(deferred_config)
LOG.warn("deferred update runtime config %s exception. Retry." %
deferred_config)
return False
else:
raise
return True
def _config_apply_runtime_manifest(self,
context,
config_uuid,
@ -11557,17 +11588,9 @@ class ConductorManager(service.PeriodicService):
self._host_deferred_runtime_config)
return
# the config will be processed so remove from deferred list if it is a
# deferred one.
if deferred_config:
self._host_deferred_runtime_config.remove(deferred_config)
# Update hiera data for all hosts prior to runtime apply if host_uuid
# is not set. If host_uuids is set only update hiera data for those hosts.
self._config_update_puppet(config_uuid,
config_dict,
host_uuids=host_uuids,
force=force)
if not self._try_config_update_puppet(
config_uuid, config_dict, deferred_config, host_uuids, force):
return
self.evaluate_apps_reapply(context, trigger={'type': constants.APP_EVALUATE_REAPPLY_TYPE_RUNTIME_APPLY_PUPPET})