Fix skipping recovery of the old app for app updates
When app update fails and it is requested that rollback is skipped it still reverts to the old version of the app. Confusion here was created by the names of the building blocks for the update logic (perform_app_rollback, perform_app_recover). In fact it is desired to skip operations that recover the old app. Added the missing logic for path of failed update operation. Now both upgrades and downgrades of the app behave the same. Tested by changing the pvc claim to trigger the armada failure. Closes-Bug: 1928671 Signed-off-by: Dan Voiculeasa <dan.voiculeasa@windriver.com> Change-Id: I6792744257a9cb249e0b1bf99f9b78f3b27859d9
This commit is contained in:
Dan Voiculeasa
parent
ed967ad81c
commit
e4bc9dc602
|
|
@ -1661,7 +1661,7 @@ APP_METADATA_DESIRED_STATES = 'desired_states'
|
|||
APP_METADATA_FORBIDDEN_MANUAL_OPERATIONS = 'forbidden_manual_operations'
|
||||
APP_METADATA_ORDERED_APPS = 'ordered_apps'
|
||||
APP_METADATA_UPGRADES = 'upgrades'
|
||||
APP_METADATA_UPDATE_FAILURE_NO_ROLLBACK = 'update_failure_no_rollback'
|
||||
APP_METADATA_UPDATE_FAILURE_SKIP_RECOVERY = 'update_failure_no_rollback'
|
||||
APP_METADATA_FROM_VERSIONS = 'from_versions'
|
||||
APP_METADATA_SUPPORTED_K8S_VERSION = 'supported_k8s_version'
|
||||
APP_METADATA_SUPPORTED_RELEASES = 'supported_releases'
|
||||
|
|
@ -1732,6 +1732,9 @@ APP_PROGRESS_RECOVER_COMPLETED = 'Application recover to version {} completed. '
|
|||
APP_PROGRESS_CLEANUP_FAILED = 'Application files/helm release cleanup for version {} failed.'
|
||||
APP_PROGRESS_RECOVER_IN_PROGRESS = 'recovering version {} '
|
||||
APP_PROGRESS_RECOVER_CHARTS = 'recovering helm charts'
|
||||
APP_PROGRESS_UPDATE_FAILED_SKIP_RECOVERY = "Application {} update from " \
|
||||
"version {} to version {} failed and recovery skipped " \
|
||||
"because skip_recovery was requested."
|
||||
|
||||
# Auto-recovery limits
|
||||
APP_AUTO_RECOVERY_MAX_COUNT = 5
|
||||
|
|
|
|||
|
|
@ -2120,13 +2120,13 @@ def find_metadata_file(path, metadata_file, upgrade_from_release=None):
|
|||
|
||||
if upgrades:
|
||||
try:
|
||||
no_rollback = \
|
||||
upgrades[constants.APP_METADATA_UPDATE_FAILURE_NO_ROLLBACK]
|
||||
if not is_valid_boolstr(no_rollback):
|
||||
skip_recovery = \
|
||||
upgrades[constants.APP_METADATA_UPDATE_FAILURE_SKIP_RECOVERY]
|
||||
if not is_valid_boolstr(skip_recovery):
|
||||
raise exception.SysinvException(_(
|
||||
"Invalid {}: {} expected value is a boolean string."
|
||||
"".format(metadata_file,
|
||||
constants.APP_METADATA_UPDATE_FAILURE_NO_ROLLBACK)))
|
||||
constants.APP_METADATA_UPDATE_FAILURE_SKIP_RECOVERY)))
|
||||
except KeyError:
|
||||
pass
|
||||
|
||||
|
|
|
|||
|
|
@ -1644,7 +1644,7 @@ class AppOperator(object):
|
|||
LOG.error("Application %s recover to version %s aborted!"
|
||||
% (old_app.name, old_app.version))
|
||||
|
||||
def _perform_app_rollback(self, from_app, to_app, no_rollback):
|
||||
def _perform_app_rollback(self, from_app, to_app):
|
||||
"""Perform application rollback request
|
||||
|
||||
This method invokes Armada to rollback the application releases to
|
||||
|
|
@ -1653,18 +1653,10 @@ class AppOperator(object):
|
|||
|
||||
:param from_app: application object that application updating from
|
||||
:param to_app: application object that application updating to
|
||||
:param no_rollback: boolean: whether application should skip rollback
|
||||
:return boolean: whether application rollback was successful
|
||||
"""
|
||||
|
||||
LOG.info("Application %s (%s) rollback started." % (to_app.name, to_app.version))
|
||||
if no_rollback:
|
||||
LOG.info("Application %s (%s) has configured no_rollback %s, "
|
||||
"rollback skipped.",
|
||||
to_app.name, to_app.version, no_rollback)
|
||||
# Assume application not aborted. The subsequent success path will
|
||||
# cleanup the from_app.
|
||||
return True
|
||||
|
||||
try:
|
||||
if AppOperator.is_app_aborted(to_app.name):
|
||||
|
|
@ -2484,6 +2476,11 @@ class AppOperator(object):
|
|||
|
||||
self._update_app_status(to_app, constants.APP_UPDATE_IN_PROGRESS)
|
||||
|
||||
# Get the skip_recovery flag from app metadata
|
||||
keys = [constants.APP_METADATA_UPGRADES,
|
||||
constants.APP_METADATA_UPDATE_FAILURE_SKIP_RECOVERY]
|
||||
skip_recovery = bool(strtobool(str(self._get_metadata_value(to_app, keys, False))))
|
||||
|
||||
result = False
|
||||
if operation == constants.APP_APPLY_OP:
|
||||
reuse_overrides = \
|
||||
|
|
@ -2517,12 +2514,23 @@ class AppOperator(object):
|
|||
self._plugins.activate_plugins(to_app)
|
||||
|
||||
# lifecycle hooks not used in perform_app_rollback
|
||||
keys = [constants.APP_METADATA_UPGRADES,
|
||||
constants.APP_METADATA_UPDATE_FAILURE_NO_ROLLBACK]
|
||||
no_rollback = bool(strtobool(str(self._get_metadata_value(to_app, keys, False))))
|
||||
result = self._perform_app_rollback(from_app, to_app, no_rollback)
|
||||
result = self._perform_app_rollback(from_app, to_app)
|
||||
|
||||
if not result:
|
||||
operation_successful = result
|
||||
|
||||
# If operation failed consider doing the app recovery
|
||||
do_recovery = not operation_successful
|
||||
|
||||
# Here the app operation failed (do_recovery is True)
|
||||
# but skip_recovery requested.
|
||||
if skip_recovery and do_recovery:
|
||||
LOG.info("Application %s (%s) has configured skip_recovery %s"
|
||||
", recovery skipped.",
|
||||
to_app.name, to_app.version, skip_recovery)
|
||||
do_recovery = False
|
||||
|
||||
# If recovery is requested stop the flow of execution here
|
||||
if do_recovery:
|
||||
LOG.error("Application %s update from version %s to version "
|
||||
"%s aborted." % (to_app.name, from_app.version, to_app.version))
|
||||
|
||||
|
|
@ -2532,7 +2540,7 @@ class AppOperator(object):
|
|||
self._update_app_status(to_app, constants.APP_UPDATE_IN_PROGRESS,
|
||||
"cleanup application version {}".format(from_app.version))
|
||||
|
||||
# App apply/rollback succeeded
|
||||
# App apply/rollback succeeded or it failed but skip_recovery was set
|
||||
# Starting cleanup old application
|
||||
from_app.charts = self._get_list_of_charts(from_app.sync_armada_mfile)
|
||||
to_app_charts = [c.release for c in to_app.charts]
|
||||
|
|
@ -2550,12 +2558,26 @@ class AppOperator(object):
|
|||
self._utils._patch_report_app_dependencies(
|
||||
from_app.name + '-' + from_app.version)
|
||||
|
||||
self._update_app_status(
|
||||
to_app, constants.APP_APPLY_SUCCESS,
|
||||
constants.APP_PROGRESS_UPDATE_COMPLETED.format(from_app.version,
|
||||
to_app.version))
|
||||
LOG.info("Application %s update from version %s to version "
|
||||
"%s completed." % (to_app.name, from_app.version, to_app.version))
|
||||
# The initial operation for to_app is successful
|
||||
if operation_successful:
|
||||
self._update_app_status(
|
||||
to_app, constants.APP_APPLY_SUCCESS,
|
||||
constants.APP_PROGRESS_UPDATE_COMPLETED.format(
|
||||
from_app.version, to_app.version))
|
||||
LOG.info("Application %s update from version %s to version "
|
||||
"%s completed." % (to_app.name, from_app.version, to_app.version))
|
||||
|
||||
# The initial operation for to_app failed
|
||||
# This is reached here only when skip_recovery is requested
|
||||
# Need to inform the user
|
||||
else:
|
||||
message = \
|
||||
constants.APP_PROGRESS_UPDATE_FAILED_SKIP_RECOVERY.format(
|
||||
to_app.name, from_app.version, to_app.version)
|
||||
self._update_app_status(
|
||||
to_app, constants.APP_APPLY_FAILURE, message)
|
||||
LOG.info(message)
|
||||
|
||||
except (exception.IncompatibleKubeVersion,
|
||||
exception.KubeAppUploadFailure,
|
||||
exception.KubeAppApplyFailure,
|
||||
|
|
|
|||
Loading…
Reference in New Issue