From a3310458a3ce7fd1f41f45a9c15eff0785d44df9 Mon Sep 17 00:00:00 2001 From: Daniel Caires Date: Thu, 13 Mar 2025 14:04:55 -0300 Subject: [PATCH] Remove MariaDB Helmrelease if update fails MariaDB does not officially supports downgrade between major versions [1]. When the update fails and the application framework tries to roll-back the STX-O deployment, the recovery fails in the MariaDB Helm chart. The solution for this is first, when the update fails it will trigger the deletion of the updated MariaDB Helm release. The other step will be to have the persistentvolume to the old MariaDB backed up at the beggining of the update, and if the update fails, it will trigger a function that will remove the persistentVolume from the new MariaDB and replace with the old one. this work is going to to be maped to the task 51826 in the storyboard: https://storyboard.openstack.org/#!/story/2011262 This review aims to address the first part of the solution, where the MariaDB Helmrelease is deleted before starting the recovery process. It was necessary to create a function do delete the Helmrelease as it was noted that the new MariaDB could be redeployed by FluxCD reconciliation, before the old one, if it was opted to only uninstall the Helmrelease. Test Plan: PASS - Build python3-k8sapp-openstack PASS - Build WRO PASS - Force an update fail and check if MariaDB Helmrelease is deleted Relates-To: https://review.opendev.org/c/starlingx/openstack-armada-app/+/945094/6 Both reviews on the relation chain should be merged together as it functions as one to MariaDB be able to rollback in case of failed update. [1] - https://mariadb.com/kb/en/downgrading-between-major-versions-of-mariadb/ Story: 2011262 Task: 51787 Change-Id: I60b009c805673d00d54b1c4ff8c868cd8f2f0190 Signed-off-by: Daniel Caires --- .../lifecycle/lifecycle_openstack.py | 11 ++++++- .../k8sapp_openstack/utils.py | 30 +++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/python3-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/lifecycle/lifecycle_openstack.py b/python3-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/lifecycle/lifecycle_openstack.py index b41be88b..061ce528 100644 --- a/python3-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/lifecycle/lifecycle_openstack.py +++ b/python3-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/lifecycle/lifecycle_openstack.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2023 Wind River Systems, Inc. +# Copyright (c) 2023-2025 Wind River Systems, Inc. # # SPDX-License-Identifier: Apache-2.0 # @@ -453,3 +453,12 @@ class OpenstackAppLifecycleOperator(base.AppLifecycleOperator): status = helm_utils.delete_helm_release( release=release_failed, namespace=app_constants.HELM_NS_OPENSTACK) LOG.info(status) + + # Downgrading is not officially supported for MariaDB: + # https://mariadb.com/kb/en/downgrading-between-major-versions-of-mariadb/ + # Because of that, we need to delete the Helmrelease for the new MariaDB + # before deploying the old one. + app_utils.delete_kubernetes_resource( + resource_type='helmrelease', + resource_name='mariadb' + ) diff --git a/python3-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/utils.py b/python3-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/utils.py index c858403a..c9ef02b3 100644 --- a/python3-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/utils.py +++ b/python3-k8sapp-openstack/k8sapp_openstack/k8sapp_openstack/utils.py @@ -606,3 +606,33 @@ def update_helmrelease(release, patch): LOG.error(f"Failed to update helmrelease: {release}, with error: {e}") except Exception as e: LOG.error(f"Unexpected error while updating helmrelease: {e}") + + +def delete_kubernetes_resource(resource_type, resource_name): + """ + Delete a Kubernetes resource. + + Args: + resource_type (str): The type of the Kubernetes resource. + resource_name (str): The name of the Kubernetes resource. + """ + cmd = [ + "kubectl", "--kubeconfig", kubernetes.KUBERNETES_ADMIN_CONF, + "delete", resource_type, resource_name, + "-n", app_constants.HELM_NS_OPENSTACK + ] + + try: + process = subprocess.run( + args=cmd, + capture_output=True, + text=True, + shell=False) + + LOG.info(f"Stdout: {process.stdout}") + LOG.info(f"Stderr: {process.stderr}") + process.check_returncode() + except KubeApiException as e: + LOG.error(f"Failed to delete {resource_type}: {resource_name}, with error: {e}") + except Exception as e: + LOG.error(f"Unexpected error while deleting {resource_type}: {e}")