From 1c751c594f2644b31a13fa7e23c1f094f1236174 Mon Sep 17 00:00:00 2001 From: Kobi Samoray Date: Thu, 29 Apr 2021 18:44:12 +0300 Subject: [PATCH] Octavia LB status admin util When Octavia loadbalancer hangs in PENDING status, it becomes immutable and cannot be recovered or deleted in any way. The following admin util commmand updates the loadbalancer status to ERROR so it can be deleted successfully, but sending an RPC message to Octavia driver agent service. Change-Id: I7ee1ba445ab4526ef8b2d271574319114a31d631 --- doc/source/admin_util.rst | 18 +++++++ .../services/lbaas/octavia/octavia_driver.py | 4 +- .../shell/admin/plugins/common/constants.py | 1 + .../admin/plugins/common/loadbalancers.py | 52 +++++++++++++++++++ .../plugins/nsxp/resources/loadbalancers.py | 25 +++++++++ .../plugins/nsxv/resources/loadbalancers.py | 25 +++++++++ .../plugins/nsxv3/resources/loadbalancers.py | 25 +++++++++ vmware_nsx/shell/resources.py | 7 +++ 8 files changed, 155 insertions(+), 2 deletions(-) create mode 100644 vmware_nsx/shell/admin/plugins/common/loadbalancers.py create mode 100644 vmware_nsx/shell/admin/plugins/nsxp/resources/loadbalancers.py create mode 100644 vmware_nsx/shell/admin/plugins/nsxv/resources/loadbalancers.py create mode 100644 vmware_nsx/shell/admin/plugins/nsxv3/resources/loadbalancers.py diff --git a/doc/source/admin_util.rst b/doc/source/admin_util.rst index 099584170a..56f9d92a88 100644 --- a/doc/source/admin_util.rst +++ b/doc/source/admin_util.rst @@ -351,6 +351,13 @@ Config nsxadmin -r config -o validate +Loadbalancers +~~~~~~~~~~~~~ + +- Set an Octavia loadbalancer id to ERROR status. Useful while the loadbalancer hangs in PENDING status:: + + nsxadmin -r loadbalancers -o set-status-error --property loadbalancer-id= + NSX-T Plugin ------------ @@ -628,6 +635,13 @@ T2P migration nsxadmin -r nsx-migrate-t2p -o clean-all +Loadbalancers +~~~~~~~~~~~~~ + +- Set an Octavia loadbalancer id to ERROR status. Useful while the loadbalancer hangs in PENDING status:: + + nsxadmin -r loadbalancers -o set-status-error --property loadbalancer-id= + NSXtvd Plugin ------------- @@ -694,6 +708,10 @@ NSX Policy Plugin nsxadmin -r nsx-migrate-v2t -o validate --property ext-net= --property ext-cidr= +- Set an Octavia loadbalancer id to ERROR status. Useful while the loadbalancer hangs in PENDING status:: + + nsxadmin -r loadbalancers -o set-status-error --property loadbalancer-id= + Client Certificate ~~~~~~~~~~~~~~~~~~ diff --git a/vmware_nsx/services/lbaas/octavia/octavia_driver.py b/vmware_nsx/services/lbaas/octavia/octavia_driver.py index deba6f98be..38f47ab645 100644 --- a/vmware_nsx/services/lbaas/octavia/octavia_driver.py +++ b/vmware_nsx/services/lbaas/octavia/octavia_driver.py @@ -44,7 +44,7 @@ RPC_SERVER = None def _log_before_retry(retry_state): - if retry_state.attempt_number > 0: + if retry_state.attempt_number > 1: LOG.warning("Retrying the call to _update_loadbalancer_status due to " "timeout") @@ -704,7 +704,7 @@ class NSXOctaviaDriverEndpoint(driver_lib.DriverLibrary): retry=tenacity.retry_if_exception_type( exceptions.UpdateStatusError)) def _update_loadbalancer_status(self, status): - super(NSXOctaviaDriverEndpoint, + return super(NSXOctaviaDriverEndpoint, self).update_loadbalancer_status(status) @log_helpers.log_method_call diff --git a/vmware_nsx/shell/admin/plugins/common/constants.py b/vmware_nsx/shell/admin/plugins/common/constants.py index ca3be2f657..9bd762329b 100644 --- a/vmware_nsx/shell/admin/plugins/common/constants.py +++ b/vmware_nsx/shell/admin/plugins/common/constants.py @@ -37,6 +37,7 @@ CONFIG = 'config' ORPHANED_NETWORKS = 'orphaned-networks' ORPHANED_ROUTERS = 'orphaned-routers' SYSTEM = 'system' +LOADBALANCERS = 'loadbalancers' # NSXV3 only Resource Constants PORTS = 'ports' diff --git a/vmware_nsx/shell/admin/plugins/common/loadbalancers.py b/vmware_nsx/shell/admin/plugins/common/loadbalancers.py new file mode 100644 index 0000000000..419a9bca83 --- /dev/null +++ b/vmware_nsx/shell/admin/plugins/common/loadbalancers.py @@ -0,0 +1,52 @@ +# Copyright 2021 VMware, Inc. +# All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_config import cfg +from oslo_log import log as logging +import oslo_messaging as messaging + +from vmware_nsx.services.lbaas.octavia import constants as octavia_const +from vmware_nsx.shell.admin.plugins.common import utils as admin_utils + +LOG = logging.getLogger(__name__) + + +@admin_utils.output_header +def set_loadbalancer_status_error(resource, event, trigger, **kwargs): + usage_msg = ("Loadbalancer id should be specified with " + "--property loadbalancer-id=") + if not kwargs.get('property'): + LOG.error(usage_msg) + return + properties = admin_utils.parse_multi_keyval_opt(kwargs['property']) + lb_id = properties.get('loadbalancer-id') + if not lb_id: + LOG.error("Need to specify loadbalancer-id. " + "Add --property loadbalancer-id=") + return + + status_dict = { + octavia_const.LOADBALANCERS: [{ + 'id': lb_id, + octavia_const.PROVISIONING_STATUS: octavia_const.ERROR}]} + kw = {'status': status_dict} + + topic = octavia_const.DRIVER_TO_OCTAVIA_TOPIC + transport = messaging.get_rpc_transport(cfg.CONF) + target = messaging.Target(topic=topic, exchange="common", + namespace='control', fanout=False, + version='1.0') + client = messaging.RPCClient(transport, target) + client.cast({}, 'update_loadbalancer_status', **kw) diff --git a/vmware_nsx/shell/admin/plugins/nsxp/resources/loadbalancers.py b/vmware_nsx/shell/admin/plugins/nsxp/resources/loadbalancers.py new file mode 100644 index 0000000000..ae713ef0f0 --- /dev/null +++ b/vmware_nsx/shell/admin/plugins/nsxp/resources/loadbalancers.py @@ -0,0 +1,25 @@ +# Copyright 2021 VMware, Inc. +# All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from neutron_lib.callbacks import registry + +from vmware_nsx.shell.admin.plugins.common import constants +from vmware_nsx.shell.admin.plugins.common import loadbalancers +from vmware_nsx.shell import resources as shell + + +registry.subscribe(loadbalancers.set_loadbalancer_status_error, + constants.LOADBALANCERS, + shell.Operations.SET_STATUS_ERROR.value) diff --git a/vmware_nsx/shell/admin/plugins/nsxv/resources/loadbalancers.py b/vmware_nsx/shell/admin/plugins/nsxv/resources/loadbalancers.py new file mode 100644 index 0000000000..ae713ef0f0 --- /dev/null +++ b/vmware_nsx/shell/admin/plugins/nsxv/resources/loadbalancers.py @@ -0,0 +1,25 @@ +# Copyright 2021 VMware, Inc. +# All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from neutron_lib.callbacks import registry + +from vmware_nsx.shell.admin.plugins.common import constants +from vmware_nsx.shell.admin.plugins.common import loadbalancers +from vmware_nsx.shell import resources as shell + + +registry.subscribe(loadbalancers.set_loadbalancer_status_error, + constants.LOADBALANCERS, + shell.Operations.SET_STATUS_ERROR.value) diff --git a/vmware_nsx/shell/admin/plugins/nsxv3/resources/loadbalancers.py b/vmware_nsx/shell/admin/plugins/nsxv3/resources/loadbalancers.py new file mode 100644 index 0000000000..ae713ef0f0 --- /dev/null +++ b/vmware_nsx/shell/admin/plugins/nsxv3/resources/loadbalancers.py @@ -0,0 +1,25 @@ +# Copyright 2021 VMware, Inc. +# All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from neutron_lib.callbacks import registry + +from vmware_nsx.shell.admin.plugins.common import constants +from vmware_nsx.shell.admin.plugins.common import loadbalancers +from vmware_nsx.shell import resources as shell + + +registry.subscribe(loadbalancers.set_loadbalancer_status_error, + constants.LOADBALANCERS, + shell.Operations.SET_STATUS_ERROR.value) diff --git a/vmware_nsx/shell/resources.py b/vmware_nsx/shell/resources.py index d243d55e52..33161bbe46 100644 --- a/vmware_nsx/shell/resources.py +++ b/vmware_nsx/shell/resources.py @@ -77,6 +77,7 @@ class Operations(enum.Enum): REUSE = 'reuse' UPDATE_TIER0 = 'update-tier0' UPDATE_FIREWALL_MATCH = 'update-nat-firewall-match' + SET_STATUS_ERROR = 'set-status-error' ops = [op.value for op in Operations] @@ -162,6 +163,8 @@ nsxv3_resources = { [Operations.IMPORT.value, Operations.CLEAN_ALL.value, Operations.VALIDATE.value]), + constants.LOADBALANCERS: Resource(constants.LOADBALANCERS, + [Operations.SET_STATUS_ERROR.value]), } # Add supported NSX-V resources in this dictionary @@ -258,6 +261,8 @@ nsxv_resources = { [Operations.VALIDATE.value]), constants.PORTS: Resource(constants.PORTS, [Operations.LIST.value]), + constants.LOADBALANCERS: Resource(constants.LOADBALANCERS, + [Operations.SET_STATUS_ERROR.value]), } @@ -296,6 +301,8 @@ nsxp_resources = { [Operations.CLEAN_ALL.value, Operations.VALIDATE.value, Operations.NSX_REDISTRIBUTE.value]), + constants.LOADBALANCERS: Resource(constants.LOADBALANCERS, + [Operations.SET_STATUS_ERROR.value]), } nsxv3_resources_names = list(nsxv3_resources.keys())