From 329b806222c21d611dfbec7ff0563f4f3514e95d Mon Sep 17 00:00:00 2001 From: Kobi Samoray Date: Thu, 29 Apr 2021 18:44:12 +0300 Subject: [PATCH] Octavia LB status admin util When Octavia loadbalancer hangs in PENDING status, it becomes immutable and cannot be recovered or deleted in any way. The following admin util commmand updates the loadbalancer status to ERROR so it can be deleted successfully, but sending an RPC message to Octavia driver agent service. Change-Id: I7ee1ba445ab4526ef8b2d271574319114a31d631 --- doc/source/admin_util.rst | 18 +++++++ .../services/lbaas/octavia/octavia_driver.py | 4 +- .../shell/admin/plugins/common/constants.py | 1 + .../admin/plugins/common/loadbalancers.py | 52 +++++++++++++++++++ .../plugins/nsxp/resources/loadbalancers.py | 25 +++++++++ .../plugins/nsxv/resources/loadbalancers.py | 25 +++++++++ .../plugins/nsxv3/resources/loadbalancers.py | 25 +++++++++ vmware_nsx/shell/resources.py | 7 +++ 8 files changed, 155 insertions(+), 2 deletions(-) create mode 100644 vmware_nsx/shell/admin/plugins/common/loadbalancers.py create mode 100644 vmware_nsx/shell/admin/plugins/nsxp/resources/loadbalancers.py create mode 100644 vmware_nsx/shell/admin/plugins/nsxv/resources/loadbalancers.py create mode 100644 vmware_nsx/shell/admin/plugins/nsxv3/resources/loadbalancers.py diff --git a/doc/source/admin_util.rst b/doc/source/admin_util.rst index 099584170a..56f9d92a88 100644 --- a/doc/source/admin_util.rst +++ b/doc/source/admin_util.rst @@ -351,6 +351,13 @@ Config nsxadmin -r config -o validate +Loadbalancers +~~~~~~~~~~~~~ + +- Set an Octavia loadbalancer id to ERROR status. Useful while the loadbalancer hangs in PENDING status:: + + nsxadmin -r loadbalancers -o set-status-error --property loadbalancer-id= + NSX-T Plugin ------------ @@ -628,6 +635,13 @@ T2P migration nsxadmin -r nsx-migrate-t2p -o clean-all +Loadbalancers +~~~~~~~~~~~~~ + +- Set an Octavia loadbalancer id to ERROR status. Useful while the loadbalancer hangs in PENDING status:: + + nsxadmin -r loadbalancers -o set-status-error --property loadbalancer-id= + NSXtvd Plugin ------------- @@ -694,6 +708,10 @@ NSX Policy Plugin nsxadmin -r nsx-migrate-v2t -o validate --property ext-net= --property ext-cidr= +- Set an Octavia loadbalancer id to ERROR status. Useful while the loadbalancer hangs in PENDING status:: + + nsxadmin -r loadbalancers -o set-status-error --property loadbalancer-id= + Client Certificate ~~~~~~~~~~~~~~~~~~ diff --git a/vmware_nsx/services/lbaas/octavia/octavia_driver.py b/vmware_nsx/services/lbaas/octavia/octavia_driver.py index deb85747fb..a79c5954bb 100644 --- a/vmware_nsx/services/lbaas/octavia/octavia_driver.py +++ b/vmware_nsx/services/lbaas/octavia/octavia_driver.py @@ -44,7 +44,7 @@ RPC_SERVER = None def _log_before_retry(retry_state): - if retry_state.attempt_number > 0: + if retry_state.attempt_number > 1: LOG.warning("Retrying the call to _update_loadbalancer_status due to " "timeout") @@ -704,7 +704,7 @@ class NSXOctaviaDriverEndpoint(driver_lib.DriverLibrary): retry=tenacity.retry_if_exception_type( exceptions.UpdateStatusError)) def _update_loadbalancer_status(self, status): - super(NSXOctaviaDriverEndpoint, + return super(NSXOctaviaDriverEndpoint, self).update_loadbalancer_status(status) @log_helpers.log_method_call diff --git a/vmware_nsx/shell/admin/plugins/common/constants.py b/vmware_nsx/shell/admin/plugins/common/constants.py index ca3be2f657..9bd762329b 100644 --- a/vmware_nsx/shell/admin/plugins/common/constants.py +++ b/vmware_nsx/shell/admin/plugins/common/constants.py @@ -37,6 +37,7 @@ CONFIG = 'config' ORPHANED_NETWORKS = 'orphaned-networks' ORPHANED_ROUTERS = 'orphaned-routers' SYSTEM = 'system' +LOADBALANCERS = 'loadbalancers' # NSXV3 only Resource Constants PORTS = 'ports' diff --git a/vmware_nsx/shell/admin/plugins/common/loadbalancers.py b/vmware_nsx/shell/admin/plugins/common/loadbalancers.py new file mode 100644 index 0000000000..419a9bca83 --- /dev/null +++ b/vmware_nsx/shell/admin/plugins/common/loadbalancers.py @@ -0,0 +1,52 @@ +# Copyright 2021 VMware, Inc. +# All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_config import cfg +from oslo_log import log as logging +import oslo_messaging as messaging + +from vmware_nsx.services.lbaas.octavia import constants as octavia_const +from vmware_nsx.shell.admin.plugins.common import utils as admin_utils + +LOG = logging.getLogger(__name__) + + +@admin_utils.output_header +def set_loadbalancer_status_error(resource, event, trigger, **kwargs): + usage_msg = ("Loadbalancer id should be specified with " + "--property loadbalancer-id=") + if not kwargs.get('property'): + LOG.error(usage_msg) + return + properties = admin_utils.parse_multi_keyval_opt(kwargs['property']) + lb_id = properties.get('loadbalancer-id') + if not lb_id: + LOG.error("Need to specify loadbalancer-id. " + "Add --property loadbalancer-id=") + return + + status_dict = { + octavia_const.LOADBALANCERS: [{ + 'id': lb_id, + octavia_const.PROVISIONING_STATUS: octavia_const.ERROR}]} + kw = {'status': status_dict} + + topic = octavia_const.DRIVER_TO_OCTAVIA_TOPIC + transport = messaging.get_rpc_transport(cfg.CONF) + target = messaging.Target(topic=topic, exchange="common", + namespace='control', fanout=False, + version='1.0') + client = messaging.RPCClient(transport, target) + client.cast({}, 'update_loadbalancer_status', **kw) diff --git a/vmware_nsx/shell/admin/plugins/nsxp/resources/loadbalancers.py b/vmware_nsx/shell/admin/plugins/nsxp/resources/loadbalancers.py new file mode 100644 index 0000000000..ae713ef0f0 --- /dev/null +++ b/vmware_nsx/shell/admin/plugins/nsxp/resources/loadbalancers.py @@ -0,0 +1,25 @@ +# Copyright 2021 VMware, Inc. +# All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from neutron_lib.callbacks import registry + +from vmware_nsx.shell.admin.plugins.common import constants +from vmware_nsx.shell.admin.plugins.common import loadbalancers +from vmware_nsx.shell import resources as shell + + +registry.subscribe(loadbalancers.set_loadbalancer_status_error, + constants.LOADBALANCERS, + shell.Operations.SET_STATUS_ERROR.value) diff --git a/vmware_nsx/shell/admin/plugins/nsxv/resources/loadbalancers.py b/vmware_nsx/shell/admin/plugins/nsxv/resources/loadbalancers.py new file mode 100644 index 0000000000..ae713ef0f0 --- /dev/null +++ b/vmware_nsx/shell/admin/plugins/nsxv/resources/loadbalancers.py @@ -0,0 +1,25 @@ +# Copyright 2021 VMware, Inc. +# All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from neutron_lib.callbacks import registry + +from vmware_nsx.shell.admin.plugins.common import constants +from vmware_nsx.shell.admin.plugins.common import loadbalancers +from vmware_nsx.shell import resources as shell + + +registry.subscribe(loadbalancers.set_loadbalancer_status_error, + constants.LOADBALANCERS, + shell.Operations.SET_STATUS_ERROR.value) diff --git a/vmware_nsx/shell/admin/plugins/nsxv3/resources/loadbalancers.py b/vmware_nsx/shell/admin/plugins/nsxv3/resources/loadbalancers.py new file mode 100644 index 0000000000..ae713ef0f0 --- /dev/null +++ b/vmware_nsx/shell/admin/plugins/nsxv3/resources/loadbalancers.py @@ -0,0 +1,25 @@ +# Copyright 2021 VMware, Inc. +# All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from neutron_lib.callbacks import registry + +from vmware_nsx.shell.admin.plugins.common import constants +from vmware_nsx.shell.admin.plugins.common import loadbalancers +from vmware_nsx.shell import resources as shell + + +registry.subscribe(loadbalancers.set_loadbalancer_status_error, + constants.LOADBALANCERS, + shell.Operations.SET_STATUS_ERROR.value) diff --git a/vmware_nsx/shell/resources.py b/vmware_nsx/shell/resources.py index 456686356b..53c7117da1 100644 --- a/vmware_nsx/shell/resources.py +++ b/vmware_nsx/shell/resources.py @@ -77,6 +77,7 @@ class Operations(enum.Enum): REUSE = 'reuse' UPDATE_TIER0 = 'update-tier0' UPDATE_FIREWALL_MATCH = 'update-nat-firewall-match' + SET_STATUS_ERROR = 'set-status-error' ops = [op.value for op in Operations] @@ -162,6 +163,8 @@ nsxv3_resources = { [Operations.IMPORT.value, Operations.CLEAN_ALL.value, Operations.VALIDATE.value]), + constants.LOADBALANCERS: Resource(constants.LOADBALANCERS, + [Operations.SET_STATUS_ERROR.value]), } # Add supported NSX-V resources in this dictionary @@ -258,6 +261,8 @@ nsxv_resources = { [Operations.VALIDATE.value]), constants.PORTS: Resource(constants.PORTS, [Operations.LIST.value]), + constants.LOADBALANCERS: Resource(constants.LOADBALANCERS, + [Operations.SET_STATUS_ERROR.value]), } @@ -296,6 +301,8 @@ nsxp_resources = { [Operations.CLEAN_ALL.value, Operations.VALIDATE.value, Operations.NSX_REDISTRIBUTE.value]), + constants.LOADBALANCERS: Resource(constants.LOADBALANCERS, + [Operations.SET_STATUS_ERROR.value]), } nsxv3_resources_names = list(nsxv3_resources.keys())