From d9dcd99d9ed78928076e337fca0203c9825ec6c0 Mon Sep 17 00:00:00 2001 From: Kobi Samoray Date: Sun, 13 May 2018 23:15:03 +0300 Subject: [PATCH] LBaaS pending objects housekeeping Monitor LBaaS objects which are stuck in PENDING_CREATE or PENDING_UPDATE states, and optionally change their mode to ERROR so they can be cleaned up. Change-Id: Ic3409590e52f885d367dae3b34f0066d01003b06 --- setup.cfg | 1 + vmware_nsx/common/config.py | 3 +- .../nsx_v/housekeeper/lbaas_pending.py | 95 +++++++++++++++++++ 3 files changed, 98 insertions(+), 1 deletion(-) create mode 100644 vmware_nsx/plugins/nsx_v/housekeeper/lbaas_pending.py diff --git a/setup.cfg b/setup.cfg index 20506e13e5..fd881cef60 100644 --- a/setup.cfg +++ b/setup.cfg @@ -84,6 +84,7 @@ openstack.nsxclient.v2 = vmware_nsx.neutron.nsxv.housekeeper.jobs = error_dhcp_edge = vmware_nsx.plugins.nsx_v.housekeeper.error_dhcp_edge:ErrorDhcpEdgeJob error_backup_edge = vmware_nsx.plugins.nsx_v.housekeeper.error_backup_edge:ErrorBackupEdgeJob + lbaas_pending = vmware_nsx.plugins.nsx_v.housekeeper.lbaas_pending:LbaasPendingJob [build_sphinx] source-dir = doc/source diff --git a/vmware_nsx/common/config.py b/vmware_nsx/common/config.py index f3e86d24c4..9a307ebf3f 100644 --- a/vmware_nsx/common/config.py +++ b/vmware_nsx/common/config.py @@ -722,7 +722,8 @@ nsxv_opts = [ help=_("If False, different tenants will not use the same " "DHCP edge or router edge.")), cfg.ListOpt('housekeeping_jobs', - default=['error_dhcp_edge', 'error_backup_edge'], + default=['error_dhcp_edge', 'error_backup_edge', + 'lbaas_pending'], help=_("List of the enabled housekeeping jobs")), cfg.ListOpt('housekeeping_readonly_jobs', default=[], diff --git a/vmware_nsx/plugins/nsx_v/housekeeper/lbaas_pending.py b/vmware_nsx/plugins/nsx_v/housekeeper/lbaas_pending.py new file mode 100644 index 0000000000..6d7a41e314 --- /dev/null +++ b/vmware_nsx/plugins/nsx_v/housekeeper/lbaas_pending.py @@ -0,0 +1,95 @@ +# Copyright 2018 VMware, Inc. +# All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import time + +from neutron_lbaas.db.loadbalancer import models +from neutron_lib import constants +from oslo_log import log + +from vmware_nsx.extensions import projectpluginmap +from vmware_nsx.plugins.common.housekeeper import base_job + +LOG = log.getLogger(__name__) + +ELEMENT_LIFETIME = 3 * 60 * 60 # Three hours lifetime + + +class LbaasPendingJob(base_job.BaseJob): + lbaas_objects = {} + lbaas_models = [models.LoadBalancer, + models.Listener, + models.L7Policy, + models.L7Rule, + models.PoolV2, + models.MemberV2, + models.HealthMonitorV2] + + def get_project_plugin(self, plugin): + return plugin.get_plugin_by_type(projectpluginmap.NsxPlugins.NSX_V) + + def get_name(self): + return 'lbaas_pending' + + def get_description(self): + return 'Monitor LBaaS objects in pending states' + + def run(self, context): + super(LbaasPendingJob, self).run(context) + curr_time = time.time() + + for model in self.lbaas_models: + sess = context.session + elements = sess.query(model).filter( + model.provisioning_status.in_( + [constants.PENDING_CREATE, + constants.PENDING_UPDATE, + constants.PENDING_DELETE])).all() + + for element in elements: + if element['id'] in self.lbaas_objects: + obj = self.lbaas_objects[element['id']] + lifetime = curr_time - obj['time_added'] + if lifetime > ELEMENT_LIFETIME: + # Entry has been pending for more than lifetime. + # Report and remove when in R/W mode + LOG.warning('Housekeeping: LBaaS %s %s is stuck in ' + 'pending state', + model.NAME, element['id']) + if not self.readonly: + element['provisioning_status'] = constants.ERROR + del self.lbaas_objects[element['id']] + else: + # Entry is still pending but haven't reached lifetime + LOG.debug('Housekeeping: LBaaS object %s %s in ' + 'PENDING state for %d seconds', model.NAME, + element['id'], lifetime) + obj['time_seen'] = curr_time + else: + # Entry wasn't seen before this iteration - add to dict + LOG.debug('Housekeeping: monitoring PENDING state for ' + 'LBaaS object %s %s', model.NAME, element['id']) + self.lbaas_objects[element.id] = { + 'model': model, + 'time_added': curr_time, + 'time_seen': curr_time} + + # Look for dictionary entries which weren't seen in this iteration. + # Such entries were either removed from DB or their state was changed. + for obj_id in self.lbaas_objects.keys(): + if self.lbaas_objects[obj_id]['time_seen'] != curr_time: + LOG.debug('Housekeeping: LBaaS %s %s is back to normal', + self.lbaas_objects[obj_id]['model'].NAME, obj_id) + del self.lbaas_objects[obj_id]