LBaaS pending objects housekeeping

Monitor LBaaS objects which are stuck in PENDING_CREATE or
PENDING_UPDATE states, and optionally change their mode to ERROR so
they can be cleaned up.

Change-Id: Ic3409590e52f885d367dae3b34f0066d01003b06

setup.cfg

@@ -84,6 +84,7 @@ openstack.nsxclient.v2 =
 vmware_nsx.neutron.nsxv.housekeeper.jobs =
     error_dhcp_edge = vmware_nsx.plugins.nsx_v.housekeeper.error_dhcp_edge:ErrorDhcpEdgeJob
     error_backup_edge = vmware_nsx.plugins.nsx_v.housekeeper.error_backup_edge:ErrorBackupEdgeJob
+    lbaas_pending = vmware_nsx.plugins.nsx_v.housekeeper.lbaas_pending:LbaasPendingJob
 
 [build_sphinx]
 source-dir = doc/source

vmware_nsx/common/config.py

@@ -722,7 +722,8 @@ nsxv_opts = [
                 help=_("If False, different tenants will not use the same "
                        "DHCP edge or router edge.")),
     cfg.ListOpt('housekeeping_jobs',
-                default=['error_dhcp_edge', 'error_backup_edge'],
+                default=['error_dhcp_edge', 'error_backup_edge',
+                         'lbaas_pending'],
                 help=_("List of the enabled housekeeping jobs")),
     cfg.ListOpt('housekeeping_readonly_jobs',
                 default=[],

vmware_nsx/plugins/nsx_v/housekeeper/lbaas_pending.py

@@ -0,0 +1,95 @@
+# Copyright 2018 VMware, Inc.
+# All Rights Reserved
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import time
+
+from neutron_lbaas.db.loadbalancer import models
+from neutron_lib import constants
+from oslo_log import log
+
+from vmware_nsx.extensions import projectpluginmap
+from vmware_nsx.plugins.common.housekeeper import base_job
+
+LOG = log.getLogger(__name__)
+
+ELEMENT_LIFETIME = 3 * 60 * 60  # Three hours lifetime
+
+
+class LbaasPendingJob(base_job.BaseJob):
+    lbaas_objects = {}
+    lbaas_models = [models.LoadBalancer,
+                    models.Listener,
+                    models.L7Policy,
+                    models.L7Rule,
+                    models.PoolV2,
+                    models.MemberV2,
+                    models.HealthMonitorV2]
+
+    def get_project_plugin(self, plugin):
+        return plugin.get_plugin_by_type(projectpluginmap.NsxPlugins.NSX_V)
+
+    def get_name(self):
+        return 'lbaas_pending'
+
+    def get_description(self):
+        return 'Monitor LBaaS objects in pending states'
+
+    def run(self, context):
+        super(LbaasPendingJob, self).run(context)
+        curr_time = time.time()
+
+        for model in self.lbaas_models:
+            sess = context.session
+            elements = sess.query(model).filter(
+                model.provisioning_status.in_(
+                    [constants.PENDING_CREATE,
+                     constants.PENDING_UPDATE,
+                     constants.PENDING_DELETE])).all()
+
+            for element in elements:
+                if element['id'] in self.lbaas_objects:
+                    obj = self.lbaas_objects[element['id']]
+                    lifetime = curr_time - obj['time_added']
+                    if lifetime > ELEMENT_LIFETIME:
+                        # Entry has been pending for more than lifetime.
+                        # Report and remove when in R/W mode
+                        LOG.warning('Housekeeping: LBaaS %s %s is stuck in '
+                                    'pending state',
+                                    model.NAME, element['id'])
+                        if not self.readonly:
+                            element['provisioning_status'] = constants.ERROR
+                        del self.lbaas_objects[element['id']]
+                    else:
+                        # Entry is still pending but haven't reached lifetime
+                        LOG.debug('Housekeeping: LBaaS object %s %s in '
+                                  'PENDING state for %d seconds', model.NAME,
+                                  element['id'], lifetime)
+                        obj['time_seen'] = curr_time
+                else:
+                    # Entry wasn't seen before this iteration - add to dict
+                    LOG.debug('Housekeeping: monitoring PENDING state for '
+                              'LBaaS object %s %s', model.NAME, element['id'])
+                    self.lbaas_objects[element.id] = {
+                        'model': model,
+                        'time_added': curr_time,
+                        'time_seen': curr_time}
+
+        # Look for dictionary entries which weren't seen in this iteration.
+        # Such entries were either removed from DB or their state was changed.
+        for obj_id in self.lbaas_objects.keys():
+            if self.lbaas_objects[obj_id]['time_seen'] != curr_time:
+                LOG.debug('Housekeeping: LBaaS %s %s is back to normal',
+                          self.lbaas_objects[obj_id]['model'].NAME, obj_id)
+                del self.lbaas_objects[obj_id]