Merge "[OVN] neutron-ovn-metadat-agent add retry logic for sb_idl"

neutron/agent/ovn/metadata/ovsdb.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from oslo_log import log
 from ovs.db import idl
 from ovsdbapp.backend.ovs_idl import connection
 from ovsdbapp.backend.ovs_idl import idlutils
@@ -23,6 +24,9 @@ from neutron.plugins.ml2.drivers.ovn.mech_driver.ovsdb import impl_idl_ovn
 from neutron.plugins.ml2.drivers.ovn.mech_driver.ovsdb import ovsdb_monitor
 
 
+LOG = log.getLogger(__name__)
+
+
 class MetadataAgentOvnSbIdl(ovsdb_monitor.OvnIdl):
 
     SCHEMA = 'OVN_Southbound'
@@ -51,7 +55,12 @@ class MetadataAgentOvnSbIdl(ovsdb_monitor.OvnIdl):
     def _get_ovsdb_helper(self, connection_string):
         return idlutils.get_schema_helper(connection_string, self.SCHEMA)
 
+    @tenacity.retry(
+        wait=tenacity.wait_exponential(
+            max=config.get_ovn_ovsdb_retry_max_interval()),
+        reraise=True)
     def start(self):
+        LOG.info('Getting OvsdbSbOvnIdl for MetadataAgent with retry')
         conn = connection.Connection(
             self, timeout=config.get_ovn_ovsdb_timeout())
         return impl_idl_ovn.OvsdbSbOvnIdl(conn)

neutron/agent/ovn/metadata/server.py

@@ -14,6 +14,7 @@
 
 import hashlib
 import hmac
+import threading
 import urllib
 
 from neutron._i18n import _
@@ -46,8 +47,21 @@ class MetadataProxyHandler(object):
     def __init__(self, conf, chassis):
         self.conf = conf
         self.chassis = chassis
+        self._sb_idl = None
+        self._post_fork_event = threading.Event()
         self.subscribe()
 
+    @property
+    def sb_idl(self):
+        if not self._sb_idl:
+            self._post_fork_event.wait()
+
+        return self._sb_idl
+
+    @sb_idl.setter
+    def sb_idl(self, val):
+        self._sb_idl = val
+
     def subscribe(self):
         registry.subscribe(self.post_fork_initialize,
                            resources.PROCESS,
@@ -56,10 +70,14 @@ class MetadataProxyHandler(object):
     def post_fork_initialize(self, resource, event, trigger, payload=None):
         # We need to open a connection to OVN SouthBound database for
         # each worker so that we can process the metadata requests.
+        self._post_fork_event.clear()
         self.sb_idl = ovsdb.MetadataAgentOvnSbIdl(
             tables=('Port_Binding', 'Datapath_Binding', 'Chassis'),
             chassis=self.chassis).start()
 
+        # Now IDL connections can be safely used.
+        self._post_fork_event.set()
+
     @webob.dec.wsgify(RequestClass=webob.Request)
     def __call__(self, req):
         try:

neutron/tests/unit/agent/ovn/metadata/test_server.py

@@ -57,6 +57,7 @@ class TestMetadataProxyHandler(base.BaseTestCase):
         self.log = self.log_p.start()
         self.handler = agent.MetadataProxyHandler(self.fake_conf, 'chassis1')
         self.handler.sb_idl = mock.Mock()
+        self.handler._post_fork_event.set()
 
     def test_call(self):
         req = mock.Mock()