Support multiple ca certificates

Split the ca file into two files if there are two ca certificates

Change-Id: I2bdef4ab4a2c6f24a28a4f41d7f58657c3e31bee

vmware_nsxlib/v3/cluster.py

@@ -35,6 +35,8 @@ from oslo_log import log
 from oslo_service import loopingcall
 import requests
 from requests import adapters
+from requests.adapters import HTTPAdapter
+from requests.packages.urllib3.util.retry import Retry
 
 from vmware_nsxlib._i18n import _
 from vmware_nsxlib.v3 import client as nsx_client
@@ -356,6 +358,19 @@ class EndpointState(object):
     DOWN = 'DOWN'
 
 
+class CAVerifyRetry(Retry):
+    def __init__(self, *args, **kw):
+        super(CAVerifyRetry, self).__init__(*args, **kw)
+
+    def increment(self, method=None, url=None, response=None, error=None,
+                  _pool=None, _stacktrace=None, ):
+        if isinstance(error, urllib3.exceptions.SSLError):
+            LOG.debug("skip retry ssl error %s", error)
+            raise error
+        return super(CAVerifyRetry, self).increment(method, url, response,
+                                                    error, _pool, _stacktrace)
+
+
 class Provider(object):
     """Data holder for a provider
 
@@ -374,6 +389,59 @@ class Provider(object):
     def __str__(self):
         return str(self.url)
 
+    def select_cert(self):
+        # If two ca certs in one file which only 'Serial Number' are different,
+        # ssl verify process will break if the first cert is not enabled in
+        # the nsxt. Put only one ca cert in one file and verify it by GET
+        # operation. Switch between the certificates after bootup need to
+        # reboot client
+        if not self.ca_file:
+            return
+
+        try:
+            ca_content = self._get_ca_files(self.ca_file)
+        except Exception as e:
+            LOG.error('read ca file %s error %s', self.ca_file, e)
+            return
+
+        if len(ca_content) <= 1:
+            return
+
+        base_file = '/tmp/ca_cert'
+        for index, buff in enumerate(ca_content):
+            ca_file = '{}_{}_{}.pem'.format(base_file, self.id, str(index))
+            try:
+                with open(ca_file, 'w') as fname:
+                    fname.writelines(buff)
+                session = requests.Session()
+                retry_strategy = CAVerifyRetry(total=6, backoff_factor=1,
+                                               method_whitelist=["GET"])
+                adaptor = HTTPAdapter(max_retries=retry_strategy)
+                session.mount('https://', adaptor)
+                session.verify = ca_file
+                session.get(self.url, timeout=60)
+                self.ca_file = ca_file
+                break
+            except requests.exceptions.SSLError as e:
+                LOG.debug("verification for ca_file %s failed. Error: %s",
+                          ca_file, e)
+                continue
+            except IOError as e:
+                LOG.debug("write ca_file %s failed. Error: %s",
+                          ca_file, e)
+
+    def _get_ca_files(self, ca_file):
+        files = []
+        with open(ca_file) as fname:
+            lines = fname.readlines()
+            buff = []
+            for line in lines:
+                buff.append(line)
+                if 'END CERTIFICATE' in line:
+                    files.append(buff)
+                    buff = []
+        return files
+
 
 class Endpoint(object):
     """A single NSX manager endpoint (host).
@@ -900,4 +968,6 @@ class NSXClusteredAPI(ClusteredAPI):
                     self.nsxlib_config.password(provider_index),
                     self.nsxlib_config.ca_file(provider_index),
                     self.nsxlib_config.thumbprint(provider_index)))
+        for provider in providers:
+            provider.select_cert()
         return providers