From 52947efccd8541831728575fe873b84273aa97f3 Mon Sep 17 00:00:00 2001
From: Le Tian Ren <renlet@cn.ibm.com>
Date: Wed, 20 Aug 2014 15:05:35 +0800
Subject: [PATCH] Timing issue in remote-keystone deployment model

This is actually timing issue between openstack service endpoints becoming
active and powervc-driver's client initialization of those.

1. get_client() call with version_filter might runs into the 'None' client
error mentioned in the bug, which means *currently* only glance sync is
affected. When no version filtered, we do a rediscover for the svc_type.

2. get_client() call without version_filter or new_client() call just choose
the latest avaiable version service, so no 'None' client error, although the
chosen version might not be the real *latest* but the default hardcoded 'v1'
version, due to the timing issue mentioned in the bug. For the latter case,
unless there is an observation/notification mechanism or poll mechanism for
service versions update, based on current design, there is no perfect solution
as far as I can imagine.

Change-Id: I9ca0f906c8954ecd621be22445c12fd6a55a1ae7
Closes-Bug: 1358215
---
 .../powervc/common/client/service.py          | 51 ++++++++++++++++---
 1 file changed, 44 insertions(+), 7 deletions(-)

diff --git a/common-powervc/powervc/common/client/service.py b/common-powervc/powervc/common/client/service.py
index a89fe71..cb22693 100644
--- a/common-powervc/powervc/common/client/service.py
+++ b/common-powervc/powervc/common/client/service.py
@@ -1,12 +1,16 @@
-# Copyright 2013 IBM Corp.
+# Copyright 2013, 2014 IBM Corp.
 
-import urlparse
+import logging
 import re
+import urlparse
+
 import powervc.common.client.delegate as delegate
 from glanceclient.openstack.common import importutils
 from powervc.common.constants import SERVICE_TYPES as SERVICE_TYPES
 from powervc.common import netutils
 
+LOG = logging.getLogger(__name__)
+
 
 class AbstractService(object):
     """a stub over a service endpoint which permits consumers
@@ -227,13 +231,35 @@ class ClientServiceCatalog(object):
         """
         if svc_type not in self.endpoints:
             return None
+
+        def _find_version(versions, version_filter):
+            for version in versions.keys():
+                if version.find(version_filter) > -1:
+                    return versions[version]
+
         versions = self.endpoints[svc_type]
-        if version_filter is None:
+        # Here we need test version_filter is None or empty, use 'if not'.
+        if not version_filter:
             return versions[max(versions, key=str)]
-        for version in versions.keys():
-            if version.find(version_filter) > -1:
-                return versions[version]
-        return None
+
+        version = _find_version(versions, version_filter)
+        if version is not None:
+            return version
+        # >> fix bug/1358215 - timing issue between openstack service endpoints
+        # becoming active and powervc-driver's client initialization of those.
+        # Check https://review.openstack.org/#/c/115519/ for the details.
+        # TODO(design): re-consider for #2 in the commit message
+        else:
+            # A lock is not necessary here. Only glance sync service use
+            # specified version apis and might run into this and starup_sync
+            # won't pass until the specified versions are ready. So there
+            # shouldn't be concurrent accesses to self.endpoints[svc_type] with
+            # svc_type='image'.
+            LOG.info(_("rediscover service for type:%s") % svc_type)
+            self._rediscover_service(svc_type)
+            versions = self.endpoints[svc_type]
+            return _find_version(versions, version_filter)
+        # << fix bug/1358215
 
     def get_versions(self, svc_type):
         """return a list of the versions for the given service type
@@ -416,3 +442,14 @@ class ClientServiceCatalog(object):
                 entry = self._normalize_catalog_entry(entry)
                 self.endpoints[svc_type] = \
                     self._build_endpoint_services(entry['url'], svc_type)
+
+    # >> fix bug/1358215, timing issue between openstack service endpoints
+    # becoming active and powervc-driver's client initialization of those.
+    def _rediscover_service(self, svc_type):
+        public_eps = (self.keystone.
+                      service_catalog.get_endpoints(endpoint_type='publicURL'))
+        for entry in public_eps[svc_type]:
+            entry = self._normalize_catalog_entry(entry)
+            self.endpoints[svc_type] = \
+                self._build_endpoint_services(entry['url'], svc_type)
+    # << fix bug/1358215