Overhaul bash-completion to support non-UUID based IDs

There are a few things currently wrong with bash-completion as it stands now:

    1) IDs are currently required to be UUIDs. This is an arbitrary limitation
       and doesn't make sense for certain kinds of objects, like `Flavors`
       where a valid ID could be `performance-16gb`.

    2) The code is spread out between Oslo's `Resource` and Novaclient's
       `Manager` class. This makes it difficult to improve the code because it
       requires changes to two separate projects. We should centralize the
       code in Novaclient until the API is stable, then import the code into
       Oslo in its entirety, not partially like it is now.

    3) The completion code is handled by the `Manager` of which there is one
       per Resource-type. In the interest of centralizing this functionality,
       we should create a `CompletionCache` class and hang it off of `Client`
       of which there is one-per-session.

    4) The completion-code currently runs by default even in headless mode
       (e.g. novaclient without the shell). It'd be much more efficient to
       only write to the completion cache if we're accessing the `Client` from
       the novaclient shell. We can make this an option to support third-party
       CLI clients that want to use the completion-cache as well.

NOTE:
    * The corresponding Oslo patch is here:

        https://review.openstack.org/#/c/101376/

    * This patch was tested in multithreaded mode to prevent any regression
      from:

        https://bugs.launchpad.net/python-novaclient/+bug/1213958.

Change-Id: Idada83de103358974b739f81d4f392574f9e1237
Closes-Bug: 1332270

novaclient/base.py

@@ -20,11 +20,7 @@ Base utilities to build API operation managers and objects on top of.
 """
 
 import abc
-import contextlib
-import hashlib
 import inspect
-import os
-import threading
 
 import six
 
@@ -52,11 +48,18 @@ class Manager(utils.HookableMixin):
     etc.) and provide CRUD operations for them.
     """
     resource_class = None
-    cache_lock = threading.RLock()
 
     def __init__(self, api):
         self.api = api
 
+    def _write_object_to_completion_cache(self, obj):
+        if hasattr(self.api, 'write_object_to_completion_cache'):
+            self.api.write_object_to_completion_cache(obj)
+
+    def _clear_completion_cache_for_class(self, obj_class):
+        if hasattr(self.api, 'clear_completion_cache_for_class'):
+            self.api.clear_completion_cache_for_class(obj_class)
+
     def _list(self, url, response_key, obj_class=None, body=None):
         if body:
             _resp, body = self.api.client.post(url, body=body)
@@ -75,77 +78,22 @@ class Manager(utils.HookableMixin):
             except KeyError:
                 pass
 
-        with self.completion_cache('human_id', obj_class, mode="w"):
+        self._clear_completion_cache_for_class(obj_class)
-            with self.completion_cache('uuid', obj_class, mode="w"):
-                return [obj_class(self, res, loaded=True)
-                        for res in data if res]
 
-    @contextlib.contextmanager
+        objs = []
-    def completion_cache(self, cache_type, obj_class, mode):
+        for res in data:
-        """
+            if res:
-        The completion cache store items that can be used for bash
+                obj = obj_class(self, res, loaded=True)
-        autocompletion, like UUIDs or human-friendly IDs.
+                self._write_object_to_completion_cache(obj)
+                objs.append(obj)
 
-        A resource listing will clear and repopulate the cache.
+        return objs
-
-        A resource create will append to the cache.
-
-        Delete is not handled because listings are assumed to be performed
-        often enough to keep the cache reasonably up-to-date.
-        """
-        # NOTE(wryan): This lock protects read and write access to the
-        # completion caches
-        with self.cache_lock:
-            base_dir = utils.env('NOVACLIENT_UUID_CACHE_DIR',
-                                 default="~/.novaclient")
-
-            # NOTE(sirp): Keep separate UUID caches for each username +
-            # endpoint pair
-            username = utils.env('OS_USERNAME', 'NOVA_USERNAME')
-            url = utils.env('OS_URL', 'NOVA_URL')
-            uniqifier = hashlib.md5(username.encode('utf-8') +
-                                    url.encode('utf-8')).hexdigest()
-
-            cache_dir = os.path.expanduser(os.path.join(base_dir, uniqifier))
-
-            try:
-                os.makedirs(cache_dir, 0o755)
-            except OSError:
-                # NOTE(kiall): This is typically either permission denied while
-                #              attempting to create the directory, or the
-                #              directory already exists. Either way, don't
-                #              fail.
-                pass
-
-            resource = obj_class.__name__.lower()
-            filename = "%s-%s-cache" % (resource, cache_type.replace('_', '-'))
-            path = os.path.join(cache_dir, filename)
-
-            cache_attr = "_%s_cache" % cache_type
-
-            try:
-                setattr(self, cache_attr, open(path, mode))
-            except IOError:
-                # NOTE(kiall): This is typically a permission denied while
-                #              attempting to write the cache file.
-                pass
-
-            try:
-                yield
-            finally:
-                cache = getattr(self, cache_attr, None)
-                if cache:
-                    cache.close()
-                    delattr(self, cache_attr)
-
-    def write_to_completion_cache(self, cache_type, val):
-        cache = getattr(self, "_%s_cache" % cache_type, None)
-        if cache:
-            cache.write("%s\n" % val)
 
     def _get(self, url, response_key):
         _resp, body = self.api.client.get(url)
-        return self.resource_class(self, body[response_key], loaded=True)
+        obj = self.resource_class(self, body[response_key], loaded=True)
+        self._write_object_to_completion_cache(obj)
+        return obj
 
     def _create(self, url, body, response_key, return_raw=False, **kwargs):
         self.run_hooks('modify_body_for_create', body, **kwargs)
@@ -153,9 +101,9 @@ class Manager(utils.HookableMixin):
         if return_raw:
             return body[response_key]
 
-        with self.completion_cache('human_id', self.resource_class, mode="a"):
+        obj = self.resource_class(self, body[response_key])
-            with self.completion_cache('uuid', self.resource_class, mode="a"):
+        self._write_object_to_completion_cache(obj)
-                return self.resource_class(self, body[response_key])
+        return obj
 
     def _delete(self, url):
         _resp, _body = self.api.client.delete(url)

novaclient/client.py

@@ -20,8 +20,11 @@
 OpenStack Client interface. Handles the REST calls and responses.
 """
 
+import errno
+import glob
 import hashlib
 import logging
+import os
 import time
 
 import requests
@@ -58,6 +61,77 @@ class _ClientConnectionPool(object):
         return self._adapters[url]
 
 
+class CompletionCache(object):
+    """The completion cache is how we support tab-completion with novaclient.
+
+    The `Manager` writes object IDs and Human-IDs to the completion-cache on
+    object-show, object-list, and object-create calls.
+
+    The `nova.bash_completion` script then uses these files to provide the
+    actual tab-completion.
+
+    The cache directory layout is:
+
+        ~/.novaclient/
+            <hash-of-endpoint-and-username>/
+                <resource>-id-cache
+                <resource>-human-id-cache
+    """
+    def __init__(self, username, auth_url, attributes=('id', 'human_id')):
+        self.directory = self._make_directory_name(username, auth_url)
+        self.attributes = attributes
+
+    def _make_directory_name(self, username, auth_url):
+        """Creates a unique directory name based on the auth_url and username
+        of the current user.
+        """
+        uniqifier = hashlib.md5(username.encode('utf-8') +
+                                auth_url.encode('utf-8')).hexdigest()
+        base_dir = utils.env('NOVACLIENT_UUID_CACHE_DIR',
+                             default="~/.novaclient")
+        return os.path.expanduser(os.path.join(base_dir, uniqifier))
+
+    def _prepare_directory(self):
+        try:
+            os.makedirs(self.directory, 0o755)
+        except OSError:
+            # NOTE(kiall): This is typically either permission denied while
+            #              attempting to create the directory, or the
+            #              directory already exists. Either way, don't
+            #              fail.
+            pass
+
+    def clear_class(self, obj_class):
+        self._prepare_directory()
+
+        resource = obj_class.__name__.lower()
+        resource_glob = os.path.join(self.directory, "%s-*-cache" % resource)
+
+        for filename in glob.iglob(resource_glob):
+            try:
+                os.unlink(filename)
+            except OSError as e:
+                if e.errno != errno.ENOENT:
+                    raise
+
+    def _write_attribute(self, resource, attribute, value):
+        self._prepare_directory()
+
+        filename = "%s-%s-cache" % (resource, attribute.replace('_', '-'))
+        path = os.path.join(self.directory, filename)
+
+        with open(path, 'a') as f:
+            f.write("%s\n" % value)
+
+    def write_object(self, obj):
+        resource = obj.__class__.__name__.lower()
+
+        for attribute in self.attributes:
+            value = getattr(obj, attribute, None)
+            if value:
+                self._write_attribute(resource, attribute, value)
+
+
 class HTTPClient(object):
     USER_AGENT = 'python-novaclient'
 

novaclient/shell.py

@@ -648,6 +648,8 @@ class OpenStackComputeShell(object):
                 raise exc.CommandError(_("You must provide an auth url "
                         "via either --os-auth-url or env[OS_AUTH_URL]"))
 
+        completion_cache = client.CompletionCache(os_username, os_auth_url)
+
         self.cs = client.Client(options.os_compute_api_version,
                 os_username, os_password, os_tenant_name,
                 tenant_id=os_tenant_id, user_id=os_user_id,
@@ -659,7 +661,8 @@ class OpenStackComputeShell(object):
                 volume_service_name=volume_service_name,
                 timings=args.timings, bypass_url=bypass_url,
                 os_cache=os_cache, http_log_debug=options.debug,
-                cacert=cacert, timeout=timeout)
+                cacert=cacert, timeout=timeout,
+                completion_cache=completion_cache)
 
         # Now check for the password/token of which pieces of the
         # identifying keyring key can come from the underlying client

novaclient/v1_1/client.py

@@ -89,7 +89,7 @@ class Client(object):
                   http_log_debug=False, auth_system='keystone',
                   auth_plugin=None, auth_token=None,
                   cacert=None, tenant_id=None, user_id=None,
-                  connection_pool=False):
+                  connection_pool=False, completion_cache=None):
         # FIXME(comstud): Rename the api_key argument above when we
         # know it's not being used as keyword argument
         password = api_key
@@ -167,6 +167,16 @@ class Client(object):
                                     cacert=cacert,
                                     connection_pool=connection_pool)
 
+        self.completion_cache = completion_cache
+
+    def write_object_to_completion_cache(self, obj):
+        if self.completion_cache:
+            self.completion_cache.write_object(obj)
+
+    def clear_completion_cache_for_class(self, obj_class):
+        if self.completion_cache:
+            self.completion_cache.clear_class(obj_class)
+
     def __enter__(self):
         self.client.open_session()
         return self

novaclient/v3/client.py

@@ -74,7 +74,7 @@ class Client(object):
                   http_log_debug=False, auth_system='keystone',
                   auth_plugin=None, auth_token=None,
                   cacert=None, tenant_id=None, user_id=None,
-                  connection_pool=False):
+                  connection_pool=False, completion_cache=None):
         self.projectid = project_id
         self.tenant_id = tenant_id
         self.user_id = user_id
@@ -130,6 +130,16 @@ class Client(object):
                                     cacert=cacert,
                                     connection_pool=connection_pool)
 
+        self.completion_cache = completion_cache
+
+    def write_object_to_completion_cache(self, obj):
+        if self.completion_cache:
+            self.completion_cache.write_object(obj)
+
+    def clear_completion_cache_for_class(self, obj_class):
+        if self.completion_cache:
+            self.completion_cache.clear_class(obj_class)
+
     def __enter__(self):
         self.client.open_session()
         return self