sync to latest oslo-incubator code

This patch also syncs with latest oslo-incubator, to solve bug
1354470 and safe log auth token when --debug is enabled, see
https://github.com/openstack/oslo-incubator/blob/master/openstack/common/apiclient/client.py#L128

Note, this patch also removes importutils.py since it is no longer used.

Change-Id: I4f2d18ef05c3be27cf2ee3474761408e3f5ccbcd
Closes-Bug: #1354470

ceilometerclient/openstack/common/_i18n.py

@@ -40,6 +40,6 @@ try:
     _LC = _translators.log_critical
 except ImportError:
     # NOTE(dims): Support for cases where a project wants to use
-    # code from ceilometerclient-incubator, but is not ready to be internationalized
+    # code from oslo-incubator, but is not ready to be internationalized
     # (like tempest)
     _ = _LI = _LW = _LE = _LC = lambda x: x

ceilometerclient/openstack/common/apiclient/auth.py

@@ -17,6 +17,19 @@
 # E0202: An attribute inherited from %s hide this method
 # pylint: disable=E0202
 
+########################################################################
+#
+# THIS MODULE IS DEPRECATED
+#
+# Please refer to
+# https://etherpad.openstack.org/p/kilo-ceilometerclient-library-proposals for
+# the discussion leading to this deprecation.
+#
+# We recommend checking out the python-openstacksdk project
+# (https://launchpad.net/python-openstacksdk) instead.
+#
+########################################################################
+
 import abc
 import argparse
 import os

ceilometerclient/openstack/common/apiclient/base.py

@@ -20,6 +20,20 @@
 Base utilities to build API operation managers and objects on top of.
 """
 
+########################################################################
+#
+# THIS MODULE IS DEPRECATED
+#
+# Please refer to
+# https://etherpad.openstack.org/p/kilo-ceilometerclient-library-proposals for
+# the discussion leading to this deprecation.
+#
+# We recommend checking out the python-openstacksdk project
+# (https://launchpad.net/python-openstacksdk) instead.
+#
+########################################################################
+
+
 # E1102: %s is not callable
 # pylint: disable=E1102
 
@@ -495,6 +509,8 @@ class Resource(object):
         new = self.manager.get(self.id)
         if new:
             self._add_details(new._info)
+            self._add_details(
+                {'x_request_id': self.manager.client.last_request_id})
 
     def __eq__(self, other):
         if not isinstance(other, Resource):

ceilometerclient/openstack/common/apiclient/client.py

@@ -25,6 +25,7 @@ OpenStack Client interface. Handles the REST calls and responses.
 # E0202: An attribute inherited from %s hide this method
 # pylint: disable=E0202
 
+import hashlib
 import logging
 import time
 
@@ -33,14 +34,15 @@ try:
 except ImportError:
     import json
 
+from oslo.utils import encodeutils
 from oslo.utils import importutils
 import requests
 
 from ceilometerclient.openstack.common._i18n import _
 from ceilometerclient.openstack.common.apiclient import exceptions
 
-
 _logger = logging.getLogger(__name__)
+SENSITIVE_HEADERS = ('X-Auth-Token', 'X-Subject-Token',)
 
 
 class HTTPClient(object):
@@ -98,19 +100,32 @@ class HTTPClient(object):
         self.http = http or requests.Session()
 
         self.cached_token = None
+        self.last_request_id = None
+
+    def _safe_header(self, name, value):
+        if name in SENSITIVE_HEADERS:
+            # because in python3 byte string handling is ... ug
+            v = value.encode('utf-8')
+            h = hashlib.sha1(v)
+            d = h.hexdigest()
+            return encodeutils.safe_decode(name), "{SHA1}%s" % d
+        else:
+            return (encodeutils.safe_decode(name),
+                    encodeutils.safe_decode(value))
 
     def _http_log_req(self, method, url, kwargs):
         if not self.debug:
             return
 
         string_parts = [
-            "curl -i",
+            "curl -g -i",
             "-X '%s'" % method,
             "'%s'" % url,
         ]
 
         for element in kwargs['headers']:
-            header = "-H '%s: %s'" % (element, kwargs['headers'][element])
+            header = ("-H '%s: %s'" %
+                      self._safe_header(element, kwargs['headers'][element]))
             string_parts.append(header)
 
         _logger.debug("REQ: %s" % " ".join(string_parts))
@@ -177,6 +192,8 @@ class HTTPClient(object):
                                start_time, time.time()))
         self._http_log_resp(resp)
 
+        self.last_request_id = resp.headers.get('x-openstack-request-id')
+
         if resp.status_code >= 400:
             _logger.debug(
                 "Request returned failure status: %s",
@@ -327,6 +344,10 @@ class BaseClient(object):
         return self.http_client.client_request(
             self, method, url, **kwargs)
 
+    @property
+    def last_request_id(self):
+        return self.http_client.last_request_id
+
     def head(self, url, **kwargs):
         return self.client_request("HEAD", url, **kwargs)
 

ceilometerclient/openstack/common/apiclient/exceptions.py

@@ -20,6 +20,19 @@
 Exception definitions.
 """
 
+########################################################################
+#
+# THIS MODULE IS DEPRECATED
+#
+# Please refer to
+# https://etherpad.openstack.org/p/kilo-ceilometerclient-library-proposals for
+# the discussion leading to this deprecation.
+#
+# We recommend checking out the python-openstacksdk project
+# (https://launchpad.net/python-openstacksdk) instead.
+#
+########################################################################
+
 import inspect
 import sys
 
@@ -34,14 +47,6 @@ class ClientException(Exception):
     pass
 
 
-class MissingArgs(ClientException):
-    """Supplied arguments are not sufficient for calling a function."""
-    def __init__(self, missing):
-        self.missing = missing
-        msg = _("Missing arguments: %s") % ", ".join(missing)
-        super(MissingArgs, self).__init__(msg)
-
-
 class ValidationError(ClientException):
     """Error in validation on API client side."""
     pass
@@ -62,11 +67,16 @@ class AuthorizationFailure(ClientException):
     pass
 
 
-class ConnectionRefused(ClientException):
+class ConnectionError(ClientException):
     """Cannot connect to API service."""
     pass
 
 
+class ConnectionRefused(ConnectionError):
+    """Connection refused while trying to connect to API service."""
+    pass
+
+
 class AuthPluginOptionsMissing(AuthorizationFailure):
     """Auth plugin misses some options."""
     def __init__(self, opt_names):
@@ -447,10 +457,13 @@ def from_response(response, method, url):
         except ValueError:
             pass
         else:
-            if isinstance(body, dict) and isinstance(body.get("error"), dict):
+            if isinstance(body, dict):
-                error = body["error"]
+                error = body.get(list(body)[0])
-                kwargs["message"] = error.get("message")
+                if isinstance(error, dict):
-                kwargs["details"] = error.get("details")
+                    kwargs["message"] = (error.get("message") or
+                                         error.get("faultstring"))
+                    kwargs["details"] = (error.get("details") or
+                                         six.text_type(body))
     elif content_type.startswith("text/"):
         kwargs["details"] = response.text
 

ceilometerclient/openstack/common/apiclient/fake_client.py

@@ -21,6 +21,19 @@ wrong the tests might raise AssertionError. I've indicated in comments the
 places where actual behavior differs from the spec.
 """
 
+########################################################################
+#
+# THIS MODULE IS DEPRECATED
+#
+# Please refer to
+# https://etherpad.openstack.org/p/kilo-ceilometerclient-library-proposals for
+# the discussion leading to this deprecation.
+#
+# We recommend checking out the python-openstacksdk project
+# (https://launchpad.net/python-openstacksdk) instead.
+#
+########################################################################
+
 # W0102: Dangerous default value %s as argument
 # pylint: disable=W0102
 
@@ -168,6 +181,8 @@ class FakeHTTPClient(client.HTTPClient):
         else:
             status, body = resp
             headers = {}
+        self.last_request_id = headers.get('x-openstack-request-id',
+                                           'req-test')
         return TestResponse({
             "status_code": status,
             "text": body,

ceilometerclient/openstack/common/apiclient/utils.py

@@ -0,0 +1,100 @@
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+########################################################################
+#
+# THIS MODULE IS DEPRECATED
+#
+# Please refer to
+# https://etherpad.openstack.org/p/kilo-ceilometerclient-library-proposals for
+# the discussion leading to this deprecation.
+#
+# We recommend checking out the python-openstacksdk project
+# (https://launchpad.net/python-openstacksdk) instead.
+#
+########################################################################
+
+from oslo.utils import encodeutils
+import six
+
+from ceilometerclient.openstack.common._i18n import _
+from ceilometerclient.openstack.common.apiclient import exceptions
+from ceilometerclient.openstack.common import uuidutils
+
+
+def find_resource(manager, name_or_id, **find_args):
+    """Look for resource in a given manager.
+
+    Used as a helper for the _find_* methods.
+    Example:
+
+    .. code-block:: python
+
+        def _find_hypervisor(cs, hypervisor):
+            #Get a hypervisor by name or ID.
+            return cliutils.find_resource(cs.hypervisors, hypervisor)
+    """
+    # first try to get entity as integer id
+    try:
+        return manager.get(int(name_or_id))
+    except (TypeError, ValueError, exceptions.NotFound):
+        pass
+
+    # now try to get entity as uuid
+    try:
+        if six.PY2:
+            tmp_id = encodeutils.safe_encode(name_or_id)
+        else:
+            tmp_id = encodeutils.safe_decode(name_or_id)
+
+        if uuidutils.is_uuid_like(tmp_id):
+            return manager.get(tmp_id)
+    except (TypeError, ValueError, exceptions.NotFound):
+        pass
+
+    # for str id which is not uuid
+    if getattr(manager, 'is_alphanum_id_allowed', False):
+        try:
+            return manager.get(name_or_id)
+        except exceptions.NotFound:
+            pass
+
+    try:
+        try:
+            return manager.find(human_id=name_or_id, **find_args)
+        except exceptions.NotFound:
+            pass
+
+        # finally try to find entity by name
+        try:
+            resource = getattr(manager, 'resource_class', None)
+            name_attr = resource.NAME_ATTR if resource else 'name'
+            kwargs = {name_attr: name_or_id}
+            kwargs.update(find_args)
+            return manager.find(**kwargs)
+        except exceptions.NotFound:
+            msg = _("No %(name)s with a name or "
+                    "ID of '%(name_or_id)s' exists.") % \
+                {
+                    "name": manager.resource_class.__name__.lower(),
+                    "name_or_id": name_or_id
+                }
+            raise exceptions.CommandError(msg)
+    except exceptions.NoUniqueMatch:
+        msg = _("Multiple %(name)s matches found for "
+                "'%(name_or_id)s', use an ID to be more specific.") % \
+            {
+                "name": manager.resource_class.__name__.lower(),
+                "name_or_id": name_or_id
+            }
+        raise exceptions.CommandError(msg)

ceilometerclient/openstack/common/importutils.py

@@ -1,73 +0,0 @@
-# Copyright 2011 OpenStack Foundation.
-# All Rights Reserved.
-#
-#    Licensed under the Apache License, Version 2.0 (the "License"); you may
-#    not use this file except in compliance with the License. You may obtain
-#    a copy of the License at
-#
-#         http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-#    License for the specific language governing permissions and limitations
-#    under the License.
-
-"""
-Import related utilities and helper functions.
-"""
-
-import sys
-import traceback
-
-
-def import_class(import_str):
-    """Returns a class from a string including module and class."""
-    mod_str, _sep, class_str = import_str.rpartition('.')
-    __import__(mod_str)
-    try:
-        return getattr(sys.modules[mod_str], class_str)
-    except AttributeError:
-        raise ImportError('Class %s cannot be found (%s)' %
-                          (class_str,
-                           traceback.format_exception(*sys.exc_info())))
-
-
-def import_object(import_str, *args, **kwargs):
-    """Import a class and return an instance of it."""
-    return import_class(import_str)(*args, **kwargs)
-
-
-def import_object_ns(name_space, import_str, *args, **kwargs):
-    """Tries to import object from default namespace.
-
-    Imports a class and return an instance of it, first by trying
-    to find the class in a default namespace, then failing back to
-    a full path if not found in the default namespace.
-    """
-    import_value = "%s.%s" % (name_space, import_str)
-    try:
-        return import_class(import_value)(*args, **kwargs)
-    except ImportError:
-        return import_class(import_str)(*args, **kwargs)
-
-
-def import_module(import_str):
-    """Import a module."""
-    __import__(import_str)
-    return sys.modules[import_str]
-
-
-def import_versioned_module(version, submodule=None):
-    module = 'ceilometerclient.v%s' % version
-    if submodule:
-        module = '.'.join((module, submodule))
-    return import_module(module)
-
-
-def try_import(import_str, default=None):
-    """Try to import a module and if it fails return default."""
-    try:
-        return import_module(import_str)
-    except ImportError:
-        return default