Fix several concurrent shade gate issues

This is a big patch because there are more than one issue happening at
the same time and we have to fix all of them to fix any of them.

Force nova microversion to 2.0

The current use of novaclient is to get the latest microversion. So far
this has not been a problem, as shade deals with different payloads
across clouds all the time. However, the latest microversion to nova
broke shade's expectations about how usage reports work. Actual
microversion support is coming soon to shade, but is too much of a task
for a gate fix. In the meantime, pin to 2.0 which is available on all of
the clouds.

Produce some debug details about nova usage objects
Capture novaclient debug logging

In chasing down the usage issue, we were missing the REST interactions
we needed to be effective in chasing down the problem. novaclient passes
its own logger to keystoneauth Session, so we needed to include it in
the debug logging setup.

Also, add a helper function to make adding things like this easier.

Consume cirros qcow2 image if it's there

The move from ami to qcow2 for cirros broke shade's finding of it as a
candidate image. Move pick_image into the base class so that we can include
add_on_exception and error messages everywhere consistently.

Add image list to debug output on failure.
When we can't find a sensible image, add the list of images to the test
output so that we can examine them.

Change-Id: Ifae65e6cdf48921eaa379b803913277affbfe22a

releasenotes/notes/nova-old-microversion-5e4b8e239ba44096.yaml

@@ -0,0 +1,5 @@
+---
+upgrade:
+  - Nova microversion is being requested. Since shade is not yet
+    actively microversion aware, but has been dealing with the 2.0 structures
+    anyway, this should not affect anyone.

shade/openstackcloud.py

@@ -545,7 +545,7 @@ class OpenStackCloud(_normalize.Normalizer):
     def nova_client(self):
         if self._nova_client is None:
             self._nova_client = self._get_client(
-                'compute', novaclient.client.Client)
+                'compute', novaclient.client.Client, version='2.0')
         return self._nova_client
 
     @property

shade/tests/base.py

@@ -18,6 +18,7 @@ import os
 import fixtures
 import logging
 import munch
+import pprint
 from six import StringIO
 import testtools
 import testtools.content
@@ -75,6 +76,12 @@ class TestCase(testtools.TestCase):
         logger.addHandler(handler)
         logger.propagate = False
 
+        # Enable HTTP level tracing
+        logger = logging.getLogger('novaclient')
+        logger.setLevel(logging.DEBUG)
+        logger.addHandler(handler)
+        logger.propagate = False
+
     def assertEqual(self, first, second, *args, **kwargs):
         '''Munch aware wrapper'''
         if isinstance(first, munch.Munch):
@@ -101,3 +108,9 @@ class TestCase(testtools.TestCase):
             testtools.content_type.UTF8_TEXT,
             False)
         self.addDetail('logging', content)
+
+    def add_info_on_exception(self, name, text):
+        def add_content(unused):
+            self.addDetail(name, testtools.content.text_content(
+                pprint.pformat(text)))
+        self.addOnException(add_content)

shade/tests/functional/base.py

@@ -37,3 +37,30 @@ class BaseFunctionalTestCase(base.TestCase):
 
         self.identity_version = \
             self.operator_cloud.cloud_config.get_api_version('identity')
+
+    def pick_image(self):
+        images = self.user_cloud.list_images()
+        self.add_info_on_exception('images', images)
+
+        image_name = os.environ.get('SHADE_IMAGE')
+        if image_name:
+            for image in images:
+                if image.name == image_name:
+                    return image
+            self.assertFalse(
+                "Cloud does not have {image}".format(image=image_name))
+
+        for image in images:
+            if image.name.startswith('cirros') and image.name.endswith('-uec'):
+                return image
+        for image in images:
+            if (image.name.startswith('cirros')
+                    and image.disk_format == 'qcow2'):
+                return image
+        for image in images:
+            if image.name.lower().startswith('ubuntu'):
+                return image
+        for image in images:
+            if image.name.lower().startswith('centos'):
+                return image
+        self.assertFalse('no sensible image available')

shade/tests/functional/test_compute.py

@@ -21,7 +21,7 @@ import six
 
 from shade import exc
 from shade.tests.functional import base
-from shade.tests.functional.util import pick_flavor, pick_image
+from shade.tests.functional.util import pick_flavor
 from shade import _utils
 
 
@@ -31,9 +31,7 @@ class TestCompute(base.BaseFunctionalTestCase):
         self.flavor = pick_flavor(self.user_cloud.list_flavors())
         if self.flavor is None:
             self.assertFalse('no sensible flavor available')
-        self.image = pick_image(self.user_cloud.list_images())
+        self.image = self.pick_image()
-        if self.image is None:
-            self.assertFalse('no sensible image available')
         self.server_name = self.getUniqueString()
 
     def _cleanup_servers_and_volumes(self, server_name):

shade/tests/functional/test_floating_ip.py

@@ -28,7 +28,7 @@ from shade import _utils
 from shade import meta
 from shade.exc import OpenStackCloudException
 from shade.tests.functional import base
-from shade.tests.functional.util import pick_flavor, pick_image
+from shade.tests.functional.util import pick_flavor
 
 
 class TestFloatingIP(base.BaseFunctionalTestCase):
@@ -42,9 +42,7 @@ class TestFloatingIP(base.BaseFunctionalTestCase):
         self.flavor = pick_flavor(self.nova.flavors.list())
         if self.flavor is None:
             self.assertFalse('no sensible flavor available')
-        self.image = pick_image(self.nova.images.list())
+        self.image = self.pick_image()
-        if self.image is None:
-            self.assertFalse('no sensible image available')
 
         # Generate a random name for these tests
         self.new_item_name = self.getUniqueString()

shade/tests/functional/test_image.py

@@ -22,13 +22,12 @@ import os
 import tempfile
 
 from shade.tests.functional import base
-from shade.tests.functional.util import pick_image
 
 
 class TestImage(base.BaseFunctionalTestCase):
     def setUp(self):
         super(TestImage, self).setUp()
-        self.image = pick_image(self.user_cloud.nova_client.images.list())
+        self.image = self.pick_image()
 
     def test_create_image(self):
         test_image = tempfile.NamedTemporaryFile(delete=False)

shade/tests/functional/test_inventory.py

@@ -22,7 +22,7 @@ Functional tests for `shade` inventory methods.
 from shade import inventory
 
 from shade.tests.functional import base
-from shade.tests.functional.util import pick_flavor, pick_image
+from shade.tests.functional.util import pick_flavor
 
 
 class TestInventory(base.BaseFunctionalTestCase):
@@ -36,9 +36,7 @@ class TestInventory(base.BaseFunctionalTestCase):
         self.flavor = pick_flavor(self.nova.flavors.list())
         if self.flavor is None:
             self.assertTrue(False, 'no sensible flavor available')
-        self.image = pick_image(self.nova.images.list())
+        self.image = self.pick_image()
-        if self.image is None:
-            self.assertTrue(False, 'no sensible image available')
         self.addCleanup(self._cleanup_servers)
         self.operator_cloud.create_server(
             name=self.server_name, image=self.image, flavor=self.flavor,

shade/tests/functional/test_usage.py

@@ -30,5 +30,6 @@ class TestUsage(base.BaseFunctionalTestCase):
         usage = self.operator_cloud.get_compute_usage('demo',
                                                       datetime.datetime.now(),
                                                       datetime.datetime.now())
+        self.add_info_on_exception('usage', usage)
         self.assertIsNotNone(usage)
         self.assertTrue(hasattr(usage, 'total_hours'))

shade/tests/functional/util.py

@@ -40,22 +40,3 @@ def pick_flavor(flavors):
             flavors,
             key=operator.attrgetter('ram')):
         return flavor
-
-
-def pick_image(images):
-    image_name = os.environ.get('SHADE_IMAGE')
-    if image_name:
-        for image in images:
-            if image.name == image_name:
-                return image
-        return None
-
-    for image in images:
-        if image.name.startswith('cirros') and image.name.endswith('-uec'):
-            return image
-    for image in images:
-        if image.name.lower().startswith('ubuntu'):
-            return image
-    for image in images:
-        if image.name.lower().startswith('centos'):
-            return image