Fixes LP Bug#899383 - Cleanup config file search

Cleans up a bunch of configuration-related errors
when trying to run tempest out of the box with a
simple call to:

$> nosetests storm

* Raises a sensible error if the config file cannot be found
* Makes it possible to set the config file directory and
  config file name via environment variables
* Removes unnecessary calls to create storm.config.StormConfig()
  and share a configuration object by passing the openstack.Manager's
  config object with the various rest client objects
* Updates the README to show how to make a config file and run
  the tests in tempest

Change-Id: I60e33595b88df596cc9585bcaf18d37ae77d6f2b

.gitignore

@@ -1,7 +1,10 @@
 .kong-venv
 *.pyc
 etc/config.ini
+etc/storm.conf
+etc/tempest.conf
 include/swift_objects/swift_small
 include/swift_objects/swift_medium
 include/swift_objects/swift_large
 *.log
+*.swp

README.rst

@@ -1,25 +1,37 @@
 ::
 
-OpenStack integration test suite
-================================
+Tempest - The OpenStack Integration Test Suite
+==============================================
 
 This is a set of integration tests to be run against a live cluster.
 
 Quickstart
 ----------
 
-You're going to want to make your own config.ini file in the /etc/ directory,
-it needs to point at your running cluster.
+To run Tempest, you first need to create a configuration file that
+will tell Tempest where to find the various OpenStack services and
+other testing behaviour switches.
 
-After that try commands such as::
+The easiest way to create a configuration file is to copy the sample
+one in the ``etc/`` directory ::
 
-  run_tests.sh --nova
-  run_tests.sh --glance
-  run_tests.sh --swift
-  run_tests.sh --auth
+    $> cd $TEMPEST_ROOT_DIR
+    $> cp etc/storm.conf.sample etc/storm.conf
 
+After that, open up the ``etc/storm.conf`` file and edit the
+variables to fit your test environment.
 
-Additional Info
----------------
+.. note::
 
-There are additional README files in the various subdirectories of this project.
+    If you have a running devstack environment, look at the
+    environment variables in your ``devstack/localrc`` file.
+    The ADMIN_PASSWORD variable should match the api_key value
+    in the storm.conf [nova] configuration section. In addition,
+    you will need to get the UUID identifier of the image that
+    devstack uploaded and set the image_ref value in the [environment]
+    section in the storm.conf to that image UUID.
+
+After setting up your configuration file, you can execute the set of
+Tempest tests by using ``nosetests`` ::
+
+    $> nosetests storm

storm/common/rest_client.py

@@ -1,14 +1,15 @@
-from storm import exceptions
-import httplib2
 import json
+
+import httplib2
+
+from storm import exceptions
 import storm.config
 
 
 class RestClient(object):
 
-    def __init__(self, user, key, auth_url, tenant_name=None):
-        self.config = storm.config.StormConfig()
-
+    def __init__(self, config, user, key, auth_url, tenant_name=None):
+        self.config = config
         if self.config.env.authentication == 'keystone_v2':
             self.token, self.base_url = self.keystone_v2_auth(user,
                                                               key,
@@ -55,21 +56,24 @@ class RestClient(object):
         resp, body = self.http_obj.request(auth_url, 'POST',
                                            headers=headers, body=body)
 
-        try:
-            auth_data = json.loads(body)['access']
-            token = auth_data['token']['id']
-            endpoints = auth_data['serviceCatalog'][0]['endpoints']
-            mgmt_url = endpoints[0]['publicURL']
+        if resp.status == 200:
+            try:
+                auth_data = json.loads(body)['access']
+                token = auth_data['token']['id']
+                endpoints = auth_data['serviceCatalog'][0]['endpoints']
+                mgmt_url = endpoints[0]['publicURL']
 
-            #TODO (dwalleck): This is a horrible stopgap.
-            #Need to join strings more cleanly
-            temp = mgmt_url.rsplit('/')
-            service_url = temp[0] + '//' + temp[2] + '/' + temp[3] + '/'
-            management_url = service_url + tenant_name
-            return token, management_url
-        except KeyError:
-            print "Failed to authenticate user"
-            raise
+                #TODO (dwalleck): This is a horrible stopgap.
+                #Need to join strings more cleanly
+                temp = mgmt_url.rsplit('/')
+                service_url = temp[0] + '//' + temp[2] + '/' + temp[3] + '/'
+                management_url = service_url + tenant_name
+                return token, management_url
+            except Exception, e:
+                print "Failed to authenticate user: %s" % e
+                raise
+        elif resp.status == 401:
+            raise exceptions.AuthenticationFailure(user=user, password=api_key)
 
     def post(self, url, body, headers):
         return self.request('POST', url, headers, body)

storm/config.py

@@ -1,4 +1,8 @@
 import ConfigParser
+import logging
+import os
+
+LOG = logging.getLogger(__name__)
 
 
 class NovaConfig(object):
@@ -119,15 +123,24 @@ class EnvironmentConfig(object):
 class StormConfig(object):
     """Provides OpenStack configuration information."""
 
-    _path = "etc/storm.conf"
+    def __init__(self, conf_dir, conf_file):
+        """
+        Initialize a configuration from a conf directory and conf file.
 
-    def __init__(self, path=None):
-        """Initialize a configuration from a path."""
-        self._conf = self.load_config(self._path)
+        :param conf_dir: Directory to look for config files
+        :param conf_file: Name of config file to use
+        """
+        path = os.path.join(conf_dir, conf_file)
+
+        if not os.path.exists(path):
+            msg = "Config file %(path)s not found" % locals()
+            raise RuntimeError(msg)
+
+        self._conf = self.load_config(path)
         self.nova = NovaConfig(self._conf)
         self.env = EnvironmentConfig(self._conf)
 
-    def load_config(self, path=None):
+    def load_config(self, path):
         """Read configuration from given path and return a config object."""
         config = ConfigParser.SafeConfigParser()
         config.read(path)

storm/exceptions.py

@@ -16,3 +16,10 @@ class BadRequest(Exception):
 
     def __str__(self):
         return repr(self.message)
+
+
+class AuthenticationFailure(Exception):
+    msg = ("Authentication with user %(user)s and password "
+           "%(password)s failed.")
+    def __init__(self, **kwargs):
+        self.message = self.msg % kwargs

storm/openstack.py

@@ -1,3 +1,5 @@
+import os
+
 from storm.services.nova.json.images_client import ImagesClient
 from storm.services.nova.json.flavors_client import FlavorsClient
 from storm.services.nova.json.servers_client import ServersClient
@@ -7,38 +9,57 @@ import storm.config
 
 class Manager(object):
 
+    DEFAULT_CONFIG_DIR = os.path.join(
+        os.path.abspath(
+          os.path.dirname(
+            os.path.dirname(__file__))),
+        "etc")
+
+    DEFAULT_CONFIG_FILE = "storm.conf"
+
     def __init__(self):
         """
         Top level manager for all Openstack APIs
         """
-
-        self.config = storm.config.StormConfig()
+        # Environment variables override defaults...
+        config_dir = os.environ.get('TEMPEST_CONFIG_DIR',
+            self.DEFAULT_CONFIG_DIR)
+        config_file = os.environ.get('TEMPEST_CONFIG',
+            self.DEFAULT_CONFIG_FILE)
+        self.config = storm.config.StormConfig(config_dir, config_file)
         self.auth_url = data_utils.build_url(self.config.nova.host,
                                         self.config.nova.port,
                                         self.config.nova.apiVer,
                                         self.config.nova.path)
 
         if self.config.env.authentication == 'keystone_v2':
-            self.servers_client = ServersClient(self.config.nova.username,
+            self.servers_client = ServersClient(self.config,
+                                                self.config.nova.username,
                                                 self.config.nova.api_key,
                                                 self.auth_url,
                                                 self.config.nova.tenant_name)
-            self.flavors_client = FlavorsClient(self.config.nova.username,
+            self.flavors_client = FlavorsClient(self.config,
+                                                self.config.nova.username,
                                                 self.config.nova.api_key,
                                                 self.auth_url,
                                                 self.config.nova.tenant_name)
-            self.images_client = ImagesClient(self.config.nova.username,
+            self.images_client = ImagesClient(self.config,
+                                              self.config.nova.username,
                                               self.config.nova.api_key,
                                               self.auth_url,
                                               self.config.nova.tenant_name)
         else:
             #Assuming basic/native authentication
-            self.servers_client = ServersClient(self.config.nova.username,
+            self.servers_client = ServersClient(self.config,
+                                                self.config.nova.username,
                                                 self.config.nova.api_key,
                                                 self.auth_url)
-            self.flavors_client = FlavorsClient(self.config.nova.username,
+            self.flavors_client = FlavorsClient(self.config,
+                                                self.config.nova.username,
                                                 self.config.nova.api_key,
                                                 self.auth_url)
-            self.images_client = ImagesClient(self.config.nova.username,
+            self.images_client = ImagesClient(self.config,
+                                              self.config.nova.username,
+                                              self.config.nova.auth_url,
                                               self.config.nova.api_key,
                                               self.auth_url)

storm/services/nova/json/flavors_client.py

@@ -5,8 +5,9 @@ import time
 
 class FlavorsClient(object):
 
-    def __init__(self, username, key, auth_url, tenant_name=None):
-        self.client = rest_client.RestClient(username, key,
+    def __init__(self, config, username, key, auth_url, tenant_name=None):
+        self.config = config
+        self.client = rest_client.RestClient(config, username, key,
                                              auth_url, tenant_name)
 
     def list_flavors(self, params=None):

storm/services/nova/json/images_client.py

@@ -6,10 +6,10 @@ import time
 
 class ImagesClient(object):
 
-    def __init__(self, username, key, auth_url, tenant_name=None):
-        self.client = rest_client.RestClient(username, key,
+    def __init__(self, config, username, key, auth_url, tenant_name=None):
+        self.config = config
+        self.client = rest_client.RestClient(config, username, key,
                                              auth_url, tenant_name)
-        self.config = storm.config.StormConfig()
         self.build_interval = self.config.nova.build_interval
         self.build_timeout = self.config.nova.build_timeout
         self.headers = {'Content-Type': 'application/json',

storm/services/nova/json/servers_client.py

@@ -7,10 +7,10 @@ import time
 
 class ServersClient(object):
 
-    def __init__(self, username, key, auth_url, tenant_name=None):
-        self.client = rest_client.RestClient(username, key,
+    def __init__(self, config, username, key, auth_url, tenant_name=None):
+        self.config = config
+        self.client = rest_client.RestClient(config, username, key,
                                              auth_url, tenant_name)
-        self.config = storm.config.StormConfig()
         self.build_interval = self.config.nova.build_interval
         self.build_timeout = self.config.nova.build_timeout
         self.headers = {'Content-Type': 'application/json',

storm/tests/test_flavors.py

@@ -10,7 +10,7 @@ class FlavorsTest(unittest.TestCase):
     def setUpClass(cls):
         cls.os = openstack.Manager()
         cls.client = cls.os.flavors_client
-        cls.config = storm.config.StormConfig()
+        cls.config = cls.os.config
         cls.flavor_id = cls.config.env.flavor_ref
 
     @attr(type='smoke')

storm/tests/test_image_metadata.py

@@ -12,7 +12,7 @@ class ImagesMetadataTest(unittest.TestCase):
         cls.os = openstack.Manager()
         cls.servers_client = cls.os.servers_client
         cls.client = cls.os.images_client
-        cls.config = storm.config.StormConfig()
+        cls.config = cls.os.config
         cls.image_ref = cls.config.env.image_ref
         cls.flavor_ref = cls.config.env.flavor_ref
         cls.ssh_timeout = cls.config.nova.ssh_timeout

storm/tests/test_images.py

@@ -4,18 +4,21 @@ from storm.common.utils.data_utils import rand_name
 import unittest2 as unittest
 import storm.config
 
+# Some module-level skip conditions
+create_image_enabled = False
+
 
 class ImagesTest(unittest.TestCase):
-    create_image_enabled = storm.config.StormConfig().env.create_image_enabled
 
     @classmethod
     def setUpClass(cls):
         cls.os = openstack.Manager()
         cls.client = cls.os.images_client
         cls.servers_client = cls.os.servers_client
-        cls.config = storm.config.StormConfig()
+        cls.config = cls.os.config
         cls.image_ref = cls.config.env.image_ref
         cls.flavor_ref = cls.config.env.flavor_ref
+        create_image_enabled = cls.config.env.create_image_enabled
 
     def _parse_image_id(self, image_ref):
         temp = image_ref.rsplit('/')

storm/tests/test_server_actions.py

@@ -4,19 +4,21 @@ import unittest2 as unittest
 import storm.config
 from storm.common.utils.data_utils import rand_name
 
+# Some module-level skip conditions
+resize_available = False
 
 class ServerActionsTest(unittest.TestCase):
-    resize_available = storm.config.StormConfig().env.resize_available
 
     @classmethod
     def setUpClass(cls):
         cls.os = openstack.Manager()
         cls.client = cls.os.servers_client
-        cls.config = storm.config.StormConfig()
+        cls.config = cls.os.config
         cls.image_ref = cls.config.env.image_ref
         cls.image_ref_alt = cls.config.env.image_ref_alt
         cls.flavor_ref = cls.config.env.flavor_ref
         cls.flavor_ref_alt = cls.config.env.flavor_ref_alt
+        resize_available = cls.config.env.resize_available
 
     def setUp(self):
         self.name = rand_name('server')

storm/tests/test_server_details.py

@@ -11,7 +11,7 @@ class ServerDetailsTest(unittest.TestCase):
     def setUpClass(cls):
         cls.os = openstack.Manager()
         cls.client = cls.os.servers_client
-        cls.config = storm.config.StormConfig()
+        cls.config = cls.os.config
         cls.image_ref = cls.config.env.image_ref
         cls.flavor_ref = cls.config.env.flavor_ref
         cls.image_ref_alt = cls.config.env.image_ref_alt

storm/tests/test_server_metadata.py

@@ -11,7 +11,7 @@ class ServerMetadataTest(unittest.TestCase):
     def setUpClass(cls):
         cls.os = openstack.Manager()
         cls.client = cls.os.servers_client
-        cls.config = storm.config.StormConfig()
+        cls.config = cls.os.config
         cls.image_ref = cls.config.env.image_ref
         cls.flavor_ref = cls.config.env.flavor_ref
 

storm/tests/test_servers.py

@@ -13,7 +13,7 @@ class ServersTest(unittest.TestCase):
     def setUpClass(cls):
         cls.os = openstack.Manager()
         cls.client = cls.os.servers_client
-        cls.config = storm.config.StormConfig()
+        cls.config = cls.os.config
         cls.image_ref = cls.config.env.image_ref
         cls.flavor_ref = cls.config.env.flavor_ref
         cls.ssh_timeout = cls.config.nova.ssh_timeout

storm/tests/test_servers_negative.py

@@ -14,7 +14,7 @@ class ServersNegativeTest(unittest.TestCase):
     def setUpClass(cls):
         cls.os = openstack.Manager()
         cls.client = cls.os.servers_client
-        cls.config = storm.config.StormConfig()
+        cls.config = cls.os.config
         cls.image_ref = cls.config.env.image_ref
         cls.flavor_ref = cls.config.env.flavor_ref
         cls.ssh_timeout = cls.config.nova.ssh_timeout