Make clouds file support configurable

Add support for some '_ID' authentication option suffixes
Use YAML parser for all files with suffix other than .json

Change-Id: I09c4229b811493bdddeceab2a87613daab12f9af

tobiko/openstack/keystone/_clouds_file.py

@@ -16,28 +16,18 @@ from __future__ import absolute_import
 import json
 import os
 
-import appdirs
 from oslo_log import log
 import yaml
 
+import tobiko
 from tobiko.openstack.keystone import _credentials
 
 
 LOG = log.getLogger(__name__)
 
-APPDIRS = appdirs.AppDirs('openstack', 'OpenStack', multipath='/etc')
-
-CONFIG_SEARCH_PATH = [os.getcwd(),
-                      APPDIRS.user_config_dir,
-                      os.path.expanduser('~/.config/openstack'),
-                      APPDIRS.site_config_dir,
-                      '/etc/openstack']
 YAML_SUFFIXES = ('.yaml', '.yml')
 JSON_SUFFIXES = ('.json',)
-DEFAULT_CLOUDS_FILES = [
+CLOUDS_FILE_SUFFIXES = JSON_SUFFIXES + YAML_SUFFIXES
-    os.path.join(d, 'clouds' + s)
-    for d in CONFIG_SEARCH_PATH
-    for s in YAML_SUFFIXES + JSON_SUFFIXES]
 
 
 try:
@@ -46,6 +36,34 @@ except NameError:
     FileNotFound = OSError
 
 
+class DefaultCloudsFileConfig(tobiko.SharedFixture):
+
+    cloud_name = None
+    clouds_file_dirs = None
+    clouds_file_names = None
+    clouds_files = None
+
+    def setup_fixture(self):
+        from tobiko import config
+        CONF = config.CONF
+        keystone_conf = CONF.tobiko.keystone
+        self.cloud_name = keystone_conf.cloud_name
+        self.clouds_file_dirs = [
+            os.path.realpath(os.path.expanduser(d))
+            for d in keystone_conf.clouds_file_dirs]
+        self.clouds_file_names = keystone_conf.clouds_file_names
+        self.clouds_files = self.list_cloud_files()
+
+    def list_cloud_files(self):
+        cloud_files = []
+        for directory in self.clouds_file_dirs:
+            directory = os.path.realpath(os.path.expanduser(directory))
+            for file_name in self.clouds_file_names:
+                file_name = os.path.join(directory, file_name)
+                cloud_files.append(file_name)
+        return cloud_files
+
+
 class CloudsFileKeystoneCredentialsFixture(
         _credentials.KeystoneCredentialsFixture):
 
@@ -53,20 +71,27 @@ class CloudsFileKeystoneCredentialsFixture(
     clouds_content = None
     clouds_file = None
 
+    config = tobiko.required_setup_fixture(DefaultCloudsFileConfig)
+
     def __init__(self, credentials=None, cloud_name=None,
                  clouds_content=None, clouds_file=None, clouds_files=None):
         super(CloudsFileKeystoneCredentialsFixture, self).__init__(
             credentials=credentials)
-        if cloud_name is not None:
+
-            self.cloud_name = cloud_name
+        config = self.config
+        if cloud_name is None:
+            cloud_name = config.cloud_name
+        self.cloud_name = cloud_name
+
         if clouds_content is not None:
             self.clouds_content = dict(clouds_content)
+
         if clouds_file is not None:
             self.clouds_file = clouds_file
+
         if clouds_files is None:
-            self.clouds_files = tuple(DEFAULT_CLOUDS_FILES)
+            clouds_files = config.clouds_files
-        else:
+        self.clouds_files = list(clouds_files)
-            self.clouds_files = tuple(clouds_files)
 
     def get_credentials(self):
         cloud_name = self._get_cloud_name()
@@ -95,27 +120,42 @@ class CloudsFileKeystoneCredentialsFixture(
                        "{!r}").format(self.clouds_file, self.cloud_name)
             raise ValueError(message)
 
+        username = auth.get('username') or auth.get('user_id')
+        password = auth.get('password')
+        project_name = (auth.get('project_name') or
+                        auth.get('tenant_namer') or
+                        auth.get('project_id') or
+                        auth.get_env('tenant_id'))
+
         api_version = (int(clouds_config.get("identity_api_version", 0)) or
                        _credentials.api_version_from_url(auth_url))
         if api_version == 2:
             return _credentials.keystone_credentials(
                 api_version=api_version,
                 auth_url=auth_url,
-                username=auth.get("username"),
+                username=username,
-                password=auth.get("password"),
+                password=password,
-                project_name=auth.get("project_name"))
+                project_name=project_name)
+
         else:
+            domain_name = (auth.get("domain_name") or
+                           auth.get("domain_id"))
+            user_domain_name = (auth.get("user_domain_name") or
+                                auth.get("user_domain_id"))
+            project_domain_name = auth.get("project_domain_name")
+            project_domain_id = auth.get("project_domain_id")
+            trust_id = auth.get("trust_id")
             return _credentials.keystone_credentials(
                 api_version=api_version,
                 auth_url=auth_url,
-                username=auth.get("username"),
+                username=username,
-                password=auth.get("password"),
+                password=password,
-                project_name=auth.get("project_name"),
+                project_name=project_name,
-                domain_name=auth.get("domain_name"),
+                domain_name=domain_name,
-                user_domain_name=auth.get("user_domain_name"),
+                user_domain_name=user_domain_name,
-                project_domain_name=auth.get("project_domain_name"),
+                project_domain_name=project_domain_name,
-                project_domain_id=auth.get("project_domain_id"),
+                project_domain_id=project_domain_id,
-                trust_id=auth.get("trust_id"))
+                trust_id=trust_id)
 
     def _get_cloud_name(self):
         cloud_name = self.cloud_name
@@ -142,13 +182,9 @@ class CloudsFileKeystoneCredentialsFixture(
                 if suffix in JSON_SUFFIXES:
                     LOG.debug('Load JSON clouds file: %r', clouds_file)
                     clouds_content = json.load(f)
-                elif suffix in YAML_SUFFIXES:
+                else:
                     LOG.debug('Load YAML clouds file: %r', clouds_file)
                     clouds_content = yaml.safe_load(f)
-                else:
-                    message = 'Invalid clouds file suffix: {!r}'.format(
-                        suffix)
-                    raise ValueError(message)
             LOG.debug('Clouds file content loaded from %r:\n%r',
                       clouds_file, json.dumps(clouds_content,
                                               indent=4,

tobiko/openstack/keystone/_credentials.py

@@ -21,6 +21,8 @@ from oslo_log import log
 import yaml
 
 import tobiko
+
+
 LOG = log.getLogger(__name__)
 
 

tobiko/openstack/keystone/config.py

@@ -17,6 +17,7 @@ import itertools
 
 from oslo_config import cfg
 
+
 GROUP_NAME = 'keystone'
 OPTIONS = [
     cfg.IntOpt('api_version',
@@ -48,11 +49,20 @@ OPTIONS = [
                help="Project domain ID"),
     cfg.StrOpt('trust_id',
                default=None,
-               help="Trust ID for trust scoping.")]
+               help="Trust ID for trust scoping."),
+    cfg.StrOpt('cloud_name',
+               default=None,
+               help=("Cloud name used pick authentication parameters from "
+                     "clouds.*")),
+    cfg.ListOpt('clouds_file_dirs',
+                default=['.', '~/.config/openstack', '/etc/openstack'],
+                help=("Directories where to look for clouds files")),
+    cfg.ListOpt('clouds_file_names',
+                default=['clouds.yaml', 'clouds.yml', 'clouds.json'],
+                help="Clouds file names")]
 
 
 def register_tobiko_options(conf):
-
     conf.register_opts(group=cfg.OptGroup(GROUP_NAME), opts=OPTIONS)
 
 

tobiko/tests/unit/openstack/_keystone.py

@@ -40,4 +40,7 @@ class DefaultKeystoneCredentialsPatch(unit.PatchFixture):
                                 keystone.KeystoneCredentials)
 
     def setup_fixture(self):
-        self.patch(config.CONF.tobiko, 'keystone', self.credentials)
+        CONF = config.CONF
+        keystone_conf = CONF.tobiko.keystone
+        for key, value in self.credentials.to_dict().items():
+            self.patch(keystone_conf, key, value)

tobiko/tests/unit/openstack/keystone/test_clouds_file.py

@@ -103,13 +103,15 @@ class V3CloudsFileFixture(CloudsFileFixture):
 
 class CloudsFileKeystoneCredentialsFixtureTest(openstack.OpenstackTest):
 
+    config = tobiko.required_setup_fixture(
+        _clouds_file.DefaultCloudsFileConfig)
+
     def test_init(self):
         fixture = keystone.CloudsFileKeystoneCredentialsFixture()
         self.assertIsNone(fixture.cloud_name)
         self.assertIsNone(fixture.clouds_content)
         self.assertIsNone(fixture.clouds_file)
-        self.assertEqual(tuple(_clouds_file.DEFAULT_CLOUDS_FILES),
+        self.assertEqual(self.config.clouds_files, fixture.clouds_files)
-                         fixture.clouds_files)
 
     def test_init_with_cloud_name(self):
         fixture = keystone.CloudsFileKeystoneCredentialsFixture(
@@ -129,11 +131,11 @@ class CloudsFileKeystoneCredentialsFixtureTest(openstack.OpenstackTest):
     def test_init_with_clouds_files(self):
         fixture = keystone.CloudsFileKeystoneCredentialsFixture(
             clouds_files=['a', 'b', 'd'])
-        self.assertEqual(('a', 'b', 'd'), fixture.clouds_files)
+        self.assertEqual(['a', 'b', 'd'], fixture.clouds_files)
 
     def test_setup_from_default_clouds_files(self):
         file_fixture = self.useFixture(V3CloudsFileFixture())
-        self.patch(_clouds_file, 'DEFAULT_CLOUDS_FILES',
+        self.patch(self.config, 'clouds_files',
                    ['/a', file_fixture.clouds_file, '/c'])
         credentials_fixture = self.useFixture(
             keystone.CloudsFileKeystoneCredentialsFixture(
@@ -176,15 +178,6 @@ class CloudsFileKeystoneCredentialsFixtureTest(openstack.OpenstackTest):
         self.assertEqual(test_credentials.V3_PARAMS,
                          credentials_fixture.credentials.to_dict())
 
-    def test_setup_from_invalid_suffix(self):
-        file_fixture = self.useFixture(V3CloudsFileFixture(suffix='.txt'))
-        credentials_fixture = keystone.CloudsFileKeystoneCredentialsFixture(
-                cloud_name=file_fixture.cloud_name,
-                clouds_file=file_fixture.clouds_file)
-        ex = self.assertRaises(ValueError, tobiko.setup_fixture,
-                               credentials_fixture)
-        self.assertEqual("Invalid clouds file suffix: '.txt'", str(ex))
-
     def test_setup_v2_credentials(self):
         file_fixture = self.useFixture(V2CloudsFileFixture())
         credentials_fixture = self.useFixture(
@@ -292,7 +285,7 @@ class CloudsFileKeystoneCredentialsFixtureTest(openstack.OpenstackTest):
             "'cloud-name'", str(ex))
 
     def test_setup_without_clouds_file(self):
-        self.patch(_clouds_file, 'DEFAULT_CLOUDS_FILES', ['/a', '/b', '/c'])
+        self.patch(self.config, 'clouds_files', ['/a', '/b', '/c'])
         fixture = keystone.CloudsFileKeystoneCredentialsFixture(
             cloud_name='cloud-name')
         ex = self.assertRaises(_clouds_file.FileNotFound, tobiko.setup_fixture,