Merge "Refactoring of deployment credentials"

rally/api.py

@@ -149,8 +149,9 @@ class _Deployment(object):
         :param deployment: Deployment object
         :returns: Service list
         """
-        # TODO(kun): put this work into objects.Deployment
+        # TODO(astudenov): put this work into Credential plugins
-        clients = osclients.Clients(objects.Credential(**deployment["admin"]))
+        admin = deployment.get_credentials_for("openstack")["admin"]
+        clients = osclients.Clients(objects.Credential(**admin))
         return clients.services()
 
     @staticmethod
@@ -167,8 +168,9 @@ class _Deployment(object):
 
         :returns: Service list
         """
+        # TODO(astudenov): put this work into Credential plugins
         services = cls.service_list(deployment)
-        users = deployment["users"]
+        users = deployment.get_credentials_for("openstack")["users"]
         for endpoint_dict in users:
             osclients.Clients(objects.Credential(**endpoint_dict)).keystone()
 
@@ -317,8 +319,9 @@ class _Task(object):
         deployment = objects.Deployment.get(deployment)
         task = task_instance or objects.Task(
             deployment_uuid=deployment["uuid"], temporary=True)
+        creds = deployment.get_credentials_for("openstack")
         benchmark_engine = engine.TaskEngine(
-            config, task, admin=deployment["admin"], users=deployment["users"])
+            config, task, admin=creds["admin"], users=creds["users"])
 
         benchmark_engine.validate()
 
@@ -345,8 +348,10 @@ class _Task(object):
 
         LOG.info("Benchmark Task %s on Deployment %s" % (task["uuid"],
                                                          deployment["uuid"]))
+
+        creds = deployment.get_credentials_for("openstack")
         benchmark_engine = engine.TaskEngine(
-            config, task, admin=deployment["admin"], users=deployment["users"],
+            config, task, admin=creds["admin"], users=creds["users"],
             abort_on_sla_failure=abort_on_sla_failure)
 
         try:

rally/cli/commands/deployment.py

@@ -202,8 +202,9 @@ class DeploymentCommands(object):
         table_rows = []
 
         deployment = api.deployment.get(deployment)
-        users = deployment["users"]
+        creds = deployment.get_credentials_for("openstack")
-        admin = deployment["admin"]
+        users = creds["users"]
+        admin = creds["admin"]
         credentials = users + [admin] if admin else users
 
         for ep in credentials:
@@ -233,7 +234,8 @@ class DeploymentCommands(object):
         try:
             services = api.deployment.check(deployment)
         except keystone_exceptions.ConnectionRefused:
-            print(_("Unable to connect %s.") % deployment["admin"]["auth_url"])
+            admin = deployment.get_credentials_for("openstack")["admin"]
+            print(_("Unable to connect %s.") % admin["auth_url"])
             return(1)
 
         except exceptions.InvalidArgumentsException:
@@ -305,9 +307,10 @@ class DeploymentCommands(object):
 
             fileutils.update_globals_file("RALLY_DEPLOYMENT",
                                           deployment["uuid"])
+
+            creds = deployment.get_credentials_for("openstack")
             self._update_openrc_deployment_file(
-                deployment["uuid"],
+                deployment["uuid"], creds["admin"] or creds["users"][0])
-                deployment["admin"] or deployment["users"][0])
             print("~/.rally/openrc was updated\n\nHINTS:\n"
                   "\n* To use standard OpenStack clients, set up your env by "
                   "running:\n\tsource ~/.rally/openrc\n"

rally/common/db/sqlalchemy/api.py

@@ -77,37 +77,6 @@ def _alembic_config():
     return config
 
 
-def fix_deployment(fn):
-    # NOTE(ikhudoshyn): Remove this once a new deployment model
-    # get adopted.
-    # New DB schema for Deployment was introduced in
-    # https://github.com/openstack/rally/
-    #        commit/433cf080ea02f448df1ce33c620c8b30910338cd
-    # yet old one is used over the codebase.
-    # This decorator restores attributes that are missing from
-    # the new model.
-    """Restore old deployment model's attributes
-
-    This decorator restores deployment properties "admin" and "users"
-    moved into "credentials" attribute of DB model.
-    """
-
-    def fix(o):
-        if isinstance(o, list):
-            return [fix(deployment) for deployment in o]
-        else:
-            if not o.get("admin"):
-                o["admin"] = o["credentials"][0][1]["admin"]
-            if not o.get("users"):
-                o["users"] = o["credentials"][0][1]["users"]
-            return o
-
-    def wrapper(*args, **kwargs):
-        deployment = fn(*args, **kwargs)
-        return fix(deployment)
-    return wrapper
-
-
 class Connection(object):
 
     def engine_reset(self):
@@ -594,20 +563,10 @@ class Connection(object):
             raise exceptions.DeploymentNotFound(deployment=deployment)
         return stored_deployment
 
-    @fix_deployment
     @db_api.serialize
     def deployment_create(self, values):
         deployment = models.Deployment()
         try:
-            # TODO(rpromyshlennikov): remove after credentials refactoring
-            values.setdefault(
-                "credentials",
-                [
-                    ["openstack",
-                     {"admin": values.get("admin"),
-                      "users": values.get("users", [])}]
-                ]
-            )
             deployment.update(values)
             deployment.save()
         except db_exc.DBDuplicateEntry:
@@ -627,12 +586,10 @@ class Connection(object):
             if not count:
                 raise exceptions.DeploymentNotFound(deployment=uuid)
 
-    @fix_deployment
     @db_api.serialize
     def deployment_get(self, deployment):
         return self._deployment_get(deployment)
 
-    @fix_deployment
     @db_api.serialize
     def deployment_update(self, deployment, values):
         session = get_session()
@@ -642,7 +599,6 @@ class Connection(object):
             dpl.update(values)
         return dpl
 
-    @fix_deployment
     @db_api.serialize
     def deployment_list(self, status=None, parent_uuid=None, name=None):
         query = (self.model_query(models.Deployment).

rally/common/db/sqlalchemy/migrations/script.py.mako

@@ -21,8 +21,8 @@ Create Date: ${create_date}
 """
 
 # revision identifiers, used by Alembic.
-revision = ${repr(up_revision)}
+revision = "${up_revision}"
-down_revision = ${repr(down_revision)}
+down_revision = "${down_revision}"
 branch_labels = ${repr(branch_labels)}
 depends_on = ${repr(depends_on)}
 

rally/common/db/sqlalchemy/migrations/versions/92aaaa2a6bb3_refactor_credentials.py

@@ -0,0 +1,73 @@
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""refactor_credentials
+
+Revision ID: 92aaaa2a6bb3
+Revises: 4ef544102ba7
+Create Date: 2017-02-01 12:52:43.499663
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = "92aaaa2a6bb3"
+down_revision = "4ef544102ba7"
+branch_labels = None
+depends_on = None
+
+from alembic import op
+import sqlalchemy as sa
+
+from rally.common.db.sqlalchemy import types as sa_types
+from rally import exceptions
+
+
+deployments_helper = sa.Table(
+    "deployments",
+    sa.MetaData(),
+    sa.Column("name", sa.String(255), unique=True),
+    sa.Column("id", sa.Integer, primary_key=True, autoincrement=True),
+    sa.Column("credentials", sa.PickleType, nullable=True),
+    sa.Column("new_credentials", sa_types.MutableJSONEncodedDict,
+              default={}, nullable=False)
+)
+
+
+def upgrade():
+    with op.batch_alter_table("deployments") as batch_op:
+        batch_op.add_column(
+            sa.Column("new_credentials", sa_types.MutableJSONEncodedDict,
+                      default={}))
+
+    connection = op.get_bind()
+    for deployment in connection.execute(deployments_helper.select()):
+        creds = {}
+        for cred_type, cred_obj in deployment.credentials:
+            creds.setdefault(cred_type, [])
+            creds[cred_type].append(cred_obj)
+
+        connection.execute(
+            deployments_helper.update().where(
+                deployments_helper.c.id == deployment.id).values(
+                new_credentials=creds))
+
+    with op.batch_alter_table("deployments") as batch_op:
+        batch_op.drop_column("credentials")
+        batch_op.alter_column("new_credentials", new_column_name="credentials",
+                              existing_type=sa_types.MutableJSONEncodedDict,
+                              nullable=False)
+
+
+def downgrade():
+    raise exceptions.DowngradeNotSupported()

rally/common/db/sqlalchemy/models.py

@@ -24,7 +24,6 @@ from oslo_utils import timeutils
 import sqlalchemy as sa
 from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy import schema
-from sqlalchemy import types
 
 from rally.common.db.sqlalchemy import types as sa_types
 from rally import consts
@@ -83,7 +82,8 @@ class Deployment(BASE, RallyBase):
         nullable=False,
     )
 
-    credentials = sa.Column(types.PickleType, default=[], nullable=False)
+    credentials = sa.Column(
+        sa_types.MutableJSONEncodedDict, default={}, nullable=False)
 
     status = sa.Column(
         sa.Enum(*consts.DeployStatus, name="enum_deploy_status"),
@@ -99,24 +99,6 @@ class Deployment(BASE, RallyBase):
         foreign_keys=parent_uuid,
     )
 
-    # TODO(rpromyshlennikov): remove admin after credentials refactoring
-    @property
-    def admin(self):
-        return self.credentials[0][1]["admin"]
-
-    @admin.setter
-    def admin(self, value):
-        pass
-
-    # TODO(rpromyshlennikov): remove users after credentials refactoring
-    @property
-    def users(self):
-        return self.credentials[0][1]["users"]
-
-    @users.setter
-    def users(self, value):
-        pass
-
 
 class Resource(BASE, RallyBase):
     """Represent a resource of a deployment."""

rally/common/objects/deploy.py

@@ -15,8 +15,38 @@
 
 import datetime as dt
 
+import jsonschema
+
+from rally.common.i18n import _, _LW
 from rally.common import db
+from rally.common import logging
 from rally import consts
+from rally import exceptions
+
+
+LOG = logging.getLogger(__name__)
+
+CREDENTIALS_SCHEMA = {
+    "type": "object",
+    "patternProperties": {
+        ".*": {
+            "type": "array",
+            "items": {
+                "type": "object",
+                "properties": {
+                    "admin": {"type": "object"},
+                    "users": {
+                        "type": "array",
+                        "items": {"type": "object"}
+                    }
+                },
+                "required": ["admin", "users"],
+                "additionalProperties": False,
+            },
+        }
+    },
+    "minProperties": 1,
+}
 
 
 class Deployment(object):
@@ -29,6 +59,12 @@ class Deployment(object):
             self.deployment = db.deployment_create(attributes)
 
     def __getitem__(self, key):
+        # TODO(astudenov): remove this in future releases
+        if key == "admin" or key == "users":
+            LOG.warning(_LW("deployment.%s is deprecated in Rally 0.9.0. "
+                            "Use deployment.get_credentials_for('openstack')"
+                            "['%s'] to get credentials.") % (key, key))
+            return self.get_credentials_for("openstack")[key]
         return self.deployment[key]
 
     @staticmethod
@@ -56,15 +92,16 @@ class Deployment(object):
         self._update({"config": config})
 
     def update_credentials(self, credentials):
-        admin = credentials.get("admin", {})
+        jsonschema.validate(credentials, CREDENTIALS_SCHEMA)
-        if admin:
+        self._update({"credentials": credentials})
-            admin = admin.to_dict(include_permission=True)
-        users = [e.to_dict(include_permission=True)
-                 for e in credentials.get("users", [])]
 
-        self._update({
+    def get_credentials_for(self, namespace):
-            "credentials": [["openstack", {"admin": admin, "users": users}]]
+        try:
-        })
+            return self.deployment["credentials"][namespace][0]
+        except (KeyError, IndexError) as e:
+            LOG.exception(e)
+            raise exceptions.RallyException(_(
+                "No credentials found for %s") % namespace)
 
     def set_started(self):
         self._update({"started_at": dt.datetime.now(),

rally/common/objects/verifier.py

@@ -68,7 +68,8 @@ class Verifier(object):
         self._db_entry = db.verifier_update(self.uuid, **properties)
 
     def set_deployment(self, deployment_id):
-        self._deployment = db.deployment_get(deployment_id)
+        from rally.common import objects
+        self._deployment = objects.Deployment.get(deployment_id)
 
     @property
     def deployment(self):

rally/deployment/engines/devstack.py

@@ -137,7 +137,13 @@ class DevstackEngine(engine.Engine):
             "http://%s:5000/v2.0/" % self.servers[0].host, "admin",
             self.local_conf["ADMIN_PASSWORD"], "admin",
             consts.EndpointPermission.ADMIN)
-        return {"admin": admin_credential}
+        return {
+            "openstack": [
+                {
+                    "admin": admin_credential.to_dict(include_permission=True),
+                    "users": []
+                }
+            ]}
 
     def cleanup(self):
         for resource in self.deployment.get_resources(type="credentials"):

rally/deployment/engines/existing.py

@@ -160,12 +160,14 @@ class ExistingCloud(engine.Engine):
 
         users = [self._create_credential(self.config, user, permissions.USER)
                  for user in self.config.get("users", [])]
+        users = [user.to_dict(include_permission=True) for user in users]
 
         admin = self._create_credential(self.config,
                                         self.config.get("admin"),
                                         permissions.ADMIN)
+        admin = admin.to_dict(include_permission=True)
 
-        return {"admin": admin, "users": users}
+        return {"openstack": [{"admin": admin, "users": users}]}
 
     def cleanup(self):
         pass

rally/deployment/engines/lxc.py

@@ -149,7 +149,10 @@ class LxcEngine(engine.Engine):
                 container.ssh.run("/bin/sh -e", stdin=open(start_script, "rb"))
             if network:
                 network += 1
-        return {"admin": objects.Credential("", "", "", "")}
+
+        admin = objects.Credential("", "", "", "").to_dict(
+            include_permission=True)
+        return {"openstack": [{"admin": admin, "users": []}]}
 
     def cleanup(self):
         resources = self.deployment.get_resources()

rally/plugins/openstack/verification/tempest/config.py

@@ -87,7 +87,7 @@ class TempestConfigfileManager(object):
     """Class to create a Tempest config file."""
 
     def __init__(self, deployment):
-        self.credential = deployment["admin"]
+        self.credential = deployment.get_credentials_for("openstack")["admin"]
         self.clients = osclients.Clients(objects.Credential(**self.credential))
         self.available_services = self.clients.services().values()
 

rally/plugins/openstack/verification/tempest/context.py

@@ -44,8 +44,8 @@ class TempestContext(context.VerifierContext):
     def __init__(self, ctx):
         super(TempestContext, self).__init__(ctx)
 
-        credential = self.verifier.deployment["admin"]
+        creds = self.verifier.deployment.get_credentials_for("openstack")
-        self.clients = osclients.Clients(objects.Credential(**credential))
+        self.clients = osclients.Clients(objects.Credential(**creds["admin"]))
         self.available_services = self.clients.services().values()
 
         self.conf = configparser.ConfigParser()

rally/task/validation.py

@@ -460,8 +460,9 @@ def required_services(config, clients, deployment, *required_services):
     available_services = list(clients.services().values())
 
     if consts.Service.NOVA_NET in required_services:
+        creds = deployment.get_credentials_for("openstack")
         nova = osclients.Clients(
-            objects.Credential(**deployment["admin"])).nova()
+            objects.Credential(**creds["admin"])).nova()
         for service in nova.services.list():
             if (service.binary == consts.Service.NOVA_NET and
                     service.status == "enabled"):
@@ -505,9 +506,9 @@ def required_cinder_services(config, clients, deployment, service_name):
 
     :param service_name: Cinder service name
     """
-
+    creds = deployment.get_credentials_for("openstack")
     admin_client = osclients.Clients(
-        objects.Credential(**deployment["admin"])).cinder()
+        objects.Credential(**creds["admin"])).cinder()
 
     for service in admin_client.services.list():
         if (service.binary == six.text_type(service_name) and
@@ -527,7 +528,8 @@ def required_clients(config, clients, deployment, *components, **kwargs):
                      admin - bool, whether to use admin clients
     """
     if kwargs.get("admin", False):
-        clients = osclients.Clients(objects.Credential(**deployment["admin"]))
+        creds = deployment.get_credentials_for("openstack")
+        clients = osclients.Clients(objects.Credential(**creds["admin"]))
 
     for client_component in components:
         try:
@@ -602,16 +604,17 @@ def required_openstack(config, clients, deployment, admin=False, users=False):
         return ValidationResult(
             False, _("You should specify admin=True or users=True or both."))
 
-    if deployment["admin"] and deployment["users"]:
+    creds = deployment.get_credentials_for("openstack")
+    if creds["admin"] and creds["users"]:
         return ValidationResult(True)
 
-    if deployment["admin"]:
+    if creds["admin"]:
         if users and not config.get("context", {}).get("users"):
             return ValidationResult(False,
                                     _("You should specify 'users' context"))
         return ValidationResult(True)
 
-    if deployment["users"] and admin:
+    if creds["users"] and admin:
         return ValidationResult(False, _("Admin credentials required"))
 
 

tests/functional/test_task_samples.py

@@ -60,7 +60,8 @@ class TestTaskSamples(unittest.TestCase):
 
         # let's use pre-created users to make TestTaskSamples quicker
         deployment = api.Deployment.get("MAIN")
-        admin_cred = objects.Credential(**deployment["admin"])
+        creds = deployment.get_credentials_for("openstack")
+        admin_cred = objects.Credential(**creds["admin"])
 
         ctx = {"admin": {"credential": admin_cred},
                "task": {"uuid": self.__class__.__name__}}

tests/unit/cli/commands/test_deployment.py

@@ -241,7 +241,8 @@ class DeploymentCommandsTestCase(test.TestCase):
             },
             "users": []
         }
-        self.fake_api.deployment.get.return_value = value
+        deployment = self.fake_api.deployment.get.return_value
+        deployment.get_credentials_for.return_value = value
         self.deployment.show(self.fake_api, deployment_id)
         self.fake_api.deployment.get.assert_called_once_with(deployment_id)
 
@@ -391,8 +392,9 @@ class DeploymentCommandsTestCase(test.TestCase):
                                                "admin",
                                                "adminpass").to_dict()
         sample_credential["not-exist-key"] = "error"
-        self.fake_api.deployment.get.return_value = {
+        deployment = self.fake_api.deployment.get.return_value
-            "admin": sample_credential}
+        deployment.get_credentials_for.return_value = {
+            "admin": sample_credential, "users": []}
         refused = keystone_exceptions.ConnectionRefused()
         self.fake_api.deployment.check.side_effect = refused
         self.assertEqual(self.deployment.check(

tests/unit/common/db/test_api.py

@@ -24,7 +24,6 @@ from six import moves
 
 from rally.common import db
 from rally.common.db import api as db_api
-import rally.common.db.sqlalchemy.api as s_api
 from rally import consts
 from rally import exceptions
 from tests.unit import test
@@ -90,25 +89,6 @@ class ConnectionTestCase(test.DBTestCase):
         self.assertEqual(drev["revision"], drev["current_head"])
 
 
-class FixDeploymentTestCase(test.DBTestCase):
-    def test_fix_deployment(self):
-        deployment = {
-            "credentials": [("bong", {"admin": "foo", "users": "bar"})]}
-
-        expected = {
-            "credentials": [("bong", {"admin": "foo", "users": "bar"})],
-            "admin": "foo",
-            "users": "bar"
-        }
-
-        @s_api.fix_deployment
-        def get_deployment():
-            return deployment
-
-        fixed_deployment = get_deployment()
-        self.assertEqual(fixed_deployment, expected)
-
-
 class TasksTestCase(test.DBTestCase):
     def setUp(self):
         super(TasksTestCase, self).setUp()
@@ -743,8 +723,7 @@ class DeploymentTestCase(test.DBTestCase):
         self.assertEqual(deploy["uuid"], deploys[0]["uuid"])
         self.assertEqual(deploy["status"], consts.DeployStatus.DEPLOY_INIT)
         self.assertEqual(deploy["config"], {"opt": "val"})
-        self.assertEqual(deploy["credentials"],
+        self.assertEqual(deploy["credentials"], {})
-                         [["openstack", {"admin": None, "users": []}]])
 
     def test_deployment_create_several(self):
         # Create a deployment
@@ -768,21 +747,21 @@ class DeploymentTestCase(test.DBTestCase):
         self.assertEqual(deploy_two["config"], {"opt2": "val2"})
 
     def test_deployment_update(self):
-        credentials = {"admin": {"foo": "bar"}, "users": ["foo_user"]}
+        credentials = {
-        deploy = db.deployment_create(copy.deepcopy(credentials))
+            "openstack": [{"admin": {"foo": "bar"}, "users": ["foo_user"]}]}
+        deploy = db.deployment_create({})
         self.assertEqual(deploy["config"], {})
-        self.assertEqual(deploy["credentials"], [["openstack", credentials]])
+        self.assertEqual(deploy["credentials"], {})
-        update_deploy = db.deployment_update(deploy["uuid"],
+        update_deploy = db.deployment_update(
-                                             {"config": {"opt": "val"}})
+            deploy["uuid"], {"config": {"opt": "val"},
+                             "credentials": copy.deepcopy(credentials)})
         self.assertEqual(update_deploy["uuid"], deploy["uuid"])
         self.assertEqual(update_deploy["config"], {"opt": "val"})
-        self.assertEqual(update_deploy["credentials"],
+        self.assertEqual(update_deploy["credentials"], credentials)
-                         [["openstack", credentials]])
         get_deploy = db.deployment_get(deploy["uuid"])
         self.assertEqual(get_deploy["uuid"], deploy["uuid"])
         self.assertEqual(get_deploy["config"], {"opt": "val"})
-        self.assertEqual(update_deploy["credentials"],
+        self.assertEqual(update_deploy["credentials"], credentials)
-                         [["openstack", credentials]])
 
     def test_deployment_update_several(self):
         # Create a deployment and update it

tests/unit/common/db/test_migrations.py

@@ -17,6 +17,7 @@
 
 import copy
 import json
+import pickle
 import pprint
 import uuid
 
@@ -1250,7 +1251,7 @@ class MigrationWalkTestCase(rtest.DBTestCase,
             conn.execute(
                 deployment_table.delete().where(
                     deployment_table.c.uuid ==
-                    self._37fdbb373e8d_deployment_uuid)
+                    self._f33f4610dcda_deployment_uuid)
             )
 
     def _pre_upgrade_4ef544102ba7(self, engine):
@@ -1369,4 +1370,50 @@ class MigrationWalkTestCase(rtest.DBTestCase,
                 deployment_table.delete().where(
                     deployment_table.c.uuid ==
                     self._4ef544102ba7_deployment_uuid)
-            )
+            )
+
+    def _pre_upgrade_92aaaa2a6bb3(self, engine):
+        self._92aaaa2a6bb3_deployments = [
+            ("1-cred", [["openstack", {"foo": "bar"}]]),
+            ("2-cred", [["openstack", {"foo": "bar1"}],
+                        ["openstack", {"foo": "bar2"}]]),
+            ("multi-cred", [["spam", {"foo": "bar1"}],
+                            ["eggs", {"foo": "bar2"}]]),
+        ]
+
+        deployment_table = db_utils.get_table(engine, "deployments")
+        deployment_status = consts.DeployStatus.DEPLOY_FINISHED
+
+        with engine.connect() as conn:
+            for deployment, creds in self._92aaaa2a6bb3_deployments:
+                conn.execute(
+                    deployment_table.insert(),
+                    [{"uuid": deployment, "name": deployment,
+                      "config": json.dumps({}),
+                      "enum_deployments_status": deployment_status,
+                      "credentials": pickle.dumps(creds),
+                      }])
+
+    def _check_92aaaa2a6bb3(self, engine, data):
+        expected_credentials = [
+            ("1-cred", {"openstack": [{"foo": "bar"}]}),
+            ("2-cred", {"openstack": [{"foo": "bar1"},
+                                      {"foo": "bar2"}]}),
+            ("multi-cred", {"spam": [{"foo": "bar1"}],
+                            "eggs": [{"foo": "bar2"}]}),
+        ]
+
+        deployment_table = db_utils.get_table(engine, "deployments")
+
+        with engine.connect() as conn:
+            for deployment, expected_creds in expected_credentials:
+
+                dep_obj = conn.execute(
+                    deployment_table.select().where(
+                        deployment_table.c.uuid == deployment)).fetchone()
+                self.assertEqual(
+                    expected_creds, json.loads(dep_obj.credentials))
+
+                conn.execute(
+                    deployment_table.delete().where(
+                        deployment_table.c.uuid == deployment))

tests/unit/common/objects/test_deploy.py

@@ -15,11 +15,12 @@
 
 """Tests for db.deploy layer."""
 
-
+import jsonschema
 import mock
 
 from rally.common import objects
 from rally import consts
+from rally import exceptions
 from tests.unit import test
 
 
@@ -30,7 +31,7 @@ class DeploymentTestCase(test.TestCase):
             "uuid": "baa1bfb6-0c38-4f6c-9bd0-45968890e4f4",
             "name": "",
             "config": {},
-            "endpoint": {},
+            "credentials": {},
             "status": consts.DeployStatus.DEPLOY_INIT,
         }
         self.resource = {
@@ -120,37 +121,44 @@ class DeploymentTestCase(test.TestCase):
     def test_update_credentials(self, mock_deployment_update):
         mock_deployment_update.return_value = self.deployment
         deploy = objects.Deployment(deployment=self.deployment)
-        credentials = {
+        credentials = {"foo": [{"admin": {"fake_admin": True},
-            "admin": objects.Credential("url", "user", "pwd", "tenant",
+                                "users": [{"fake_user": True}]}]}
-                                        consts.EndpointPermission.ADMIN),
-            "users": [
-                objects.Credential("url1", "user1", "pwd1", "tenant1",
-                                   consts.EndpointPermission.USER),
-                objects.Credential("url2", "user2", "pwd2", "tenant2",
-                                   consts.EndpointPermission.USER),
-            ]
-        }
 
-        expected_users = [u.to_dict(include_permission=True)
-                          for u in credentials["users"]]
-        expected_admin = credentials["admin"].to_dict(include_permission=True)
         deploy.update_credentials(credentials)
         mock_deployment_update.assert_called_once_with(
             self.deployment["uuid"],
             {
-                "credentials": [["openstack", {"admin": expected_admin,
+                "credentials": {"foo": [{"admin": {"fake_admin": True},
-                                               "users": expected_users}]]
+                                         "users": [{"fake_user": True}]}]}
             })
 
-    @mock.patch("rally.common.objects.deploy.db.deployment_update")
+    def test_get_credentials_for(self):
-    def test_update_empty_credentials(self, mock_deployment_update):
+        credentials = {"foo": [{"admin": {"fake_admin": True},
-        mock_deployment_update.return_value = self.deployment
+                                "users": [{"fake_user": True}]}]}
+        self.deployment["credentials"] = credentials
         deploy = objects.Deployment(deployment=self.deployment)
-        deploy.update_credentials({})
+
-        mock_deployment_update.assert_called_once_with(
+        creds = deploy.get_credentials_for("foo")
-            self.deployment["uuid"], {
+        self.assertEqual(credentials["foo"][0], creds)
-                "credentials": [["openstack", {"admin": {}, "users": []}]]
+
-            })
+    def test_get_deprecated(self):
+        credentials = {"openstack": [{"admin": {"fake_admin": True},
+                                      "users": [{"fake_user": True}]}]}
+        self.deployment["credentials"] = credentials
+        deploy = objects.Deployment(deployment=self.deployment)
+
+        self.assertEqual(credentials["openstack"][0]["admin"], deploy["admin"])
+        self.assertEqual(credentials["openstack"][0]["users"], deploy["users"])
+
+    def test_update_empty_credentials(self):
+        deploy = objects.Deployment(deployment=self.deployment)
+        self.assertRaises(jsonschema.ValidationError,
+                          deploy.update_credentials, {})
+
+    def test_get_credentials_error(self):
+        deploy = objects.Deployment(deployment=self.deployment)
+        self.assertRaises(exceptions.RallyException,
+                          deploy.get_credentials_for, "bar")
 
     @mock.patch("rally.common.objects.deploy.db.resource_create")
     def test_add_resource(self, mock_resource_create):

tests/unit/deployment/engines/test_devstack.py

@@ -84,16 +84,20 @@ class DevstackEngineTestCase(test.TestCase):
             mock.Mock()
         )
         server = mock.Mock(host="host")
-        mock_credential.return_value = "fake_credential"
+        mock_credential.return_value.to_dict.return_value = "fake_credential"
         mock_get_updated_server.return_value = ds_server = mock.Mock()
         mock_get_script.return_value = "fake_script"
         server.get_credentials.return_value = "fake_credentials"
         fake_provider.create_servers.return_value = [server]
         with mock.patch.object(self.engine, "deployment") as mock_deployment:
             credentials = self.engine.deploy()
-        self.assertEqual({"admin": "fake_credential"}, credentials)
+        self.assertEqual(
+            {"openstack": [{"admin": "fake_credential", "users": []}]},
+            credentials)
         mock_credential.assert_called_once_with(
             "http://host:5000/v2.0/", "admin", "secret", "admin", "admin")
+        mock_credential.return_value.to_dict.assert_called_once_with(
+            include_permission=True)
         mock_deployment.add_resource.assert_called_once_with(
             info="fake_credentials",
             provider_name="DevstackEngine",

tests/unit/deployment/engines/test_existing.py

@@ -94,12 +94,14 @@ class TestExistingCloud(test.TestCase):
         deployment = self.deployments[keystone_version]
         engine = existing.ExistingCloud(deployment)
         credentials = engine.deploy()
+        credentials = credentials["openstack"][0]
         admin_credential = deployment["config"].copy()
         admin_credential.pop("type")
         admin_credential["endpoint"] = None
         admin_credential.update(admin_credential.pop("admin"))
+        admin_credential["permission"] = consts.EndpointPermission.ADMIN
 
-        actual_credentials = credentials["admin"].to_dict()
+        actual_credentials = credentials["admin"]
 
         if keystone_version == "v3":
             # NOTE(andreykurilin): credentials obj uses `tenant_name` for both

tests/unit/deployment/test_lxc.py

@@ -15,7 +15,6 @@
 
 import mock
 
-from rally.common import objects
 from rally.deployment import engine
 from tests.unit import test
 
@@ -145,9 +144,15 @@ class LxcEngineTestCase(test.TestCase):
         fake_deployment.add_resource = add_resource
 
         with mock.patch.object(self.engine, "deployment", fake_deployment):
-            credential = self.engine.deploy()
+            credentials = self.engine.deploy()
 
-        self.assertIsInstance(credential["admin"], objects.Credential)
+        self.assertEqual(1, len(credentials.keys()))
+        self.assertIn("openstack", credentials)
+        self.assertEqual(1, len(credentials["openstack"]))
+        credential = credentials["openstack"][0]
+
+        self.assertIsInstance(credential["admin"], dict)
+        self.assertEqual([], credential["users"])
         lxc_host_calls = [
             mock.call(fake_servers[0], {"network": "10.128.128.0/28",
                                         "tunnel_to": ["1.1.1.1", "2.2.2.2"]}),

tests/unit/fakes.py

@@ -1827,6 +1827,16 @@ class FakeUserContext(FakeContext):
 class FakeDeployment(dict):
     update_status = mock.Mock()
 
+    def __init__(self, **kwargs):
+        namespace = kwargs.pop("namespace", "openstack")
+        kwargs["credentials"] = {
+            namespace: [{"admin": kwargs.pop("admin", None),
+                         "users": kwargs.pop("users", [])}]}
+        dict.__init__(self, **kwargs)
+
+    def get_credentials_for(self, namespace):
+        return self["credentials"][namespace][0]
+
 
 class FakeTask(dict):
 

tests/unit/plugins/openstack/verification/tempest/test_config.py

@@ -18,6 +18,7 @@ import mock
 from oslo_config import cfg
 
 from rally.plugins.openstack.verification.tempest import config
+from tests.unit import fakes
 from tests.unit import test
 
 
@@ -51,7 +52,8 @@ class TempestConfigfileManagerTestCase(test.TestCase):
 
         mock.patch("rally.osclients.Clients").start()
 
-        self.tempest = config.TempestConfigfileManager(CREDS)
+        deployment = fakes.FakeDeployment(**CREDS)
+        self.tempest = config.TempestConfigfileManager(deployment)
 
     def test__configure_auth(self):
         self.tempest.conf.add_section("auth")

tests/unit/plugins/openstack/verification/tempest/test_context.py

@@ -59,7 +59,8 @@ class TempestContextTestCase(test.TestCase):
         self.mock_isfile = mock.patch("os.path.isfile",
                                       return_value=True).start()
 
-        cfg = {"verifier": mock.Mock(deployment=CREDS),
+        self.deployment = fakes.FakeDeployment(**CREDS)
+        cfg = {"verifier": mock.Mock(deployment=self.deployment),
                "verification": {"uuid": "uuid"}}
         cfg["verifier"].manager.home_dir = "/p/a/t/h"
         cfg["verifier"].manager.configfile = "/fake/path/to/config"
@@ -336,11 +337,13 @@ class TempestContextTestCase(test.TestCase):
     def test_setup(self, mock_clients, mock_create_dir,
                    mock__create_tempest_roles, mock__configure_option,
                    mock_open):
-        verifier = mock.MagicMock(deployment=CREDS)
+        self.deployment = fakes.FakeDeployment(**CREDS)
+        verifier = mock.Mock(deployment=self.deployment)
         verifier.manager.home_dir = "/p/a/t/h"
 
         # case #1: no neutron and heat
         mock_clients.return_value.services.return_value = {}
+
         ctx = context.TempestContext({"verifier": verifier})
         ctx.conf = mock.Mock()
         ctx.setup()
@@ -376,6 +379,7 @@ class TempestContextTestCase(test.TestCase):
         # case #2: neutron and heat are presented
         mock_clients.return_value.services.return_value = {
             "network": "neutron", "orchestration": "heat"}
+
         ctx = context.TempestContext({"verifier": verifier})
         ctx.conf = mock.Mock()
         ctx.setup()

tests/unit/task/test_validation.py

@@ -26,6 +26,7 @@ from rally import consts
 from rally import exceptions
 import rally.osclients
 from rally.task import validation
+from tests.unit import fakes
 from tests.unit import test
 
 
@@ -572,7 +573,8 @@ class ValidatorsTestCase(test.TestCase):
         with mock.patch("rally.osclients.Clients") as clients_cls:
             nova_client = clients_cls.return_value.nova.return_value
             nova_client.services.list.return_value = [fake_service]
-            result = validator({}, clients, {"admin": {"info": "admin"}})
+            deployment = fakes.FakeDeployment(admin={"info": "admin"})
+            result = validator({}, clients, deployment)
             clients_cls.assert_called_once_with(mock_credential.return_value)
             mock_credential.assert_called_once_with(info="admin")
         self.assertTrue(result.is_valid, result.msg)
@@ -666,11 +668,11 @@ class ValidatorsTestCase(test.TestCase):
                                            admin=True)
 
         # admin presented in deployment
-        fake_deployment = {"admin": "admin_credential", "users": []}
+        fake_deployment = fakes.FakeDeployment(admin="admin_credential")
         self.assertTrue(validator(None, None, fake_deployment).is_valid)
 
         # admin not presented in deployment
-        fake_deployment = {"admin": None, "users": ["u1", "h2"]}
+        fake_deployment = fakes.FakeDeployment(users=["u1", "h2"])
         self.assertFalse(validator(None, None, fake_deployment).is_valid)
 
     def test_required_openstack_with_users(self):
@@ -678,34 +680,35 @@ class ValidatorsTestCase(test.TestCase):
                                            users=True)
 
         # users presented in deployment
-        fake_deployment = {"admin": None, "users": ["u_credential"]}
+        fake_deployment = fakes.FakeDeployment(
+            admin=None, users=["u_credential"])
         self.assertTrue(validator({}, None, fake_deployment).is_valid)
 
         # admin and users presented in deployment
-        fake_deployment = {"admin": "a", "users": ["u1", "h2"]}
+        fake_deployment = fakes.FakeDeployment(admin="a", users=["u1", "h2"])
         self.assertTrue(validator({}, None, fake_deployment).is_valid)
 
         # admin and user context
-        fake_deployment = {"admin": "a", "users": []}
+        fake_deployment = fakes.FakeDeployment(admin="a", users=[])
         context = {"context": {"users": True}}
         self.assertTrue(validator(context, None, fake_deployment).is_valid)
 
         # just admin presented
-        fake_deployment = {"admin": "a", "users": []}
+        fake_deployment = fakes.FakeDeployment(admin="a", users=[])
         self.assertFalse(validator({}, None, fake_deployment).is_valid)
 
     def test_required_openstack_with_admin_and_users(self):
         validator = self._unwrap_validator(validation.required_openstack,
                                            admin=True, users=True)
 
-        fake_deployment = {"admin": "a", "users": []}
+        fake_deployment = fakes.FakeDeployment(admin="a", users=[])
         self.assertFalse(validator({}, None, fake_deployment).is_valid)
 
-        fake_deployment = {"admin": "a", "users": ["u"]}
+        fake_deployment = fakes.FakeDeployment(admin="a", users=["u"])
         self.assertTrue(validator({}, None, fake_deployment).is_valid)
 
         # admin and user context
-        fake_deployment = {"admin": "a", "users": []}
+        fake_deployment = fakes.FakeDeployment(admin="a", users=[])
         context = {"context": {"users": True}}
         self.assertTrue(validator(context, None, fake_deployment).is_valid)
 
@@ -756,12 +759,13 @@ class ValidatorsTestCase(test.TestCase):
         clients = mock.MagicMock()
         clients.keystone.return_value = "keystone"
         clients.nova.return_value = "nova"
-        result = validator({}, clients, {})
+        deployment = fakes.FakeDeployment()
+        result = validator({}, clients, deployment)
         self.assertTrue(result.is_valid, result.msg)
         self.assertFalse(mock_osclients.Clients.called)
 
         clients.nova.side_effect = ImportError
-        result = validator({}, clients, {})
+        result = validator({}, clients, deployment)
         self.assertFalse(result.is_valid, result.msg)
 
     @mock.patch(MODULE + "objects")
@@ -774,12 +778,13 @@ class ValidatorsTestCase(test.TestCase):
         clients.nova.return_value = "nova"
         mock_osclients.Clients.return_value = clients
         mock_objects.Credential.return_value = "foo_credential"
-        result = validator({}, clients, {"admin": {"foo": "bar"}})
+        deployment = fakes.FakeDeployment(admin={"foo": "bar"})
+        result = validator({}, clients, deployment)
         self.assertTrue(result.is_valid, result.msg)
         mock_objects.Credential.assert_called_once_with(foo="bar")
         mock_osclients.Clients.assert_called_once_with("foo_credential")
         clients.nova.side_effect = ImportError
-        result = validator({}, clients, {"admin": {"foo": "bar"}})
+        result = validator({}, clients, deployment)
         self.assertFalse(result.is_valid, result.msg)
 
     @ddt.data(
@@ -813,9 +818,10 @@ class ValidatorsTestCase(test.TestCase):
             cinder_client.services = services
             c.return_value = cinder_client
 
-            deployment = {"admin": {"auth_url": "fake_credential",
+            deployment = fakes.FakeDeployment(
-                                    "username": "username",
+                admin={"auth_url": "fake_credential",
-                                    "password": "password"}}
+                       "username": "username",
+                       "password": "password"})
             result = validator({}, None, deployment)
             self.assertTrue(result.is_valid, result.msg)
 

tests/unit/test_api.py

@@ -61,8 +61,8 @@ class TaskAPITestCase(test.TestCase):
     @mock.patch("rally.api.objects.Task")
     @mock.patch("rally.api.objects.Deployment.get",
                 return_value=fakes.FakeDeployment(uuid="deployment_uuid",
-                                                  admin=mock.MagicMock(),
+                                                  admin="fake_admin",
-                                                  users=[]))
+                                                  users=["fake_user"]))
     @mock.patch("rally.api.engine.TaskEngine")
     def test_validate(
             self, mock_task_engine, mock_deployment_get, mock_task):
@@ -70,8 +70,8 @@ class TaskAPITestCase(test.TestCase):
 
         mock_task_engine.assert_has_calls([
             mock.call("config", mock_task.return_value,
-                      admin=mock_deployment_get.return_value["admin"],
+                      admin="fake_admin",
-                      users=[]),
+                      users=["fake_user"]),
             mock.call().validate()
         ])
 
@@ -83,9 +83,7 @@ class TaskAPITestCase(test.TestCase):
 
     @mock.patch("rally.api.objects.Task")
     @mock.patch("rally.api.objects.Deployment",
-                return_value=fakes.FakeDeployment(uuid="deployment_uuid",
+                return_value=fakes.FakeDeployment(uuid="deployment_uuid"))
-                                                  admin=mock.MagicMock(),
-                                                  users=[]))
     @mock.patch("rally.api.engine.TaskEngine")
     def test_validate_engine_exception(self, mock_task_engine,
                                        mock_deployment, mock_task):
@@ -178,8 +176,8 @@ class TaskAPITestCase(test.TestCase):
                 return_value=fakes.FakeTask(uuid="some_uuid"))
     @mock.patch("rally.api.objects.Deployment.get",
                 return_value=fakes.FakeDeployment(uuid="deployment_uuid",
-                                                  admin=mock.MagicMock(),
+                                                  admin="fake_admin",
-                                                  users=[]))
+                                                  users=["fake_user"]))
     @mock.patch("rally.api.engine.TaskEngine")
     def test_start(self, mock_task_engine, mock_deployment_get,
                    mock_task):
@@ -187,8 +185,8 @@ class TaskAPITestCase(test.TestCase):
 
         mock_task_engine.assert_has_calls([
             mock.call("config", mock_task.return_value,
-                      admin=mock_deployment_get.return_value["admin"],
+                      admin="fake_admin", users=["fake_user"],
-                      users=[], abort_on_sla_failure=False),
+                      abort_on_sla_failure=False),
             mock.call().run(),
         ])
 
@@ -203,8 +201,8 @@ class TaskAPITestCase(test.TestCase):
                                             temporary=True))
     @mock.patch("rally.api.objects.Deployment.get",
                 return_value=fakes.FakeDeployment(uuid="deployment_uuid",
-                                                  admin=mock.MagicMock(),
+                                                  admin="fake_admin",
-                                                  users=[]))
+                                                  users=["fake_user"]))
     def test_start_temporary_task(self, mock_deployment_get,
                                   mock_task):
 
@@ -331,9 +329,9 @@ class TaskAPITestCase(test.TestCase):
 class BaseDeploymentTestCase(test.TestCase):
     def setUp(self):
         super(BaseDeploymentTestCase, self).setUp()
-        self.deployment_config = FAKE_DEPLOYMENT_CONFIG
+        self.deployment_config = copy.deepcopy(FAKE_DEPLOYMENT_CONFIG)
         self.deployment_uuid = "599bdf1d-fe77-461a-a810-d59b1490f4e3"
-        admin_credential = FAKE_DEPLOYMENT_CONFIG.copy()
+        admin_credential = copy.deepcopy(FAKE_DEPLOYMENT_CONFIG)
         admin_credential.pop("type")
         admin_credential["endpoint"] = None
         admin_credential.update(admin_credential.pop("admin"))
@@ -345,8 +343,7 @@ class BaseDeploymentTestCase(test.TestCase):
             "uuid": self.deployment_uuid,
             "name": "fake_name",
             "config": self.deployment_config,
-            "admin": self.credentials["admin"],
+            "credentials": {"openstack": [self.credentials]}
-            "users": []
         }
 
 
@@ -358,18 +355,16 @@ class DeploymentAPITestCase(BaseDeploymentTestCase):
                     mock_deployment_create, mock_deployment_update):
         mock_deployment_create.return_value = self.deployment
         mock_deployment_update.return_value = self.deployment
-        api._Deployment.create(self.deployment_config, "fake_deployment")
+        dep = api._Deployment.create(self.deployment_config, "fake_deployment")
+        self.assertIsInstance(dep, objects.Deployment)
         mock_deployment_create.assert_called_once_with({
             "name": "fake_deployment",
             "config": self.deployment_config,
         })
-        mock_engine_validate.assert_called_with()
+        mock_engine_validate.assert_called_once_with()
         mock_deployment_update.assert_has_calls([
             mock.call(self.deployment_uuid,
-                      {"credentials": [["openstack",
+                      {"credentials": {"openstack": [self.credentials]}})
-                                        {"admin": self.credentials["admin"],
-                                         "users": self.credentials["users"]}]]
-                       })
         ])
 
     @mock.patch("rally.common.objects.deploy.db.deployment_update")
@@ -428,11 +423,11 @@ class DeploymentAPITestCase(BaseDeploymentTestCase):
         api._Deployment.recreate(self.deployment_uuid)
         mock_deployment_get.assert_called_once_with(self.deployment_uuid)
         mock_deployment_update.assert_has_calls([
-            mock.call(self.deployment_uuid,
+            mock.call(
-                      {"credentials": [["openstack",
+                self.deployment_uuid,
-                                        {"admin": self.credentials["admin"],
+                {"credentials":
-                                         "users": self.credentials["users"]}]]
+                    {"openstack": [{"admin": self.credentials["admin"],
-                       })
+                                    "users": self.credentials["users"]}]}})
         ])
 
     @mock.patch("rally.common.objects.deploy.db.deployment_update")
@@ -501,8 +496,9 @@ class DeploymentAPITestCase(BaseDeploymentTestCase):
         sample_credential = objects.Credential("http://192.168.1.1:5000/v2.0/",
                                                "admin",
                                                "adminpass").to_dict()
-        deployment = {"admin": sample_credential,
+        deployment = mock.Mock(spec=objects.Deployment)
-                      "users": [sample_credential]}
+        deployment.get_credentials_for.return_value = {
+            "admin": sample_credential, "users": [sample_credential]}
         api._Deployment.check(deployment)
         mock_keystone_create_client.assert_called_with()
         mock_clients_services.assert_called_once_with()
@@ -512,7 +508,7 @@ class DeploymentAPITestCase(BaseDeploymentTestCase):
                                                "admin",
                                                "adminpass").to_dict()
         sample_credential["not-exist-key"] = "error"
-        deployment = {"admin": sample_credential}
+        deployment = mock.Mock(spec=objects.Deployment)
         self.assertRaises(TypeError, api._Deployment.check, deployment)
 
     @mock.patch("rally.osclients.Clients.services")
@@ -520,7 +516,9 @@ class DeploymentAPITestCase(BaseDeploymentTestCase):
         sample_credential = objects.Credential("http://192.168.1.1:5000/v2.0/",
                                                "admin",
                                                "adminpass").to_dict()
-        deployment = {"admin": sample_credential}
+        deployment = mock.Mock(spec=objects.Deployment)
+        deployment.get_credentials_for.return_value = {
+            "admin": sample_credential, "users": []}
         refused = keystone_exceptions.ConnectionRefused()
         mock_clients_services.side_effect = refused
         self.assertRaises(