Add support of datastore types

implements blueprint db-type-version

Change-Id: Ie87f72b898e993044803e7b37cad78372c2cd3f4

bin/trove-guestagent

@@ -53,9 +53,10 @@ if __name__ == '__main__':
 
     try:
         get_db_api().configure_db(CONF)
-        manager = dbaas.service_registry().get(CONF.service_type)
+        manager = dbaas.datastore_registry().get(CONF.datastore_manager)
         if not manager:
-            msg = "Manager not found for service type " + CONF.service_type
+            msg = ("Manager class not registered for datastore manager " +
+                   CONF.datastore_manager)
             raise RuntimeError(msg)
         server = rpc_service.RpcService(manager=manager, host=CONF.guest_id)
         launcher = openstack_service.launch(server)

bin/trove-manage

@@ -36,11 +36,13 @@ if os.path.exists(os.path.join(possible_topdir, 'trove', '__init__.py')):
 
 from trove import version
 from trove.common import cfg
+from trove.common import exception
 from trove.common import utils
 from trove.db import get_db_api
 from trove.openstack.common import log as logging
 from trove.openstack.common import uuidutils
 from trove.instance import models as instance_models
+from trove.datastore import models as datastore_models
 
 
 CONF = cfg.CONF
@@ -69,28 +71,30 @@ class Commands(object):
             kwargs[arg] = getattr(CONF.action, arg)
         exec_method(**kwargs)
 
-    def image_update(self, service_name, image_id):
-        self.db_api.configure_db(CONF)
-        image = self.db_api.find_by(instance_models.ServiceImage,
-                                    service_name=service_name)
-        if image is None:
-            # Create a new one
-            image = instance_models.ServiceImage()
-            image.id = uuidutils.generate_uuid()
-            image.service_name = service_name
-        image.image_id = image_id
-        self.db_api.save(image)
+    def datastore_update(self, datastore_name, manager, default_version):
+        try:
+            datastore_models.update_datastore(datastore_name, manager,
+                                              default_version)
+            print("Datastore '%s' updated." % datastore_name)
+        except exception.DatastoreVersionNotFound as e:
+            print(e)
 
-    def db_wipe(self, repo_path, service_name, image_id):
+    def datastore_version_update(self, datastore, version_name, image_id,
+                                 packages, active):
+        try:
+            datastore_models.update_datastore_version(datastore,
+                                                      version_name, image_id,
+                                                      packages, active)
+            print("Datastore version '%s' updated." % version_name)
+        except exception.DatastoreNotFound as e:
+            print(e)
+
+    def db_wipe(self, repo_path):
         """Drops the database and recreates it."""
         from trove.instance import models
         from trove.db.sqlalchemy import session
         self.db_api.drop_db(CONF)
         self.db_sync()
-        # Sets up database engine, so the next line will work...
-        session.configure_db(CONF)
-        models.ServiceImage.create(service_name=service_name,
-                                   image_id=image_id)
 
     def params_of(self, command_name):
         if Commands.has(command_name):
@@ -108,13 +112,18 @@ if __name__ == '__main__':
         parser = subparser.add_parser('db_downgrade')
         parser.add_argument('version')
         parser.add_argument('--repo_path')
-        parser = subparser.add_parser('image_update')
-        parser.add_argument('service_name')
+        parser = subparser.add_parser('datastore_update')
+        parser.add_argument('datastore_name')
+        parser.add_argument('manager')
+        parser.add_argument('default_version')
+        parser = subparser.add_parser('datastore_version_update')
+        parser.add_argument('datastore')
+        parser.add_argument('version_name')
         parser.add_argument('image_id')
+        parser.add_argument('packages')
+        parser.add_argument('active')
         parser = subparser.add_parser('db_wipe')
         parser.add_argument('repo_path')
-        parser.add_argument('service_name')
-        parser.add_argument('image_id')
 
     cfg.custom_parser('action', actions)
     cfg.parse_args(sys.argv)

etc/trove/trove.conf.test

@@ -92,6 +92,9 @@ http_post_rate = 200
 http_put_rate = 200
 http_delete_rate = 200
 
+# default datastore
+default_datastore = a00000a0-00a0-0a00-00a0-000a000000aa
+
 # Auth
 admin_roles = admin
 

run_tests.py

@@ -52,16 +52,41 @@ def initialize_trove(config_file):
     return pastedeploy.paste_deploy_app(config_file, 'trove', {})
 
 
+def datastore_init():
+    # Adds the datastore for mysql (needed to make most calls work).
+    from trove.datastore import models
+    models.DBDatastore.create(id="a00000a0-00a0-0a00-00a0-000a000000aa",
+                              name=CONFIG.dbaas_datastore, manager='mysql',
+                              default_version_id=
+                              "b00000b0-00b0-0b00-00b0-000b000000bb")
+    models.DBDatastore.create(id="e00000e0-00e0-0e00-00e0-000e000000ee",
+                              name='Test_Datastore_1', manager='manager1',
+                              default_version_id=None)
+    models.DBDatastoreVersion.create(id="b00000b0-00b0-0b00-00b0-000b000000bb",
+                                     datastore_id=
+                                     "a00000a0-00a0-0a00-00a0-000a000000aa",
+                                     name=CONFIG.dbaas_datastore_version,
+                                     image_id=
+                                     'c00000c0-00c0-0c00-00c0-000c000000cc',
+                                     packages='test packages',
+                                     active=1)
+    models.DBDatastoreVersion.create(id="d00000d0-00d0-0d00-00d0-000d000000dd",
+                                     datastore_id=
+                                     "a00000a0-00a0-0a00-00a0-000a000000aa",
+                                     name='mysql_inactive_version',
+                                     image_id=
+                                     'c00000c0-00c0-0c00-00c0-000c000000cc',
+                                     packages=None, active=0)
+
+
 def initialize_database():
     from trove.db import get_db_api
-    from trove.instance import models
     from trove.db.sqlalchemy import session
     db_api = get_db_api()
     db_api.drop_db(CONF)  # Destroys the database, if it exists.
     db_api.db_sync(CONF)
     session.configure_db(CONF)
-    # Adds the image for mysql (needed to make most calls work).
-    models.ServiceImage.create(service_name="mysql", image_id="fake")
+    datastore_init()
     db_api.configure_db(CONF)
 
 
@@ -125,6 +150,7 @@ if __name__ == "__main__":
         from trove.tests.api import instances_mysql_down
         from trove.tests.api import instances_resize
         from trove.tests.api import databases
+        from trove.tests.api import datastores
         from trove.tests.api import root
         from trove.tests.api import root_on_create
         from trove.tests.api import users

trove/common/api.py

@@ -20,6 +20,7 @@ from trove.instance.service import InstanceController
 from trove.limits.service import LimitsController
 from trove.backup.service import BackupController
 from trove.versions import VersionsController
+from trove.datastore.service import DatastoreController
 
 
 class API(wsgi.Router):
@@ -28,6 +29,7 @@ class API(wsgi.Router):
         mapper = routes.Mapper()
         super(API, self).__init__(mapper)
         self._instance_router(mapper)
+        self._datastore_router(mapper)
         self._flavor_router(mapper)
         self._versions_router(mapper)
         self._limits_router(mapper)
@@ -37,6 +39,17 @@ class API(wsgi.Router):
         versions_resource = VersionsController().create_resource()
         mapper.connect("/", controller=versions_resource, action="show")
 
+    def _datastore_router(self, mapper):
+        datastore_resource = DatastoreController().create_resource()
+        mapper.resource("datastore", "/{tenant_id}/datastores",
+                        controller=datastore_resource)
+        mapper.connect("/{tenant_id}/datastores/{datastore}/versions",
+                       controller=datastore_resource,
+                       action="version_index")
+        mapper.connect("/{tenant_id}/datastores/{datastore}/versions/{id}",
+                       controller=datastore_resource,
+                       action="version_show")
+
     def _instance_router(self, mapper):
         instance_resource = InstanceController().create_resource()
         path = "/{tenant_id}/instances"

trove/common/apischema.py

@@ -183,7 +183,6 @@ instance = {
                     "volume": volume,
                     "databases": databases_def,
                     "users": users_list,
-                    "service_type": non_empty_string,
                     "restorePoint": {
                         "type": "object",
                         "required": ["backupRef"],
@@ -192,7 +191,15 @@ instance = {
                             "backupRef": uuid
                         }
                     },
-                    "availability_zone": non_empty_string
+                    "availability_zone": non_empty_string,
+                    "datastore": {
+                        "type": "object",
+                        "additionalProperties": True,
+                        "properties": {
+                            "type": non_empty_string,
+                            "version": non_empty_string
+                        }
+                    }
                 }
             }
         }

trove/common/cfg.py

@@ -64,8 +64,6 @@ common_opts = [
     cfg.IntOpt('periodic_interval', default=60),
     cfg.BoolOpt('trove_dns_support', default=False),
     cfg.StrOpt('db_api_implementation', default='trove.db.sqlalchemy.api'),
-    cfg.StrOpt('mysql_pkg', default='mysql-server-5.5'),
-    cfg.StrOpt('percona_pkg', default='percona-server-server-5.5'),
     cfg.StrOpt('dns_driver', default='trove.dns.driver.DnsDriver'),
     cfg.StrOpt('dns_instance_entry_factory',
                default='trove.dns.driver.DnsInstanceEntryFactory'),
@@ -111,13 +109,18 @@ common_opts = [
     cfg.BoolOpt('use_heat', default=False),
     cfg.StrOpt('device_path', default='/dev/vdb'),
     cfg.StrOpt('mount_point', default='/var/lib/mysql'),
-    cfg.StrOpt('service_type', default='mysql'),
+    cfg.StrOpt('default_datastore', default=None,
+               help="The default datastore id or name to use if one is not "
+               "provided by the user. If the default value is None, the field"
+               " becomes required in the instance-create request."),
+    cfg.StrOpt('datastore_manager', default=None,
+               help='manager class in guestagent, setup by taskmanager on '
+               'instance provision'),
     cfg.StrOpt('block_device_mapping', default='vdb'),
     cfg.IntOpt('server_delete_time_out', default=60),
     cfg.IntOpt('volume_time_out', default=60),
     cfg.IntOpt('heat_time_out', default=60),
     cfg.IntOpt('reboot_time_out', default=60 * 2),
-    cfg.StrOpt('service_options', default=['mysql']),
     cfg.IntOpt('dns_time_out', default=60 * 2),
     cfg.IntOpt('resize_time_out', default=60 * 10),
     cfg.IntOpt('revert_time_out', default=60 * 10),
@@ -212,10 +215,10 @@ common_opts = [
     cfg.StrOpt('guest_config',
                default='$pybasedir/etc/trove/trove-guestagent.conf.sample',
                help="Path to guestagent config file"),
-    cfg.DictOpt('service_registry_ext', default=dict(),
-                help='Extention for default service managers.'
+    cfg.DictOpt('datastore_registry_ext', default=dict(),
+                help='Extention for default datastore managers.'
                      ' Allows to use custom managers for each of'
-                     ' service type supported in trove'),
+                     ' datastore supported in trove'),
     cfg.StrOpt('template_path',
                default='/etc/trove/templates/',
                help='Path which leads to datastore templates'),

trove/common/exception.py

@@ -96,6 +96,41 @@ class DnsRecordNotFound(NotFound):
     message = _("DnsRecord with name= %(name)s not found.")
 
 
+class DatastoreNotFound(NotFound):
+
+    message = _("Datastore '%(datastore)s' cannot be found.")
+
+
+class DatastoreVersionNotFound(NotFound):
+
+    message = _("Datastore version '%(version)s' cannot be found.")
+
+
+class DatastoresNotFound(NotFound):
+
+    message = _("Datastores cannot be found.")
+
+
+class DatastoreNoVersion(TroveError):
+
+    message = _("Datastore '%(datastore)s' has no version '%(version)s'.")
+
+
+class DatastoreVersionInactive(TroveError):
+
+    message = _("Datastore version '%(version)s' is not active.")
+
+
+class DatastoreDefaultDatastoreNotFound(TroveError):
+
+    message = _("Please specify datastore.")
+
+
+class DatastoreDefaultVersionNotFound(TroveError):
+
+    message = _("Default version for datastore '%(datastore)s' not found.")
+
+
 class OverLimit(TroveError):
 
     internal_message = _("The server rejected the request due to its size or "

trove/common/template.py

@@ -30,10 +30,10 @@ class SingleInstanceConfigTemplate(object):
     """ This class selects a single configuration file by database type for
     rendering on the guest """
 
-    def __init__(self, datastore_type, flavor_dict, instance_id):
+    def __init__(self, datastore_manager, flavor_dict, instance_id):
         """ Constructor
 
-        :param datastore_type: The database type.
+        :param datastore_manager: The datastore manager.
         :type name: str.
         :param flavor_dict: dict containing flavor details for use in jinja.
         :type flavor_dict: dict.
@@ -42,7 +42,7 @@ class SingleInstanceConfigTemplate(object):
 
         """
         self.flavor_dict = flavor_dict
-        template_filename = "%s/config.template" % datastore_type
+        template_filename = "%s/config.template" % datastore_manager
         self.template = ENV.get_template(template_filename)
         self.instance_id = instance_id
 
@@ -66,12 +66,12 @@ class SingleInstanceConfigTemplate(object):
         return abs(hash(self.instance_id) % (2 ** 31))
 
 
-def load_heat_template(datastore_type):
-    template_filename = "%s/heat.template" % datastore_type
+def load_heat_template(datastore_manager):
+    template_filename = "%s/heat.template" % datastore_manager
     try:
         template_obj = ENV.get_template(template_filename)
         return template_obj
     except jinja2.TemplateNotFound:
-        msg = "Missing heat template for %s" % datastore_type
+        msg = "Missing heat template for %s" % datastore_manager
         LOG.error(msg)
         raise exception.TroveError(msg)

trove/datastore/models.py

@@ -0,0 +1,185 @@
+from trove.common import cfg
+from trove.common import exception
+from trove.db import models as dbmodels
+from trove.db import get_db_api
+from trove.openstack.common import uuidutils
+
+
+CONF = cfg.CONF
+db_api = get_db_api()
+
+
+def persisted_models():
+    return {
+        'datastore': DBDatastore,
+        'datastore_version': DBDatastoreVersion,
+    }
+
+
+class DBDatastore(dbmodels.DatabaseModelBase):
+
+    _data_fields = ['id', 'name', 'manager', 'default_version_id']
+
+
+class DBDatastoreVersion(dbmodels.DatabaseModelBase):
+
+    _data_fields = ['id', 'datastore_id', 'name', 'image_id', 'packages',
+                    'active']
+
+
+class Datastore(object):
+
+    def __init__(self, db_info):
+        self.db_info = db_info
+
+    @classmethod
+    def load(cls, id_or_name):
+        try:
+            return cls(DBDatastore.find_by(id=id_or_name))
+        except exception.ModelNotFoundError:
+            try:
+                return cls(DBDatastore.find_by(name=id_or_name))
+            except exception.ModelNotFoundError:
+                raise exception.DatastoreNotFound(datastore=id_or_name)
+
+    @property
+    def id(self):
+        return self.db_info.id
+
+    @property
+    def name(self):
+        return self.db_info.name
+
+    @property
+    def manager(self):
+        return self.db_info.manager
+
+    @property
+    def default_version_id(self):
+        return self.db_info.default_version_id
+
+
+class Datastores(object):
+
+    def __init__(self, db_info):
+        self.db_info = db_info
+
+    @classmethod
+    def load(cls):
+        return cls(DBDatastore.find_all())
+
+    def __iter__(self):
+        for item in self.db_info:
+            yield item
+
+
+class DatastoreVersion(object):
+
+    def __init__(self, db_info):
+        self.db_info = db_info
+
+    @classmethod
+    def load(cls, id_or_name):
+        try:
+            return cls(DBDatastoreVersion.find_by(id=id_or_name))
+        except exception.ModelNotFoundError:
+            try:
+                return cls(DBDatastoreVersion.find_by(name=id_or_name))
+            except exception.ModelNotFoundError:
+                raise exception.DatastoreVersionNotFound(version=id_or_name)
+
+    @property
+    def id(self):
+        return self.db_info.id
+
+    @property
+    def datastore_id(self):
+        return self.db_info.datastore_id
+
+    @property
+    def name(self):
+        return self.db_info.name
+
+    @property
+    def image_id(self):
+        return self.db_info.image_id
+
+    @property
+    def packages(self):
+        return self.db_info.packages
+
+    @property
+    def active(self):
+        return self.db_info.active
+
+
+class DatastoreVersions(object):
+
+    def __init__(self, db_info):
+        self.db_info = db_info
+
+    @classmethod
+    def load(cls, id_or_name, active=True):
+        datastore = Datastore.load(id_or_name)
+        return cls(DBDatastoreVersion.find_all(datastore_id=datastore.id,
+                                               active=active))
+
+    def __iter__(self):
+        for item in self.db_info:
+            yield item
+
+
+def get_datastore_version(type=None, version=None):
+    datastore = type or CONF.default_datastore
+    if not datastore:
+        raise exception.DatastoreDefaultDatastoreNotFound()
+    datastore = Datastore.load(datastore)
+    version = version or datastore.default_version_id
+    if not version:
+        raise exception.DatastoreDefaultVersionNotFound(datastore=
+                                                        datastore.name)
+    datastore_version = DatastoreVersion.load(version)
+    if datastore_version.datastore_id != datastore.id:
+        raise exception.DatastoreNoVersion(datastore=datastore.name,
+                                           version=datastore_version.name)
+    if not datastore_version.active:
+        raise exception.DatastoreVersionInactive(version=
+                                                 datastore_version.name)
+    return (datastore, datastore_version)
+
+
+def update_datastore(name, manager, default_version):
+    db_api.configure_db(CONF)
+    if default_version:
+        version = DatastoreVersion.load(default_version)
+        if not version.active:
+            raise exception.DatastoreVersionInactive(version=
+                                                     version.name)
+    try:
+        datastore = DBDatastore.find_by(name=name)
+    except exception.ModelNotFoundError:
+        # Create a new one
+        datastore = DBDatastore()
+        datastore.id = uuidutils.generate_uuid()
+        datastore.name = name
+    datastore.manager = manager
+    if default_version:
+        datastore.default_version_id = version.id
+    db_api.save(datastore)
+
+
+def update_datastore_version(datastore, name, image_id, packages, active):
+    db_api.configure_db(CONF)
+    datastore = Datastore.load(datastore)
+    try:
+        version = DBDatastoreVersion.find_by(name=name)
+    except exception.ModelNotFoundError:
+        # Create a new one
+        version = DBDatastoreVersion()
+        version.id = uuidutils.generate_uuid()
+        version.name = name
+    version.datastore_id = datastore.id
+    version.image_id = image_id
+    version.packages = packages
+    version.active = active
+    db_api.save(version)

trove/datastore/service.py

@@ -0,0 +1,31 @@
+from trove.common import cfg
+from trove.common import exception
+from trove.common import utils
+from trove.common import wsgi
+from trove.datastore import models, views
+
+
+class DatastoreController(wsgi.Controller):
+
+    def show(self, req, tenant_id, id):
+        datastore = models.Datastore.load(id)
+        return wsgi.Result(views.
+                           DatastoreView(datastore, req).data(), 200)
+
+    def index(self, req, tenant_id):
+        datastores = models.Datastores.load()
+        return wsgi.Result(views.
+                           DatastoresView(datastores, req).data(),
+                           200)
+
+    def version_show(self, req, tenant_id, datastore, id):
+        datastore, datastore_version = models.get_datastore_version(datastore,
+                                                                    id)
+        return wsgi.Result(views.DatastoreVersionView(datastore_version,
+                                                      req).data(), 200)
+
+    def version_index(self, req, tenant_id, datastore):
+        datastore_versions = models.DatastoreVersions.load(datastore)
+        return wsgi.Result(views.
+                           DatastoreVersionsView(datastore_versions,
+                                                 req).data(), 200)

trove/datastore/views.py

@@ -0,0 +1,76 @@
+from trove.common.views import create_links
+
+
+class DatastoreView(object):
+
+    def __init__(self, datastore, req=None):
+        self.datastore = datastore
+        self.req = req
+
+    def data(self):
+        datastore_dict = {
+            "id": self.datastore.id,
+            "name": self.datastore.name,
+            "links": self._build_links(),
+        }
+
+        return {"datastore": datastore_dict}
+
+    def _build_links(self):
+        return create_links("datastores", self.req,
+                            self.datastore.id)
+
+
+class DatastoresView(object):
+
+    def __init__(self, datastores, req=None):
+        self.datastores = datastores
+        self.req = req
+
+    def data(self):
+        data = []
+        for datastore in self.datastores:
+            data.append(self.data_for_datastore(datastore))
+        return {'datastores': data}
+
+    def data_for_datastore(self, datastore):
+        view = DatastoreView(datastore, req=self.req)
+        return view.data()['datastore']
+
+
+class DatastoreVersionView(object):
+
+    def __init__(self, datastore_version, req=None):
+        self.datastore_version = datastore_version
+        self.req = req
+
+    def data(self):
+        datastore_version_dict = {
+            "id": self.datastore_version.id,
+            "name": self.datastore_version.name,
+            "links": self._build_links(),
+        }
+
+        return {"version": datastore_version_dict}
+
+    def _build_links(self):
+        return create_links("datastores/versions",
+                            self.req, self.datastore_version.id)
+
+
+class DatastoreVersionsView(object):
+
+    def __init__(self, datastore_versions, req=None):
+        self.datastore_versions = datastore_versions
+        self.req = req
+
+    def data(self):
+        data = []
+        for datastore_version in self.datastore_versions:
+            data.append(self.
+                        data_for_datastore_version(datastore_version))
+        return {'versions': data}
+
+    def data_for_datastore_version(self, datastore_version):
+        view = DatastoreVersionView(datastore_version, req=self.req)
+        return view.data()['version']

trove/db/__init__.py

@@ -51,6 +51,10 @@ class Query(object):
         return self.db_api.count(self._query_func, self._model,
                                  **self._conditions)
 
+    def first(self):
+        return self.db_api.first(self._query_func, self._model,
+                                 **self._conditions)
+
     def __iter__(self):
         return iter(self.all())
 

trove/db/sqlalchemy/api.py

@@ -35,6 +35,10 @@ def count(query, *args, **kwargs):
     return query(*args, **kwargs).count()
 
 
+def first(query, *args, **kwargs):
+    return query(*args, **kwargs).first()
+
+
 def find_all(model, **conditions):
     return _query_by(model, **conditions)
 

trove/db/sqlalchemy/mappers.py

@@ -30,8 +30,10 @@ def map(engine, models):
     orm.mapper(models['instance'], Table('instances', meta, autoload=True))
     orm.mapper(models['root_enabled_history'],
                Table('root_enabled_history', meta, autoload=True))
-    orm.mapper(models['service_image'],
-               Table('service_images', meta, autoload=True))
+    orm.mapper(models['datastore'],
+               Table('datastores', meta, autoload=True))
+    orm.mapper(models['datastore_version'],
+               Table('datastore_versions', meta, autoload=True))
     orm.mapper(models['service_statuses'],
                Table('service_statuses', meta, autoload=True))
     orm.mapper(models['dns_records'],

trove/db/sqlalchemy/migrate_repo/versions/016_add_datastore_type.py

@@ -0,0 +1,76 @@
+# Copyright 2012 OpenStack Foundation
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from sqlalchemy import ForeignKey
+from sqlalchemy.schema import Column
+from sqlalchemy.schema import MetaData
+from sqlalchemy.schema import UniqueConstraint
+
+from trove.db.sqlalchemy.migrate_repo.schema import Boolean
+from trove.db.sqlalchemy.migrate_repo.schema import create_tables
+from trove.db.sqlalchemy.migrate_repo.schema import DateTime
+from trove.db.sqlalchemy.migrate_repo.schema import drop_tables
+from trove.db.sqlalchemy.migrate_repo.schema import Integer
+from trove.db.sqlalchemy.migrate_repo.schema import BigInteger
+from trove.db.sqlalchemy.migrate_repo.schema import String
+from trove.db.sqlalchemy.migrate_repo.schema import Table
+
+
+meta = MetaData()
+
+
+datastores = Table(
+    'datastores',
+    meta,
+    Column('id', String(36), primary_key=True, nullable=False),
+    Column('name', String(255), unique=True),
+    Column('manager', String(255), nullable=False),
+    Column('default_version_id', String(36)),
+)
+
+
+datastore_versions = Table(
+    'datastore_versions',
+    meta,
+    Column('id', String(36), primary_key=True, nullable=False),
+    Column('datastore_id', String(36), ForeignKey('datastores.id')),
+    Column('name', String(255), unique=True),
+    Column('image_id', String(36), nullable=False),
+    Column('packages', String(511)),
+    Column('active', Boolean(), nullable=False),
+    UniqueConstraint('datastore_id', 'name', name='ds_versions')
+)
+
+
+def upgrade(migrate_engine):
+    meta.bind = migrate_engine
+    create_tables([datastores, datastore_versions])
+    instances = Table('instances', meta, autoload=True)
+    datastore_version_id = Column('datastore_version_id', String(36),
+                                  ForeignKey('datastore_versions.id'))
+    instances.create_column(datastore_version_id)
+    instances.drop_column('service_type')
+    # Table 'service_images' is deprecated since this version.
+    # Leave it for few releases.
+    #drop_tables([service_images])
+
+
+def downgrade(migrate_engine):
+    meta.bind = migrate_engine
+    drop_tables([datastores, datastore_versions])
+    instances = Table('instances', meta, autoload=True)
+    instances.drop_column('datastore_version_id')
+    service_type = Column('service_type', String(36))
+    instances.create_column(service_type)
+    instances.update().values({'service_type': 'mysql'}).execute()

trove/db/sqlalchemy/session.py

@@ -42,6 +42,7 @@ def configure_db(options, models_mapper=None):
         models_mapper.map(_ENGINE)
     else:
         from trove.instance import models as base_models
+        from trove.datastore import models as datastores_models
         from trove.dns import models as dns_models
         from trove.extensions.mysql import models as mysql_models
         from trove.guestagent import models as agent_models
@@ -51,6 +52,7 @@ def configure_db(options, models_mapper=None):
 
         model_modules = [
             base_models,
+            datastores_models,
             dns_models,
             mysql_models,
             agent_models,

trove/extensions/mgmt/instances/models.py

@@ -180,14 +180,14 @@ class NotificationTransformer(object):
             subsecond=True)
         return audit_start, audit_end
 
-    def _get_service_id(self, service_type, id_map):
-        if service_type in id_map:
-            service_type_id = id_map[service_type]
+    def _get_service_id(self, datastore_manager, id_map):
+        if datastore_manager in id_map:
+            datastore_manager_id = id_map[datastore_manager]
         else:
-            service_type_id = cfg.UNKNOWN_SERVICE_ID
-            LOG.error("Service ID for Type (%s) is not configured"
-                      % service_type)
-        return service_type_id
+            datastore_manager_id = cfg.UNKNOWN_SERVICE_ID
+            LOG.error("Datastore ID for Manager (%s) is not configured"
+                      % datastore_manager)
+        return datastore_manager_id
 
     def transform_instance(self, instance, audit_start, audit_end):
         payload = {
@@ -206,7 +206,7 @@ class NotificationTransformer(object):
             'tenant_id': instance.tenant_id
         }
         payload['service_id'] = self._get_service_id(
-            instance.service_type, CONF.notification_service_id)
+            instance.datastore.manager, CONF.notification_service_id)
         return payload
 
     def __call__(self):

trove/guestagent/api.py

@@ -212,7 +212,7 @@ class API(proxy.RpcProxy):
         LOG.debug(_("Check diagnostics on Instance %s"), self.id)
         return self._call("get_diagnostics", AGENT_LOW_TIMEOUT)
 
-    def prepare(self, memory_mb, databases, users,
+    def prepare(self, memory_mb, packages, databases, users,
                 device_path='/dev/vdb', mount_point='/mnt/volume',
                 backup_id=None, config_contents=None, root_password=None):
         """Make an asynchronous call to prepare the guest
@@ -220,10 +220,10 @@ class API(proxy.RpcProxy):
         """
         LOG.debug(_("Sending the call to prepare the Guest"))
         self._cast_with_consumer(
-            "prepare", databases=databases, memory_mb=memory_mb,
-            users=users, device_path=device_path, mount_point=mount_point,
-            backup_id=backup_id, config_contents=config_contents,
-            root_password=root_password)
+            "prepare", packages=packages, databases=databases,
+            memory_mb=memory_mb, users=users, device_path=device_path,
+            mount_point=mount_point, backup_id=backup_id,
+            config_contents=config_contents, root_password=root_password)
 
     def restart(self):
         """Restart the MySQL server."""

trove/guestagent/common/operating_system.py

@@ -25,3 +25,46 @@ def get_os():
         return REDHAT
     else:
         return DEBIAN
+
+
+def service_discovery(service_candidates):
+    """
+    This function discovering how to start, stop, enable, disable service
+    in current environment. "service_candidates" is array with possible
+    system service names. Works for upstart, systemd, sysvinit.
+    """
+    result = {}
+    for service in service_candidates:
+        # check upstart
+        if os.path.isfile("/etc/init/%s.conf" % service):
+            # upstart returns error code when service already started/stopped
+            result['cmd_start'] = "sudo start %s || true" % service
+            result['cmd_stop'] = "sudo stop %s || true" % service
+            result['cmd_enable'] = ("sudo sed -i '/^manual$/d' "
+                                    "/etc/init/%s.conf" % service)
+            result['cmd_disable'] = ("sudo sh -c 'echo manual >> "
+                                     "/etc/init/%s.conf'" % service)
+            break
+        # check sysvinit
+        if os.path.isfile("/etc/init.d/%s" % service):
+            result['cmd_start'] = "sudo service %s start" % service
+            result['cmd_stop'] = "sudo service %s stop" % service
+            if os.path.isfile("/usr/sbin/update-rc.d"):
+                result['cmd_enable'] = "sudo update-rc.d %s defaults; sudo " \
+                                       "update-rc.d %s enable" % (service,
+                                                                  service)
+                result['cmd_disable'] = "sudo update-rc.d %s defaults; sudo " \
+                                        "update-rc.d %s disable" % (service,
+                                                                    service)
+            elif os.path.isfile("/sbin/chkconfig"):
+                result['cmd_enable'] = "sudo chkconfig %s on" % service
+                result['cmd_disable'] = "sudo chkconfig %s off" % service
+            break
+        # check systemd
+        if os.path.isfile("/lib/systemd/system/%s.service" % service):
+            result['cmd_start'] = "sudo systemctl start %s" % service
+            result['cmd_stop'] = "sudo systemctl stop %s" % service
+            result['cmd_enable'] = "sudo systemctl enable %s" % service
+            result['cmd_disable'] = "sudo systemctl disable %s" % service
+            break
+    return result

trove/guestagent/datastore/mysql/manager.py

@@ -84,32 +84,26 @@ class Manager(periodic_task.PeriodicTasks):
             raise
         LOG.info(_("Restored database successfully"))
 
-    def prepare(self, context, databases, memory_mb, users, device_path=None,
-                mount_point=None, backup_id=None, config_contents=None,
-                root_password=None):
+    def prepare(self, context, packages, databases, memory_mb, users,
+                device_path=None, mount_point=None, backup_id=None,
+                config_contents=None, root_password=None):
         """Makes ready DBAAS on a Guest container."""
         MySqlAppStatus.get().begin_install()
         # status end_mysql_install set with secure()
         app = MySqlApp(MySqlAppStatus.get())
-        restart_mysql = False
+        app.install_if_needed(packages)
         if device_path:
+            #stop and do not update database
+            app.stop_db()
             device = volume.VolumeDevice(device_path)
             device.format()
-            #if a /var/lib/mysql folder exists, back it up.
             if os.path.exists(CONF.mount_point):
-                #stop and do not update database
-                app.stop_db()
                 #rsync exiting data
-                if not backup_id:
-                    restart_mysql = True
-                    device.migrate_data(CONF.mount_point)
+                device.migrate_data(CONF.mount_point)
             #mount the volume
             device.mount(mount_point)
             LOG.debug(_("Mounted the volume."))
-            #check mysql was installed and stopped
-            if restart_mysql:
-                app.start_mysql()
-        app.install_if_needed()
+            app.start_mysql()
         if backup_id:
             self._perform_restore(backup_id, context, CONF.mount_point, app)
         LOG.info(_("Securing mysql now."))

trove/guestagent/datastore/mysql/service.py

@@ -13,11 +13,11 @@ from trove.common import cfg
 from trove.common import utils as utils
 from trove.common import exception
 from trove.common import instance as rd_instance
+from trove.guestagent.common import operating_system
 from trove.guestagent.common import sql_query
 from trove.guestagent.db import models
 from trove.guestagent import pkg
 from trove.guestagent.datastore import service
-from trove.guestagent.datastore.mysql import system
 from trove.openstack.common import log as logging
 from trove.openstack.common.gettextutils import _
 from trove.extensions.mysql.models import RootHistory
@@ -26,7 +26,6 @@ ADMIN_USER_NAME = "os_admin"
 LOG = logging.getLogger(__name__)
 FLUSH = text(sql_query.FLUSH)
 ENGINE = None
-MYSQLD_ARGS = None
 PREPARING = False
 UUID = False
 
@@ -39,6 +38,11 @@ INCLUDE_MARKER_OPERATORS = {
     False: ">"
 }
 
+MYSQL_CONFIG = "/etc/mysql/my.cnf"
+MYSQL_SERVICE_CANDIDATES = ["mysql", "mysqld", "mysql-server"]
+MYSQL_BIN_CANDIDATES = ["/usr/sbin/mysqld", "/usr/libexec/mysqld"]
+
+
 # Create a package impl
 packager = pkg.Package()
 
@@ -47,12 +51,41 @@ def generate_random_password():
     return passlib.utils.generate_password(size=CONF.default_password_length)
 
 
+def clear_expired_password():
+    """
+    Some mysql installations generating random root password
+    and save it in /root/.mysql_secret, this password is
+    expired and should be changed by client that supports expired passwords.
+    """
+    LOG.debug("Removing expired password.")
+    secret_file = "/root/.mysql_secret"
+    try:
+        out, err = utils.execute("cat", secret_file,
+                                 run_as_root=True, root_helper="sudo")
+    except exception.ProcessExecutionError:
+        LOG.debug("/root/.mysql_secret is not exists.")
+        return
+    m = re.match('# The random password set for the root user at .*: (.*)',
+                 out)
+    if m:
+        try:
+            out, err = utils.execute("mysqladmin", "-p%s" % m.group(1),
+                                     "password", "", run_as_root=True,
+                                     root_helper="sudo")
+        except exception.ProcessExecutionError:
+            LOG.error("Cannot change mysql password.")
+            return
+        utils.execute("rm", "-f", secret_file, run_as_root=True,
+                      root_helper="sudo")
+        LOG.debug("Expired password removed.")
+
+
 def get_auth_password():
     pwd, err = utils.execute_with_timeout(
         "sudo",
         "awk",
         "/password\\t=/{print $3; exit}",
-        system.MYSQL_CONFIG)
+        MYSQL_CONFIG)
     if err:
         LOG.error(err)
         raise RuntimeError("Problem reading my.cnf! : %s" % err)
@@ -76,8 +109,13 @@ def get_engine():
 
 
 def load_mysqld_options():
+    #find mysqld bin
+    for bin in MYSQL_BIN_CANDIDATES:
+        if os.path.isfile(bin):
+            mysqld_bin = bin
+            break
     try:
-        out, err = utils.execute(system.MYSQL_BIN, "--print-defaults",
+        out, err = utils.execute(mysqld_bin, "--print-defaults",
                                  run_as_root=True, root_helper="sudo")
         arglist = re.split("\n", out)[1].split()
         args = {}
@@ -89,7 +127,7 @@ def load_mysqld_options():
                 args[item.lstrip("--")] = None
         return args
     except exception.ProcessExecutionError:
-        return None
+        return {}
 
 
 class MySqlAppStatus(service.BaseDbStatus):
@@ -100,7 +138,6 @@ class MySqlAppStatus(service.BaseDbStatus):
         return cls._instance
 
     def _get_actual_db_status(self):
-        global MYSQLD_ARGS
         try:
             out, err = utils.execute_with_timeout(
                 "/usr/bin/mysqladmin",
@@ -119,10 +156,9 @@ class MySqlAppStatus(service.BaseDbStatus):
                 LOG.info("Service Status is BLOCKED.")
                 return rd_instance.ServiceStatuses.BLOCKED
             except exception.ProcessExecutionError:
-                if not MYSQLD_ARGS:
-                    MYSQLD_ARGS = load_mysqld_options()
-                pid_file = MYSQLD_ARGS.get('pid_file',
-                                           '/var/run/mysqld/mysqld.pid')
+                mysql_args = load_mysqld_options()
+                pid_file = mysql_args.get('pid_file',
+                                          '/var/run/mysqld/mysqld.pid')
                 if os.path.exists(pid_file):
                     LOG.info("Service Status is CRASHED.")
                     return rd_instance.ServiceStatuses.CRASHED
@@ -492,10 +528,6 @@ class MySqlApp(object):
     """Prepares DBaaS on a Guest container."""
 
     TIME_OUT = 1000
-    if CONF.service_type == "mysql":
-        MYSQL_PACKAGE_VERSION = CONF.mysql_pkg
-    elif CONF.service_type == "percona":
-        MYSQL_PACKAGE_VERSION = CONF.percona_pkg
 
     def __init__(self, status):
         """ By default login with root no password for initial setup. """
@@ -522,11 +554,19 @@ class MySqlApp(object):
         t = text(str(uu))
         client.execute(t)
 
-    def install_if_needed(self):
+    def install_if_needed(self, packages):
         """Prepare the guest machine with a secure mysql server installation"""
         LOG.info(_("Preparing Guest as MySQL Server"))
-        if not self.is_installed():
-            self._install_mysql()
+        if not packager.pkg_is_installed(packages):
+            LOG.debug(_("Installing mysql server"))
+            self._clear_mysql_config()
+            # set blank password on pkg configuration stage
+            pkg_opts = {'root_password': '',
+                        'root_password_again': ''}
+            packager.pkg_install(packages, pkg_opts, self.TIME_OUT)
+            self._create_mysql_confd_dir()
+            LOG.debug(_("Finished installing mysql server"))
+        self.start_mysql()
         LOG.info(_("Dbaas install_if_needed complete"))
 
     def complete_install_or_restart(self):
@@ -535,7 +575,7 @@ class MySqlApp(object):
     def secure(self, config_contents):
         LOG.info(_("Generating admin password..."))
         admin_password = generate_random_password()
-
+        clear_expired_password()
         engine = sqlalchemy.create_engine("mysql://root:@localhost:3306",
                                           echo=True)
         with LocalSqlClient(engine) as client:
@@ -549,22 +589,25 @@ class MySqlApp(object):
         LOG.info(_("Dbaas secure complete."))
 
     def secure_root(self, secure_remote_root=True):
-        engine = sqlalchemy.create_engine("mysql://root:@localhost:3306",
-                                          echo=True)
-        with LocalSqlClient(engine) as client:
+        with LocalSqlClient(get_engine()) as client:
             LOG.info(_("Preserving root access from restore"))
             self._generate_root_password(client)
             if secure_remote_root:
                 self._remove_remote_root_access(client)
 
-    def _install_mysql(self):
-        """Install mysql server. The current version is 5.5"""
-        LOG.debug(_("Installing mysql server"))
-        self._create_mysql_confd_dir()
-        packager.pkg_install(self.MYSQL_PACKAGE_VERSION, self.TIME_OUT)
-        self.start_mysql()
-        LOG.debug(_("Finished installing mysql server"))
-        #TODO(rnirmal): Add checks to make sure the package got installed
+    def _clear_mysql_config(self):
+        """Clear old configs, which can be incompatible with new version """
+        LOG.debug("Clearing old mysql config")
+        random_uuid = str(uuid.uuid4())
+        configs = ["/etc/my.cnf", "/etc/mysql/conf.d", "/etc/mysql/my.cnf"]
+        for config in configs:
+            command = "mv %s %s_%s" % (config, config, random_uuid)
+            try:
+                utils.execute_with_timeout(command, shell=True,
+                                           root_helper="sudo")
+                LOG.debug("%s saved to %s_%s" % (config, config, random_uuid))
+            except exception.ProcessExecutionError:
+                pass
 
     def _create_mysql_confd_dir(self):
         conf_dir = "/etc/mysql/conf.d"
@@ -573,44 +616,33 @@ class MySqlApp(object):
         utils.execute_with_timeout(command, shell=True)
 
     def _enable_mysql_on_boot(self):
-        """
-        There is a difference between the init.d mechanism and the upstart
-        The stock mysql uses the upstart mechanism, therefore, there is a
-        mysql.conf file responsible for the job. to toggle enable/disable
-        on boot one needs to modify this file. Percona uses the init.d
-        mechanism and there is no mysql.conf file. Instead, the update-rc.d
-        command needs to be used to modify the /etc/rc#.d/[S/K]##mysql links
-        """
         LOG.info("Enabling mysql on boot.")
-        conf = "/etc/init/mysql.conf"
-        if os.path.isfile(conf):
-            command = "sudo sed -i '/^manual$/d' %(conf)s" % {'conf': conf}
-        else:
-            command = system.MYSQL_CMD_ENABLE
-        utils.execute_with_timeout(command, shell=True)
+        try:
+            mysql_service = operating_system.service_discovery(
+                MYSQL_SERVICE_CANDIDATES)
+            utils.execute_with_timeout(mysql_service['cmd_enable'], shell=True)
+        except KeyError:
+            raise RuntimeError("Service is not discovered.")
 
     def _disable_mysql_on_boot(self):
-        """
-        There is a difference between the init.d mechanism and the upstart
-        The stock mysql uses the upstart mechanism, therefore, there is a
-        mysql.conf file responsible for the job. to toggle enable/disable
-        on boot one needs to modify this file. Percona uses the init.d
-        mechanism and there is no mysql.conf file. Instead, the update-rc.d
-        command needs to be used to modify the /etc/rc#.d/[S/K]##mysql links
-        """
-        LOG.info("Disabling mysql on boot.")
-        conf = "/etc/init/mysql.conf"
-        if os.path.isfile(conf):
-            command = "sudo sh -c 'echo manual >> %(conf)s'" % {'conf': conf}
-        else:
-            command = system.MYSQL_CMD_DISABLE
-        utils.execute_with_timeout(command, shell=True)
+        try:
+            mysql_service = operating_system.service_discovery(
+                MYSQL_SERVICE_CANDIDATES)
+            utils.execute_with_timeout(mysql_service['cmd_disable'],
+                                       shell=True)
+        except KeyError:
+            raise RuntimeError("Service is not discovered.")
 
     def stop_db(self, update_db=False, do_not_start_on_reboot=False):
         LOG.info(_("Stopping mysql..."))
         if do_not_start_on_reboot:
             self._disable_mysql_on_boot()
-        utils.execute_with_timeout(system.MYSQL_CMD_STOP, shell=True)
+        try:
+            mysql_service = operating_system.service_discovery(
+                MYSQL_SERVICE_CANDIDATES)
+            utils.execute_with_timeout(mysql_service['cmd_stop'], shell=True)
+        except KeyError:
+            raise RuntimeError("Service is not discovered.")
         if not self.status.wait_for_real_status_to_change_to(
                 rd_instance.ServiceStatuses.SHUTDOWN,
                 self.state_change_wait_time, update_db):
@@ -696,13 +728,13 @@ class MySqlApp(object):
         with open(TMP_MYCNF, 'w') as t:
             t.write(config_contents)
         utils.execute_with_timeout("sudo", "mv", TMP_MYCNF,
-                                   system.MYSQL_CONFIG)
+                                   MYSQL_CONFIG)
 
-        self._write_temp_mycnf_with_admin_account(system.MYSQL_CONFIG,
+        self._write_temp_mycnf_with_admin_account(MYSQL_CONFIG,
                                                   TMP_MYCNF,
                                                   admin_password)
         utils.execute_with_timeout("sudo", "mv", TMP_MYCNF,
-                                   system.MYSQL_CONFIG)
+                                   MYSQL_CONFIG)
 
         self.wipe_ib_logfiles()
 
@@ -715,8 +747,11 @@ class MySqlApp(object):
         self._enable_mysql_on_boot()
 
         try:
-            utils.execute_with_timeout(system.
-                                       MYSQL_CMD_START, shell=True)
+            mysql_service = operating_system.service_discovery(
+                MYSQL_SERVICE_CANDIDATES)
+            utils.execute_with_timeout(mysql_service['cmd_start'], shell=True)
+        except KeyError:
+            raise RuntimeError("Service is not discovered.")
         except exception.ProcessExecutionError:
             # it seems mysql (percona, at least) might come back with [Fail]
             # but actually come up ok. we're looking into the timing issue on
@@ -756,11 +791,6 @@ class MySqlApp(object):
         LOG.info(_("Resetting configuration"))
         self._write_mycnf(None, config_contents)
 
-    def is_installed(self):
-        #(cp16net) could raise an exception, does it need to be handled here?
-        version = packager.pkg_version(self.MYSQL_PACKAGE_VERSION)
-        return not version is None
-
 
 class MySqlRootAccess(object):
     @classmethod

trove/guestagent/datastore/mysql/system.py

@@ -1,51 +0,0 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
-# Copyright (c) 2011 OpenStack Foundation
-# All Rights Reserved.
-#
-#    Licensed under the Apache License, Version 2.0 (the "License"); you may
-#    not use this file except in compliance with the License. You may obtain
-#    a copy of the License at
-#
-#         http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-#    License for the specific language governing permissions and limitations
-#    under the License.
-
-"""
-Determines operating system version and os depended commands.
-"""
-import os.path
-from trove.common import cfg
-
-CONF = cfg.CONF
-
-REDHAT = 'redhat'
-DEBIAN = 'debian'
-
-# The default is debian
-OS = DEBIAN
-MYSQL_CONFIG = "/etc/mysql/my.cnf"
-MYSQL_BIN = "/usr/sbin/mysqld"
-MYSQL_CMD_ENABLE = "sudo update-rc.d mysql enable"
-MYSQL_CMD_DISABLE = "sudo update-rc.d mysql disable"
-MYSQL_CMD_START = "sudo service mysql start || /bin/true"
-MYSQL_CMD_STOP = "sudo service mysql stop || /bin/true"
-
-if os.path.isfile("/etc/redhat-release"):
-    OS = REDHAT
-    MYSQL_CONFIG = "/etc/my.cnf"
-    if CONF.service_type == 'percona':
-        MYSQL_CMD_ENABLE = "sudo chkconfig mysql on"
-        MYSQL_CMD_DISABLE = "sudo chkconfig mysql off"
-        MYSQL_CMD_START = "sudo service mysql start"
-        MYSQL_CMD_STOP = "sudo service mysql stop"
-    else:
-        MYSQL_BIN = "/usr/libexec/mysqld"
-        MYSQL_CMD_ENABLE = "sudo chkconfig mysqld on"
-        MYSQL_CMD_DISABLE = "sudo chkconfig mysqld off"
-        MYSQL_CMD_START = "sudo service mysqld start"
-        MYSQL_CMD_STOP = "sudo service mysqld stop"

trove/guestagent/dbaas.py

@@ -42,10 +42,10 @@ CONF = cfg.CONF
 
 
 def get_custom_managers():
-    return CONF.service_registry_ext
+    return CONF.datastore_registry_ext
 
 
-def service_registry():
+def datastore_registry():
     return dict(chain(defaults.iteritems(),
                 get_custom_managers().iteritems()))
 

trove/guestagent/pkg.py

@@ -20,6 +20,7 @@ Manages packages on the Guest VM.
 """
 import commands
 import re
+from tempfile import NamedTemporaryFile
 
 import pexpect
 
@@ -35,6 +36,7 @@ LOG = logging.getLogger(__name__)
 OK = 0
 RUN_DPKG_FIRST = 1
 REINSTALL_FIRST = 2
+CONFLICT_REMOVED = 3
 
 
 class PkgAdminLockError(exception.TroveError):
@@ -61,11 +63,15 @@ class PkgScriptletError(exception.TroveError):
     pass
 
 
-class PkgTransactionCheckError(exception.TroveError):
+class PkgDownloadError(exception.TroveError):
     pass
 
 
-class PkgDownloadError(exception.TroveError):
+class PkgSignError(exception.TroveError):
+    pass
+
+
+class PkgBrokenError(exception.TroveError):
     pass
 
 
@@ -84,16 +90,29 @@ class BasePackagerMixin:
         child = pexpect.spawn(cmd, timeout=time_out)
         try:
             i = child.expect(output_expects)
+            match = child.match
             self.pexpect_wait_and_close_proc(child)
         except pexpect.TIMEOUT:
             self.pexpect_kill_proc(child)
             raise PkgTimeout("Process timeout after %i seconds." % time_out)
-        return i
+        return (i, match)
 
 
 class RedhatPackagerMixin(BasePackagerMixin):
 
-    def _install(self, package_name, time_out):
+    def _rpm_remove_nodeps(self, package_name):
+        """
+        Sometimes transaction errors happens, easy way is to remove
+        conflicted package without dependencies and hope it will replaced
+        by anoter package
+        """
+        try:
+            utils.execute("rpm", "-e", "--nodeps", package_name,
+                          run_as_root=True, root_helper="sudo")
+        except ProcessExecutionError:
+            LOG.error(_("Error removing conflict %s") % package_name)
+
+    def _install(self, packages, time_out):
         """Attempts to install a package.
 
         Returns OK if the package installs fine or a result code if a
@@ -101,27 +120,35 @@ class RedhatPackagerMixin(BasePackagerMixin):
         Raises an exception if a non-recoverable error or time out occurs.
 
         """
-        cmd = "sudo yum --color=never -y install %s" % package_name
+        cmd = "sudo yum --color=never -y install %s" % packages
         output_expects = ['\[sudo\] password for .*:',
-                          'No package %s available.' % package_name,
-                          'Transaction Check Error:',
+                          'No package (.*) available.',
+                          ('file .* from install of .* conflicts with file'
+                           ' from package (.*?)\r\n'),
+                          'Error: (.*?) conflicts with .*?\r\n',
+                          'Processing Conflict: .* conflicts (.*?)\r\n',
                           '.*scriptlet failed*',
                           'HTTP Error',
                           'No more mirrors to try.',
+                          'GPG key retrieval failed:',
                           '.*already installed and latest version',
                           'Updated:',
                           'Installed:']
-        i = self.pexpect_run(cmd, output_expects, time_out)
+        LOG.debug("Running package install command: %s" % cmd)
+        i, match = self.pexpect_run(cmd, output_expects, time_out)
         if i == 0:
             raise PkgPermissionError("Invalid permissions.")
         elif i == 1:
-            raise PkgNotFoundError("Could not find pkg %s" % package_name)
-        elif i == 2:
-            raise PkgTransactionCheckError("Transaction Check Error")
-        elif i == 3:
+            raise PkgNotFoundError("Could not find pkg %s" % match.group(1))
+        elif i == 2 or i == 3 or i == 4:
+            self._rpm_remove_nodeps(match.group(1))
+            return CONFLICT_REMOVED
+        elif i == 5:
             raise PkgScriptletError("Package scriptlet failed")
-        elif i == 4 or i == 5:
+        elif i == 6 or i == 7:
             raise PkgDownloadError("Package download problem")
+        elif i == 8:
+            raise PkgSignError("GPG key retrieval failed")
         return OK
 
     def _remove(self, package_name, time_out):
@@ -136,18 +163,35 @@ class RedhatPackagerMixin(BasePackagerMixin):
         output_expects = ['\[sudo\] password for .*:',
                           'No Packages marked for removal',
                           'Removed:']
-        i = self.pexpect_run(cmd, output_expects, time_out)
+        i, match = self.pexpect_run(cmd, output_expects, time_out)
         if i == 0:
             raise PkgPermissionError("Invalid permissions.")
         elif i == 1:
             raise PkgNotFoundError("Could not find pkg %s" % package_name)
         return OK
 
-    def pkg_install(self, package_name, time_out):
-        result = self._install(package_name, time_out)
+    def pkg_install(self, packages, config_opts, time_out):
+        result = self._install(packages, time_out)
         if result != OK:
-            raise PkgPackageStateError("Package %s is in a bad state."
-                                       % package_name)
+            while result == CONFLICT_REMOVED:
+                result = self._install(packages, time_out)
+            if result != OK:
+                raise PkgPackageStateError("Cannot install packages.")
+
+    def pkg_is_installed(self, packages):
+        pkg_list = packages.split()
+        cmd = "rpm -qa"
+        p = commands.getstatusoutput(cmd)
+        std_out = p[1]
+        for pkg in pkg_list:
+            found = False
+            for line in std_out.split("\n"):
+                if line.find(pkg) != -1:
+                    found = True
+                    break
+            if not found:
+                return False
+        return True
 
     def pkg_version(self, package_name):
         cmd_list = ["rpm", "-qa", "--qf", "'%{VERSION}-%{RELEASE}\n'",
@@ -185,33 +229,71 @@ class DebianPackagerMixin(BasePackagerMixin):
         except ProcessExecutionError:
             LOG.error(_("Error fixing dpkg"))
 
-    def _install(self, package_name, time_out):
-        """Attempts to install a package.
+    def _fix_package_selections(self, packages, config_opts):
+        """
+        Sometimes you have to run this command before a pkg will install.
+        This command sets package selections to configure package.
+        """
+        selections = ""
+        for package in packages:
+            m = re.match('(.+)=(.+)', package)
+            if m:
+                package_name = m.group(1)
+            else:
+                package_name = package
+            command = "sudo debconf-show %s" % package_name
+            p = commands.getstatusoutput(command)
+            std_out = p[1]
+            for line in std_out.split("\n"):
+                for selection, value in config_opts.items():
+                    m = re.match(".* (.*/%s):.*" % selection, line)
+                    if m:
+                        selections += ("%s %s string '%s'\n" %
+                                       (package_name, m.group(1), value))
+        if selections:
+            with NamedTemporaryFile(delete=False) as f:
+                fname = f.name
+                f.write(selections)
+            utils.execute("debconf-set-selections %s && dpkg --configure -a"
+                          % fname, run_as_root=True, root_helper="sudo",
+                          shell=True)
+            os.remove(fname)
+
+    def _install(self, packages, time_out):
+        """Attempts to install a packages.
 
         Returns OK if the package installs fine or a result code if a
         recoverable-error occurred.
         Raises an exception if a non-recoverable error or time out occurs.
 
         """
-        cmd = "sudo -E DEBIAN_FRONTEND=noninteractive " \
-              "apt-get -y --allow-unauthenticated install %s" % package_name
+        cmd = "sudo -E DEBIAN_FRONTEND=noninteractive apt-get -y " \
+              "--force-yes --allow-unauthenticated -o " \
+              "DPkg::options::=--force-confmiss --reinstall " \
+              "install %s" % packages
         output_expects = ['.*password*',
-                          'E: Unable to locate package %s' % package_name,
-                          "Couldn't find package % s" % package_name,
+                          'E: Unable to locate package (.*)',
+                          "Couldn't find package (.*)",
+                          "E: Version '.*' for '(.*)' was not found",
                           ("dpkg was interrupted, you must manually run "
                            "'sudo dpkg --configure -a'"),
                           "Unable to lock the administration directory",
-                          "Setting up %s*" % package_name,
+                          ("E: Unable to correct problems, you have held "
+                           "broken packages."),
+                          "Setting up (.*)",
                           "is already the newest version"]
-        i = self.pexpect_run(cmd, output_expects, time_out)
+        LOG.debug("Running package install command: %s" % cmd)
+        i, match = self.pexpect_run(cmd, output_expects, time_out)
         if i == 0:
             raise PkgPermissionError("Invalid permissions.")
-        elif i == 1 or i == 2:
-            raise PkgNotFoundError("Could not find apt %s" % package_name)
-        elif i == 3:
-            return RUN_DPKG_FIRST
+        elif i == 1 or i == 2 or i == 3:
+            raise PkgNotFoundError("Could not find apt %s" % match.group(1))
         elif i == 4:
+            return RUN_DPKG_FIRST
+        elif i == 5:
             raise PkgAdminLockError()
+        elif i == 6:
+            raise PkgBrokenError()
         return OK
 
     def _remove(self, package_name, time_out):
@@ -232,7 +314,7 @@ class DebianPackagerMixin(BasePackagerMixin):
                            "'sudo dpkg --configure -a'"),
                           "Unable to lock the administration directory",
                           "Removing %s*" % package_name]
-        i = self.pexpect_run(cmd, output_expects, time_out)
+        i, match = self.pexpect_run(cmd, output_expects, time_out)
         if i == 0:
             raise PkgPermissionError("Invalid permissions.")
         elif i == 1:
@@ -245,58 +327,54 @@ class DebianPackagerMixin(BasePackagerMixin):
             raise PkgAdminLockError()
         return OK
 
-    def pkg_install(self, package_name, time_out):
-        """Installs a package."""
+    def pkg_install(self, packages, config_opts, time_out):
+        """Installs a packages."""
         try:
             utils.execute("apt-get", "update", run_as_root=True,
                           root_helper="sudo")
         except ProcessExecutionError:
             LOG.error(_("Error updating the apt sources"))
 
-        result = self._install(package_name, time_out)
+        result = self._install(packages, time_out)
         if result != OK:
             if result == RUN_DPKG_FIRST:
                 self._fix(time_out)
-            result = self._install(package_name, time_out)
+            result = self._install(packages, time_out)
             if result != OK:
-                raise PkgPackageStateError("Package %s is in a bad state."
-                                           % package_name)
+                raise PkgPackageStateError("Packages is in a bad state.")
+        # even after successful install, packages can stay unconfigured
+        # config_opts - is dict with name/value for questions asked by
+        # interactive configure script
+        self._fix_package_selections(packages, config_opts)
 
     def pkg_version(self, package_name):
-        cmd_list = ["dpkg", "-l", package_name]
-        p = commands.getstatusoutput(' '.join(cmd_list))
-        # check the command status code
-        if not p[0] == 0:
-            return None
-        # Need to capture the version string
-        # check the command output
+        p = commands.getstatusoutput("apt-cache policy %s" % package_name)
         std_out = p[1]
-        patterns = ['.*No packages found matching.*',
-                    "\w\w\s+(\S+)\s+(\S+)\s+(.*)$"]
         for line in std_out.split("\n"):
-            for p in patterns:
-                regex = re.compile(p)
-                matches = regex.match(line)
-                if matches:
-                    line = matches.group()
-                    parts = line.split()
-                    if not parts:
-                        msg = _("returned nothing")
-                        LOG.error(msg)
-                        raise exception.GuestError(msg)
-                    if len(parts) <= 2:
-                        msg = _("Unexpected output.")
-                        LOG.error(msg)
-                        raise exception.GuestError(msg)
-                    if parts[1] != package_name:
-                        msg = _("Unexpected output:[1] = %s") % str(parts[1])
-                        LOG.error(msg)
-                        raise exception.GuestError(msg)
-                    if parts[0] == 'un' or parts[2] == '<none>':
-                        return None
-                    return parts[2]
-        msg = _("version() saw unexpected output from dpkg!")
-        LOG.error(msg)
+            m = re.match("\s+Installed: (.*)", line)
+            if m:
+                version = m.group(1)
+                if version == "(none)":
+                    version = None
+                return version
+
+    def pkg_is_installed(self, packages):
+        pkg_list = packages.split()
+        for pkg in pkg_list:
+            m = re.match('(.+)=(.+)', pkg)
+            if m:
+                package_name = m.group(1)
+                package_version = m.group(2)
+            else:
+                package_name = pkg
+                package_version = None
+            installed_version = self.pkg_version(package_name)
+            if ((package_version and installed_version == package_version) or
+               (installed_version and not package_version)):
+                LOG.debug(_("Package %s already installed.") % package_name)
+            else:
+                return False
+        return True
 
     def pkg_remove(self, package_name, time_out):
         """Removes a package."""

trove/instance/models.py

@@ -30,6 +30,7 @@ from trove.common import utils
 from trove.extensions.security_group.models import SecurityGroup
 from trove.db import get_db_api
 from trove.db import models as dbmodels
+from trove.datastore import models as datastore_models
 from trove.backup.models import Backup
 from trove.quota.quota import run_with_quotas
 from trove.instance.tasks import InstanceTask
@@ -121,6 +122,10 @@ class SimpleInstance(object):
         self.db_info = db_info
         self.service_status = service_status
         self.root_pass = root_password
+        self.ds_version = (datastore_models.DatastoreVersion.
+                           load(self.db_info.datastore_version_id))
+        self.ds = (datastore_models.Datastore.
+                   load(self.ds_version.datastore_id))
 
     @property
     def addresses(self):
@@ -227,8 +232,12 @@ class SimpleInstance(object):
         return self.db_info.volume_size
 
     @property
-    def service_type(self):
-        return self.db_info.service_type
+    def datastore_version(self):
+        return self.ds_version
+
+    @property
+    def datastore(self):
+        return self.ds
 
     @property
     def root_password(self):
@@ -426,8 +435,8 @@ class Instance(BuiltInstance):
     """
 
     @classmethod
-    def create(cls, context, name, flavor_id, image_id,
-               databases, users, service_type, volume_size, backup_id,
+    def create(cls, context, name, flavor_id, image_id, databases, users,
+               datastore, datastore_version, volume_size, backup_id,
                availability_zone=None):
 
         client = create_nova_client(context)
@@ -463,7 +472,8 @@ class Instance(BuiltInstance):
             db_info = DBInstance.create(name=name, flavor_id=flavor_id,
                                         tenant_id=context.tenant,
                                         volume_size=volume_size,
-                                        service_type=service_type,
+                                        datastore_version_id=
+                                        datastore_version.id,
                                         task_status=InstanceTasks.BUILDING)
             LOG.debug(_("Tenant %(tenant)s created new "
                         "Trove instance %(db)s...") %
@@ -485,8 +495,9 @@ class Instance(BuiltInstance):
 
             task_api.API(context).create_instance(db_info.id, name, flavor,
                                                   image_id, databases, users,
-                                                  service_type, volume_size,
-                                                  backup_id,
+                                                  datastore.manager,
+                                                  datastore_version.packages,
+                                                  volume_size, backup_id,
                                                   availability_zone,
                                                   root_password)
 
@@ -694,7 +705,8 @@ class DBInstance(dbmodels.DatabaseModelBase):
 
     _data_fields = ['name', 'created', 'compute_instance_id',
                     'task_id', 'task_description', 'task_start_time',
-                    'volume_id', 'deleted', 'tenant_id', 'service_type']
+                    'volume_id', 'deleted', 'tenant_id',
+                    'datastore_version_id']
 
     def __init__(self, task_status, **kwargs):
         kwargs["task_id"] = task_status.code
@@ -719,12 +731,6 @@ class DBInstance(dbmodels.DatabaseModelBase):
     task_status = property(get_task_status, set_task_status)
 
 
-class ServiceImage(dbmodels.DatabaseModelBase):
-    """Defines the status of the service being run."""
-
-    _data_fields = ['service_name', 'image_id']
-
-
 class InstanceServiceStatus(dbmodels.DatabaseModelBase):
     _data_fields = ['instance_id', 'status_id', 'status_description',
                     'updated_at']
@@ -758,7 +764,6 @@ class InstanceServiceStatus(dbmodels.DatabaseModelBase):
 def persisted_models():
     return {
         'instance': DBInstance,
-        'service_image': ServiceImage,
         'service_statuses': InstanceServiceStatus,
     }
 

trove/instance/service.py

@@ -25,6 +25,7 @@ from trove.common import wsgi
 from trove.extensions.mysql.common import populate_validated_databases
 from trove.extensions.mysql.common import populate_users
 from trove.instance import models, views
+from trove.datastore import models as datastore_models
 from trove.backup.models import Backup as backup_model
 from trove.backup import views as backup_views
 from trove.openstack.common import log as logging
@@ -178,11 +179,10 @@ class InstanceController(wsgi.Controller):
         LOG.info(_("req : '%s'\n\n") % req)
         LOG.info(_("body : '%s'\n\n") % body)
         context = req.environ[wsgi.CONTEXT_KEY]
-        # Set the service type to mysql if its not in the request
-        service_type = (body['instance'].get('service_type') or
-                        CONF.service_type)
-        service = models.ServiceImage.find_by(service_name=service_type)
-        image_id = service['image_id']
+        datastore_args = body['instance'].get('datastore', {})
+        datastore, datastore_version = (
+            datastore_models.get_datastore_version(**datastore_args))
+        image_id = datastore_version.image_id
         name = body['instance']['name']
         flavor_ref = body['instance']['flavorRef']
         flavor_id = utils.get_id_from_href(flavor_ref)
@@ -214,8 +214,9 @@ class InstanceController(wsgi.Controller):
 
         instance = models.Instance.create(context, name, flavor_id,
                                           image_id, databases, users,
-                                          service_type, volume_size,
-                                          backup_id, availability_zone)
+                                          datastore, datastore_version,
+                                          volume_size, backup_id,
+                                          availability_zone)
 
         view = views.InstanceDetailView(instance, req=req)
         return wsgi.Result(view.data(), 200)

trove/instance/views.py

@@ -55,6 +55,7 @@ class InstanceView(object):
             "status": self.instance.status,
             "links": self._build_links(),
             "flavor": self._build_flavor_info(),
+            "datastore": {"type": self.instance.datastore.name},
         }
         if CONF.trove_volume_support:
             instance_dict['volume'] = {'size': self.instance.volume_size}
@@ -88,6 +89,9 @@ class InstanceDetailView(InstanceView):
         result['instance']['created'] = self.instance.created
         result['instance']['updated'] = self.instance.updated
 
+        result['instance']['datastore']['version'] = (self.instance.
+                                                      datastore_version.name)
+
         dns_support = CONF.trove_dns_support
         if dns_support:
             result['instance']['hostname'] = self.instance.hostname

trove/taskmanager/api.py

@@ -99,10 +99,10 @@ class API(proxy.RpcProxy):
         self.cast(self.context, self.make_msg("delete_backup",
                                               backup_id=backup_id))
 
-    def create_instance(self, instance_id, name, flavor, image_id,
-                        databases, users, service_type, volume_size,
-                        backup_id=None, availability_zone=None,
-                        root_password=None):
+    def create_instance(self, instance_id, name, flavor,
+                        image_id, databases, users, datastore_manager,
+                        packages, volume_size, backup_id=None,
+                        availability_zone=None, root_password=None):
         LOG.debug("Making async call to create instance %s " % instance_id)
         self.cast(self.context,
                   self.make_msg("create_instance",
@@ -111,7 +111,8 @@ class API(proxy.RpcProxy):
                                 image_id=image_id,
                                 databases=databases,
                                 users=users,
-                                service_type=service_type,
+                                datastore_manager=datastore_manager,
+                                packages=packages,
                                 volume_size=volume_size,
                                 backup_id=backup_id,
                                 availability_zone=availability_zone,

trove/taskmanager/manager.py

@@ -81,12 +81,13 @@ class Manager(periodic_task.PeriodicTasks):
         instance_tasks.create_backup(backup_id)
 
     def create_instance(self, context, instance_id, name, flavor,
-                        image_id, databases, users, service_type,
-                        volume_size, backup_id, availability_zone,
+                        image_id, databases, users, datastore_manager,
+                        packages, volume_size, backup_id, availability_zone,
                         root_password):
         instance_tasks = FreshInstanceTasks.load(context, instance_id)
         instance_tasks.create_instance(flavor, image_id, databases, users,
-                                       service_type, volume_size, backup_id,
+                                       datastore_manager, packages,
+                                       volume_size, backup_id,
                                        availability_zone, root_password)
 
     if CONF.exists_notification_transformer:

trove/taskmanager/models.py

@@ -69,14 +69,14 @@ class NotifyMixin(object):
     This adds the ability to send usage events to an Instance object.
     """
 
-    def _get_service_id(self, service_type, id_map):
-        if service_type in id_map:
-            service_type_id = id_map[service_type]
+    def _get_service_id(self, datastore_manager, id_map):
+        if datastore_manager in id_map:
+            datastore_manager_id = id_map[datastore_manager]
         else:
-            service_type_id = cfg.UNKNOWN_SERVICE_ID
-            LOG.error(_("Service ID for Type (%s) is not configured")
-                      % service_type)
-        return service_type_id
+            datastore_manager_id = cfg.UNKNOWN_SERVICE_ID
+            LOG.error("Datastore ID for Manager (%s) is not configured"
+                      % datastore_manager)
+        return datastore_manager_id
 
     def send_usage_event(self, event_type, **kwargs):
         event_type = 'trove.instance.%s' % event_type
@@ -117,7 +117,7 @@ class NotifyMixin(object):
             })
 
         payload['service_id'] = self._get_service_id(
-            self.service_type, CONF.notification_service_id)
+            self.datastore.manager, CONF.notification_service_id)
 
         # Update payload with all other kwargs
         payload.update(kwargs)
@@ -133,17 +133,17 @@ class ConfigurationMixin(object):
     Configuration related tasks for instances and resizes.
     """
 
-    def _render_config(self, service_type, flavor, instance_id):
+    def _render_config(self, datastore_manager, flavor, instance_id):
         config = template.SingleInstanceConfigTemplate(
-            service_type, flavor, instance_id)
+            datastore_manager, flavor, instance_id)
         config.render()
         return config
 
 
 class FreshInstanceTasks(FreshInstance, NotifyMixin, ConfigurationMixin):
     def create_instance(self, flavor, image_id, databases, users,
-                        service_type, volume_size, backup_id,
-                        availability_zone, root_password):
+                        datastore_manager, packages, volume_size,
+                        backup_id, availability_zone, root_password):
 
         LOG.debug(_("begin create_instance for id: %s") % self.id)
         security_groups = None
@@ -170,7 +170,7 @@ class FreshInstanceTasks(FreshInstance, NotifyMixin, ConfigurationMixin):
                 flavor,
                 image_id,
                 security_groups,
-                service_type,
+                datastore_manager,
                 volume_size,
                 availability_zone)
         elif use_nova_server_volume:
@@ -178,7 +178,7 @@ class FreshInstanceTasks(FreshInstance, NotifyMixin, ConfigurationMixin):
                 flavor['id'],
                 image_id,
                 security_groups,
-                service_type,
+                datastore_manager,
                 volume_size,
                 availability_zone)
         else:
@@ -186,15 +186,15 @@ class FreshInstanceTasks(FreshInstance, NotifyMixin, ConfigurationMixin):
                 flavor['id'],
                 image_id,
                 security_groups,
-                service_type,
+                datastore_manager,
                 volume_size,
                 availability_zone)
 
-        config = self._render_config(service_type, flavor, self.id)
+        config = self._render_config(datastore_manager, flavor, self.id)
 
         if server:
             self._guest_prepare(server, flavor['ram'], volume_info,
-                                databases, users, backup_id,
+                                packages, databases, users, backup_id,
                                 config.config_contents, root_password)
 
         if not self.db_info.task_status.is_error:
@@ -285,15 +285,15 @@ class FreshInstanceTasks(FreshInstance, NotifyMixin, ConfigurationMixin):
         return False
 
     def _create_server_volume(self, flavor_id, image_id, security_groups,
-                              service_type, volume_size,
+                              datastore_manager, volume_size,
                               availability_zone):
         LOG.debug(_("begin _create_server_volume for id: %s") % self.id)
         server = None
         try:
-            files = {"/etc/guest_info": ("[DEFAULT]\n--guest_id=%s\n"
-                                         "--service_type=%s\n"
+            files = {"/etc/guest_info": ("[DEFAULT]\n--guest_id="
+                                         "%s\n--datastore_manager=%s\n"
                                          "--tenant_id=%s\n" %
-                                         (self.id, service_type,
+                                         (self.id, datastore,
                                           self.tenant_id))}
             name = self.hostname or self.name
             volume_desc = ("mysql volume for %s" % self.id)
@@ -332,14 +332,14 @@ class FreshInstanceTasks(FreshInstance, NotifyMixin, ConfigurationMixin):
         return server, volume_info
 
     def _create_server_volume_heat(self, flavor, image_id,
-                                   security_groups, service_type,
+                                   security_groups, datastore_manager,
                                    volume_size, availability_zone):
         LOG.debug(_("begin _create_server_volume_heat for id: %s") % self.id)
         client = create_heat_client(self.context)
         novaclient = create_nova_client(self.context)
         cinderclient = create_cinder_client(self.context)
 
-        template_obj = template.load_heat_template(service_type)
+        template_obj = template.load_heat_template(datastore_manager)
         heat_template_unicode = template_obj.render()
         try:
             heat_template = heat_template_unicode.encode('ascii')
@@ -351,6 +351,7 @@ class FreshInstanceTasks(FreshInstance, NotifyMixin, ConfigurationMixin):
                       "VolumeSize": volume_size,
                       "InstanceId": self.id,
                       "ImageId": image_id,
+                      "DatastoreManager": datastore_manager,
                       "AvailabilityZone": availability_zone}
         stack_name = 'trove-%s' % self.id
         client.stacks.create(stack_name=stack_name,
@@ -377,7 +378,7 @@ class FreshInstanceTasks(FreshInstance, NotifyMixin, ConfigurationMixin):
         return server, volume_info
 
     def _create_server_volume_individually(self, flavor_id, image_id,
-                                           security_groups, service_type,
+                                           security_groups, datastore_manager,
                                            volume_size,
                                            availability_zone):
         LOG.debug(_("begin _create_server_volume_individually for id: %s") %
@@ -387,7 +388,8 @@ class FreshInstanceTasks(FreshInstance, NotifyMixin, ConfigurationMixin):
         block_device_mapping = volume_info['block_device']
         try:
             server = self._create_server(flavor_id, image_id, security_groups,
-                                         service_type, block_device_mapping,
+                                         datastore_manager,
+                                         block_device_mapping,
                                          availability_zone)
             server_id = server.id
             # Save server ID.
@@ -477,17 +479,19 @@ class FreshInstanceTasks(FreshInstance, NotifyMixin, ConfigurationMixin):
         return volume_info
 
     def _create_server(self, flavor_id, image_id, security_groups,
-                       service_type, block_device_mapping,
+                       datastore_manager, block_device_mapping,
                        availability_zone):
         files = {"/etc/guest_info": ("[DEFAULT]\nguest_id=%s\n"
-                                     "service_type=%s\n" "tenant_id=%s\n" %
-                                     (self.id, service_type, self.tenant_id))}
+                                     "datastore_manager=%s\n"
+                                     "tenant_id=%s\n" %
+                                     (self.id, datastore_manager,
+                                      self.tenant_id))}
         if os.path.isfile(CONF.get('guest_config')):
             with open(CONF.get('guest_config'), "r") as f:
                 files["/etc/trove-guestagent.conf"] = f.read()
         userdata = None
         cloudinit = os.path.join(CONF.get('cloudinit_location'),
-                                 "%s.cloudinit" % service_type)
+                                 "%s.cloudinit" % datastore_manager)
         if os.path.isfile(cloudinit):
             with open(cloudinit, "r") as f:
                 userdata = f.read()
@@ -503,11 +507,11 @@ class FreshInstanceTasks(FreshInstance, NotifyMixin, ConfigurationMixin):
         return server
 
     def _guest_prepare(self, server, flavor_ram, volume_info,
-                       databases, users, backup_id=None,
+                       packages, databases, users, backup_id=None,
                        config_contents=None, root_password=None):
         LOG.info(_("Entering guest_prepare"))
         # Now wait for the response from the create to do additional work
-        self.guest.prepare(flavor_ram, databases, users,
+        self.guest.prepare(flavor_ram, packages, databases, users,
                            device_path=volume_info['device_path'],
                            mount_point=volume_info['mount_point'],
                            backup_id=backup_id,
@@ -1007,7 +1011,7 @@ class ResizeAction(ResizeActionBase):
                   % self.instance.id)
         LOG.debug(_("Repairing config."))
         try:
-            config = self._render_config(self.instance.service_type,
+            config = self._render_config(self.instance.datastore.manager,
                                          self.old_flavor, self.instance.id)
             config = {'config_contents': config.config_contents}
             self.instance.guest.reset_configuration(config)
@@ -1028,7 +1032,7 @@ class ResizeAction(ResizeActionBase):
             modify_at=timeutils.isotime(self.instance.updated))
 
     def _start_mysql(self):
-        config = self._render_config(self.instance.service_type,
+        config = self._render_config(self.instance.datastore.manager,
                                      self.new_flavor, self.instance.id)
         self.instance.guest.start_db_with_conf_changes(config.config_contents)
 

trove/templates/mysql/heat.template

@@ -10,6 +10,8 @@ Parameters:
     Type: String
   ImageId:
     Type: String
+  DatastoreManager:
+    Type: String
   AvailabilityZone:
     Type: String
     Default: nova
@@ -25,7 +27,7 @@ Resources:
                 Fn::Join:
                 - ''
                 - ["[DEFAULT]\nguest_id=", {Ref: InstanceId},
-                  "\nservice_type=mysql"]
+                  "\\ndatastore_manager=", {Ref: DatastoreManager}]
               mode: '000644'
               owner: root
               group: root
@@ -69,4 +71,4 @@ Resources:
     Type: AWS::EC2::EIPAssociation
     Properties:
       InstanceId: {Ref: BaseInstance}
-      EIP: {Ref: DatabaseIPAddress}
+      EIP: {Ref: DatabaseIPAddress}

trove/tests/api/datastores.py

@@ -0,0 +1,114 @@
+# Copyright (c) 2011 OpenStack Foundation
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import os
+
+from nose.tools import assert_equal
+from nose.tools import assert_false
+from nose.tools import assert_true
+from troveclient.compat import exceptions
+
+from proboscis import before_class
+from proboscis import test
+from proboscis.asserts import assert_raises
+from proboscis import SkipTest
+
+from trove import tests
+from trove.tests.util import create_dbaas_client
+from trove.tests.util import test_config
+from trove.tests.util.users import Requirements
+from trove.tests.util.check import TypeCheck
+
+GROUP = "dbaas.api.datastores"
+NAME = "nonexistent"
+
+
+@test(groups=[tests.DBAAS_API, GROUP, tests.PRE_INSTANCES],
+      depends_on_groups=["services.initialize"])
+class Datastores(object):
+
+    @before_class
+    def setUp(self):
+        rd_user = test_config.users.find_user(
+            Requirements(is_admin=False, services=["trove"]))
+        self.rd_client = create_dbaas_client(rd_user)
+
+    @test
+    def test_datastore_list_attrs(self):
+        datastores = self.rd_client.datastores.list()
+        for datastore in datastores:
+            with TypeCheck('Datastore', datastore) as check:
+                check.has_field("id", basestring)
+                check.has_field("name", basestring)
+                check.has_field("links", list)
+
+    @test
+    def test_datastore_get_attrs(self):
+        datastore = self.rd_client.datastores.get(test_config.
+                                                  dbaas_datastore)
+        with TypeCheck('Datastore', datastore) as check:
+            check.has_field("id", basestring)
+            check.has_field("name", basestring)
+            check.has_field("links", list)
+        assert_equal(datastore.name, test_config.dbaas_datastore)
+
+    @test
+    def test_datastore_not_found(self):
+        try:
+            assert_raises(exceptions.NotFound,
+                          self.rd_client.datastores.get, NAME)
+        except exceptions.BadRequest as e:
+            assert_equal(e.message,
+                         "Datastore '%s' cannot be found." % NAME)
+
+    @test
+    def test_datastore_version_list_attrs(self):
+        versions = self.rd_client.datastore_versions.list(test_config.
+                                                          dbaas_datastore)
+        for version in versions:
+            with TypeCheck('DatastoreVersion', version) as check:
+                check.has_field("id", basestring)
+                check.has_field("name", basestring)
+                check.has_field("links", list)
+
+    @test
+    def test_datastore_version_get_attrs(self):
+        version = self.rd_client.datastore_versions.get(
+            test_config.dbaas_datastore, test_config.dbaas_datastore_version)
+        with TypeCheck('DatastoreVersion', version) as check:
+            check.has_field("id", basestring)
+            check.has_field("name", basestring)
+            check.has_field("links", list)
+        assert_equal(version.name, test_config.dbaas_datastore_version)
+
+    @test
+    def test_datastore_version_datastore_not_found(self):
+        try:
+            assert_raises(exceptions.NotFound,
+                          self.rd_client.datastore_versions.get,
+                          NAME, NAME)
+        except exceptions.BadRequest as e:
+            assert_equal(e.message,
+                         "Datastore '%s' cannot be found." % NAME)
+
+    @test
+    def test_datastore_version_not_found(self):
+        try:
+            assert_raises(exceptions.NotFound,
+                          self.rd_client.datastore_versions.get,
+                          test_config.dbaas_datastore, NAME)
+        except exceptions.BadRequest as e:
+            assert_equal(e.message,
+                         "Datastore version '%s' cannot be found." % NAME)

trove/tests/api/instances.py

@@ -36,6 +36,7 @@ GROUP_SECURITY_GROUPS = "dbaas.api.security_groups"
 from datetime import datetime
 from time import sleep
 
+from trove.datastore import models as datastore_models
 from trove.common import exception as rd_exceptions
 from troveclient.compat import exceptions
 
@@ -65,6 +66,9 @@ from trove.tests.util import string_in_list
 from trove.common.utils import poll_until
 from trove.tests.util.check import AttrCheck
 from trove.tests.util.check import TypeCheck
+from trove.tests.util import test_config
+
+FAKE = test_config.values['fake_mode']
 
 
 class InstanceTestInfo(object):
@@ -75,8 +79,8 @@ class InstanceTestInfo(object):
         self.dbaas_admin = None  # The rich client with admin access.
         self.dbaas_flavor = None  # The flavor object of the instance.
         self.dbaas_flavor_href = None  # The flavor of the instance.
-        self.dbaas_image = None  # The image used to create the instance.
-        self.dbaas_image_href = None  # The link of the image.
+        self.dbaas_datastore = None  # The datastore id
+        self.dbaas_datastore_version = None  # The datastore version id
         self.id = None  # The ID of the instance in the database.
         self.local_id = None
         self.address = None
@@ -162,7 +166,7 @@ def clear_messages_off_queue():
 class InstanceSetup(object):
     """Makes sure the client can hit the ReST service.
 
-    This test also uses the API to find the image and flavor to use.
+    This test also uses the API to find the flavor to use.
 
     """
 
@@ -223,6 +227,7 @@ class CreateInstanceQuotaTest(unittest.TestCase):
         import copy
 
         self.test_info = copy.deepcopy(instance_info)
+        self.test_info.dbaas_datastore = CONFIG.dbaas_datastore
 
     def tearDown(self):
         quota_dict = {'instances': CONFIG.trove_max_instances_per_user}
@@ -323,6 +328,7 @@ class CreateInstance(object):
         users.append({"name": "lite", "password": "litepass",
                       "databases": [{"name": "firstdb"}]})
         instance_info.users = users
+        instance_info.dbaas_datastore = CONFIG.dbaas_datastore
         if VOLUME_SUPPORT:
             instance_info.volume = {'size': 1}
         else:
@@ -335,7 +341,9 @@ class CreateInstance(object):
                 instance_info.volume,
                 databases,
                 users,
-                availability_zone="nova")
+                availability_zone="nova",
+                datastore=instance_info.dbaas_datastore,
+                datastore_version=instance_info.dbaas_datastore_version)
             assert_equal(200, dbaas.last_http_code)
         else:
             id = existing_instance()
@@ -355,7 +363,7 @@ class CreateInstance(object):
 
         # Check these attrs only are returned in create response
         expected_attrs = ['created', 'flavor', 'addresses', 'id', 'links',
-                          'name', 'status', 'updated']
+                          'name', 'status', 'updated', 'datastore']
         if ROOT_ON_CREATE:
             expected_attrs.append('password')
         if VOLUME_SUPPORT:
@@ -369,6 +377,7 @@ class CreateInstance(object):
                                   msg="Create response")
             # Don't CheckInstance if the instance already exists.
             check.flavor()
+            check.datastore()
             check.links(result._info['links'])
             if VOLUME_SUPPORT:
                 check.volume()
@@ -454,7 +463,7 @@ class CreateInstance(object):
             result = dbaas_admin.management.show(instance_info.id)
             expected_attrs = ['account_id', 'addresses', 'created',
                               'databases', 'flavor', 'guest_status', 'host',
-                              'hostname', 'id', 'name',
+                              'hostname', 'id', 'name', 'datastore',
                               'server_state_description', 'status', 'updated',
                               'users', 'volume', 'root_enabled_at',
                               'root_enabled_by']
@@ -462,8 +471,122 @@ class CreateInstance(object):
                 check.attrs_exist(result._info, expected_attrs,
                                   msg="Mgmt get instance")
                 check.flavor()
+                check.datastore()
                 check.guest_status()
 
+    @test
+    def test_create_failure_with_datastore_default_notfound(self):
+        if not FAKE:
+            raise SkipTest("This test only for fake mode.")
+        if VOLUME_SUPPORT:
+            volume = {'size': 1}
+        else:
+            volume = None
+        instance_name = "datastore_default_notfound"
+        databases = []
+        users = []
+        origin_default_datastore = (datastore_models.CONF.
+                                    default_datastore)
+        datastore_models.CONF.default_datastore = ""
+        try:
+            assert_raises(exceptions.NotFound,
+                          dbaas.instances.create, instance_name,
+                          instance_info.dbaas_flavor_href,
+                          volume, databases, users)
+        except exceptions.BadRequest as e:
+            assert_equal(e.message,
+                         "Please specify datastore.")
+        datastore_models.CONF.default_datastore = \
+            origin_default_datastore
+
+    @test
+    def test_create_failure_with_datastore_default_version_notfound(self):
+        if VOLUME_SUPPORT:
+            volume = {'size': 1}
+        else:
+            volume = None
+        instance_name = "datastore_default_version_notfound"
+        databases = []
+        users = []
+        datastore = "Test_Datastore_1"
+        try:
+            assert_raises(exceptions.NotFound,
+                          dbaas.instances.create, instance_name,
+                          instance_info.dbaas_flavor_href,
+                          volume, databases, users,
+                          datastore=datastore)
+        except exceptions.BadRequest as e:
+            assert_equal(e.message,
+                         "Default version for datastore '%s' not found." %
+                         datastore)
+
+    @test
+    def test_create_failure_with_datastore_notfound(self):
+        if VOLUME_SUPPORT:
+            volume = {'size': 1}
+        else:
+            volume = None
+        instance_name = "datastore_notfound"
+        databases = []
+        users = []
+        datastore = "nonexistent"
+        try:
+            assert_raises(exceptions.NotFound,
+                          dbaas.instances.create, instance_name,
+                          instance_info.dbaas_flavor_href,
+                          volume, databases, users,
+                          datastore=datastore)
+        except exceptions.BadRequest as e:
+            assert_equal(e.message,
+                         "Datastore '%s' cannot be found." %
+                         datastore)
+
+    @test
+    def test_create_failure_with_datastore_version_notfound(self):
+        if VOLUME_SUPPORT:
+            volume = {'size': 1}
+        else:
+            volume = None
+        instance_name = "datastore_version_notfound"
+        databases = []
+        users = []
+        datastore = "Test_Mysql"
+        datastore_version = "nonexistent"
+        try:
+            assert_raises(exceptions.NotFound,
+                          dbaas.instances.create, instance_name,
+                          instance_info.dbaas_flavor_href,
+                          volume, databases, users,
+                          datastore=datastore,
+                          datastore_version=datastore_version)
+        except exceptions.BadRequest as e:
+            assert_equal(e.message,
+                         "Datastore version '%s' cannot be found." %
+                         datastore_version)
+
+    @test
+    def test_create_failure_with_datastore_version_inactive(self):
+        if VOLUME_SUPPORT:
+            volume = {'size': 1}
+        else:
+            volume = None
+        instance_name = "datastore_version_inactive"
+        databases = []
+        users = []
+        datastore = "Test_Mysql"
+        datastore_version = "mysql_inactive_version"
+        try:
+            assert_raises(exceptions.NotFound,
+                          dbaas.instances.create, instance_name,
+                          instance_info.dbaas_flavor_href,
+                          volume, databases, users,
+                          datastore=datastore,
+                          datastore_version=datastore_version)
+        except exceptions.BadRequest as e:
+            assert_equal(e.message,
+                         "Datastore version '%s' is not active." %
+                         datastore_version)
+
 
 def assert_unprocessable(func, *args):
     try:
@@ -730,7 +853,8 @@ class TestInstanceListing(object):
 
     @test
     def test_index_list(self):
-        expected_attrs = ['id', 'links', 'name', 'status', 'flavor']
+        expected_attrs = ['id', 'links', 'name', 'status', 'flavor',
+                          'datastore']
         if VOLUME_SUPPORT:
             expected_attrs.append('volume')
         instances = dbaas.instances.list()
@@ -743,12 +867,14 @@ class TestInstanceListing(object):
                                   msg="Instance Index")
                 check.links(instance_dict['links'])
                 check.flavor()
+                check.datastore()
                 check.volume()
 
     @test
     def test_get_instance(self):
         expected_attrs = ['created', 'databases', 'flavor', 'hostname', 'id',
-                          'links', 'name', 'status', 'updated', 'ip']
+                          'links', 'name', 'status', 'updated', 'ip',
+                          'datastore']
         if VOLUME_SUPPORT:
             expected_attrs.append('volume')
         else:
@@ -761,6 +887,7 @@ class TestInstanceListing(object):
             check.attrs_exist(instance_dict, expected_attrs,
                               msg="Get Instance")
             check.flavor()
+            check.datastore()
             check.links(instance_dict['links'])
             check.used_volume()
 
@@ -835,12 +962,13 @@ class TestInstanceListing(object):
         expected_attrs = ['account_id', 'addresses', 'created', 'databases',
                           'flavor', 'guest_status', 'host', 'hostname', 'id',
                           'name', 'root_enabled_at', 'root_enabled_by',
-                          'server_state_description', 'status',
+                          'server_state_description', 'status', 'datastore',
                           'updated', 'users', 'volume']
         with CheckInstance(result._info) as check:
             check.attrs_exist(result._info, expected_attrs,
                               msg="Mgmt get instance")
             check.flavor()
+            check.datastore()
             check.guest_status()
             check.addresses()
             check.volume_mgmt()
@@ -1063,6 +1191,14 @@ class CheckInstance(AttrCheck):
                              msg="Flavor")
             self.links(self.instance['flavor']['links'])
 
+    def datastore(self):
+        if 'datastore' not in self.instance:
+            self.fail("'datastore' not found in instance.")
+        else:
+            expected_attrs = ['type', 'version']
+            self.attrs_exist(self.instance['datastore'], expected_attrs,
+                             msg="datastore")
+
     def volume_key_exists(self):
         if 'volume' not in self.instance:
             self.fail("'volume' not found in instance.")

trove/tests/api/instances_resize.py

@@ -31,6 +31,7 @@ from trove.instance.tasks import InstanceTasks
 from trove.openstack.common.rpc.common import RPCException
 from trove.taskmanager import models as models
 from trove.tests.fakes import nova
+from trove.tests.util import test_config
 
 GROUP = 'dbaas.api.instances.resize'
 
@@ -51,7 +52,7 @@ class ResizeTestBase(TestCase):
             flavor_id=OLD_FLAVOR_ID,
             tenant_id=999,
             volume_size=None,
-            service_type='mysql',
+            datastore_version_id=test_config.dbaas_datastore_version,
             task_status=InstanceTasks.RESIZING)
         self.server = self.mock.CreateMock(Server)
         self.instance = models.BuiltInstanceTasks(context,

trove/tests/api/mgmt/instances.py

@@ -53,6 +53,12 @@ def flavor_check(flavor):
         check.has_element("links", list)
 
 
+def datastore_check(datastore):
+    with CollectionCheck("datastore", datastore) as check:
+        check.has_element("type", basestring)
+        check.has_element("version", basestring)
+
+
 def guest_status_check(guest_status):
     with CollectionCheck("guest_status", guest_status) as check:
         check.has_element("state_description", basestring)
@@ -87,6 +93,7 @@ def mgmt_instance_get():
         # lets avoid creating more ordering work.
         instance.has_field('deleted_at', (basestring, None))
         instance.has_field('flavor', dict, flavor_check)
+        instance.has_field('datastore', dict, datastore_check)
         instance.has_field('guest_status', dict, guest_status_check)
         instance.has_field('id', basestring)
         instance.has_field('links', list)
@@ -175,6 +182,7 @@ class WhenMgmtInstanceGetIsCalledButServerIsNotReady(object):
             # lets avoid creating more ordering work.
             instance.has_field('deleted_at', (basestring, None))
             instance.has_field('flavor', dict, flavor_check)
+            instance.has_field('datastore', dict, datastore_check)
             instance.has_field('guest_status', dict, guest_status_check)
             instance.has_field('id', basestring)
             instance.has_field('links', list)
@@ -211,6 +219,7 @@ class MgmtInstancesIndex(object):
             'deleted',
             'deleted_at',
             'flavor',
+            'datastore',
             'id',
             'links',
             'name',

trove/tests/api/mgmt/instances_actions.py

@@ -52,9 +52,9 @@ class MgmtInstanceBase(object):
         self.db_info = DBInstance.create(
             name="instance",
             flavor_id=1,
+            datastore_version_id=test_config.dbaas_datastore_version,
             tenant_id=self.tenant_id,
             volume_size=None,
-            service_type='mysql',
             task_status=InstanceTasks.NONE)
         self.server = self.mock.CreateMock(Server)
         self.instance = imodels.Instance(self.context,

trove/tests/api/mgmt/malformed_json.py

@@ -41,8 +41,7 @@ class MalformedJson(object):
         users = "bar"
         try:
             self.dbaas.instances.create("bad_instance", 3, 3,
-                                        databases=databases,
-                                        users=users)
+                                        databases=databases, users=users)
         except Exception as e:
             resp, body = self.dbaas.client.last_response
             httpCode = resp.status
@@ -260,6 +259,33 @@ class MalformedJson(object):
                              "instance['volume'] 2 is not of type 'object'" %
                              (flavorId, flavorId, flavorId, flavorId))
 
+    @test
+    def test_bad_body_datastore_create_instance(self):
+        tests_utils.skip_if_xml()
+
+        datastore = "*"
+        datastore_version = "*"
+        try:
+            self.dbaas.instances.create("test_instance",
+                                        3, {"size": 2},
+                                        datastore=datastore,
+                                        datastore_version=datastore_version)
+        except Exception as e:
+            resp, body = self.dbaas.client.last_response
+            httpCode = resp.status
+            assert_equal(httpCode, 400,
+                         "Create instance failed with code %s, exception %s" %
+                         (httpCode, e))
+
+            if not isinstance(self.dbaas.client,
+                              troveclient.compat.xml.TroveXmlClient):
+                assert_equal(e.message,
+                             "Validation error: instance['datastore']['type']"
+                             " u'%s' does not match '^.*[0-9a-zA-Z]+.*$'; "
+                             "instance['datastore']['version'] u'%s' does not"
+                             " match '^.*[0-9a-zA-Z]+.*$'" %
+                             (datastore, datastore_version))
+
     @test
     def test_bad_body_volsize_create_instance(self):
         volsize = "h3ll0"

trove/tests/config.py

@@ -70,8 +70,9 @@ class TestConfig(object):
             'dbaas_url': "http://localhost:8775/v1.0/dbaas",
             'version_url': "http://localhost:8775/",
             'nova_url': "http://localhost:8774/v1.1",
+            'dbaas_datastore': "Test_Mysql",
+            'dbaas_datastore_version': "mysql_test_version",
             'instance_create_time': 16 * 60,
-            'dbaas_image': None,
             'mysql_connection_method': {"type": "direct"},
             'typical_nova_image_name': None,
             'white_box': os.environ.get("WHITE_BOX", "False") == "True",

trove/tests/fakes/guestagent.py

@@ -207,7 +207,7 @@ class FakeGuest(object):
                 % (username, hostname))
         return self.users.get((username, hostname), None)
 
-    def prepare(self, memory_mb, databases, users, device_path=None,
+    def prepare(self, memory_mb, packages, databases, users, device_path=None,
                 mount_point=None, backup_id=None, config_contents=None,
                 root_password=None):
         from trove.instance.models import DBInstance

trove/tests/unittests/guestagent/test_api.py

@@ -252,14 +252,15 @@ class ApiTest(testtools.TestCase):
         mock_conn = mock()
         when(rpc).create_connection(new=True).thenReturn(mock_conn)
         when(mock_conn).create_consumer(any(), any(), any()).thenReturn(None)
-        exp_msg = RpcMsgMatcher('prepare', 'memory_mb', 'databases', 'users',
-                                'device_path', 'mount_point', 'backup_id',
-                                'config_contents', 'root_password')
+        exp_msg = RpcMsgMatcher('prepare', 'memory_mb', 'packages',
+                                'databases', 'users', 'device_path',
+                                'mount_point', 'backup_id', 'config_contents',
+                                'root_password')
 
         when(rpc).cast(any(), any(), exp_msg).thenReturn(None)
 
-        self.api.prepare('2048', 'db1', 'user1', '/dev/vdt', '/mnt/opt',
-                         'bkup-1232', 'cont', '1-2-3-4')
+        self.api.prepare('2048', 'package1', 'db1', 'user1', '/dev/vdt',
+                         '/mnt/opt', 'bkup-1232', 'cont', '1-2-3-4')
 
         self._verify_rpc_connection_and_cast(rpc, mock_conn, exp_msg)
 
@@ -267,13 +268,14 @@ class ApiTest(testtools.TestCase):
         mock_conn = mock()
         when(rpc).create_connection(new=True).thenReturn(mock_conn)
         when(mock_conn).create_consumer(any(), any(), any()).thenReturn(None)
-        exp_msg = RpcMsgMatcher('prepare', 'memory_mb', 'databases', 'users',
-                                'device_path', 'mount_point', 'backup_id',
-                                'config_contents', 'root_password')
+        exp_msg = RpcMsgMatcher('prepare', 'memory_mb', 'packages',
+                                'databases', 'users', 'device_path',
+                                'mount_point', 'backup_id', 'config_contents',
+                                'root_password')
         when(rpc).cast(any(), any(), exp_msg).thenReturn(None)
 
-        self.api.prepare('2048', 'db1', 'user1', '/dev/vdt', '/mnt/opt',
-                         'backup_id_123', 'cont', '1-2-3-4')
+        self.api.prepare('2048', 'package1', 'db1', 'user1', '/dev/vdt',
+                         '/mnt/opt', 'backup_id_123', 'cont', '1-2-3-4')
 
         self._verify_rpc_connection_and_cast(rpc, mock_conn, exp_msg)
 
@@ -291,11 +293,13 @@ class ApiTest(testtools.TestCase):
     def test_rpc_cast_with_consumer_exception(self):
         mock_conn = mock()
         when(rpc).create_connection(new=True).thenRaise(IOError('host down'))
-        exp_msg = RpcMsgMatcher('prepare', 'memory_mb', 'databases', 'users',
-                                'device_path', 'mount_point')
+        exp_msg = RpcMsgMatcher('prepare', 'memory_mb', 'packages',
+                                'databases', 'users', 'device_path',
+                                'mount_point')
 
         with testtools.ExpectedException(exception.GuestError, '.* host down'):
-            self.api.prepare('2048', 'db1', 'user1', '/dev/vdt', '/mnt/opt')
+            self.api.prepare('2048', 'package1', 'db1', 'user1', '/dev/vdt',
+                             '/mnt/opt')
 
         verify(rpc).create_connection(new=True)
         verifyZeroInteractions(mock_conn)

trove/tests/unittests/guestagent/test_dbaas.py

@@ -39,6 +39,7 @@ from trove.common import utils
 from trove.common import instance as rd_instance
 import trove.guestagent.datastore.mysql.service as dbaas
 from trove.guestagent import dbaas as dbaas_sr
+from trove.guestagent import pkg
 from trove.guestagent.dbaas import to_gb
 from trove.guestagent.dbaas import get_filesystem_volume_stats
 from trove.guestagent.datastore.service import BaseDbStatus
@@ -108,11 +109,18 @@ class DbaasTest(testtools.TestCase):
 
         self.assertRaises(RuntimeError, dbaas.get_auth_password)
 
+    def test_service_discovery(self):
+        when(os.path).isfile(any()).thenReturn(True)
+        mysql_service = dbaas.operating_system.service_discovery(["mysql"])
+        self.assertIsNotNone(mysql_service['cmd_start'])
+        self.assertIsNotNone(mysql_service['cmd_enable'])
+
     def test_load_mysqld_options(self):
 
         output = "mysqld would've been started with the these args:\n"\
                  "--user=mysql --port=3306 --basedir=/usr "\
                  "--tmpdir=/tmp --skip-external-locking"
+        when(os.path).isfile(any()).thenReturn(True)
         dbaas.utils.execute = Mock(return_value=(output, None))
 
         options = dbaas.load_mysqld_options()
@@ -453,6 +461,13 @@ class MySqlAppTest(testtools.TestCase):
         self.appStatus = FakeAppStatus(self.FAKE_ID,
                                        rd_instance.ServiceStatuses.NEW)
         self.mySqlApp = MySqlApp(self.appStatus)
+        mysql_service = {'cmd_start': Mock(),
+                         'cmd_stop': Mock(),
+                         'cmd_enable': Mock(),
+                         'cmd_disable': Mock(),
+                         'bin': Mock()}
+        dbaas.operating_system.service_discovery = Mock(return_value=
+                                                        mysql_service)
         dbaas.time.sleep = Mock()
 
     def tearDown(self):
@@ -564,13 +579,14 @@ class MySqlAppTest(testtools.TestCase):
 
         dbaas.utils.execute_with_timeout = Mock()
         self.appStatus.set_next_status(rd_instance.ServiceStatuses.RUNNING)
-
+        self.mySqlApp._enable_mysql_on_boot = Mock()
         self.mySqlApp.start_mysql()
         self.assert_reported_status(rd_instance.ServiceStatuses.NEW)
 
     def test_start_mysql_with_db_update(self):
 
         dbaas.utils.execute_with_timeout = Mock()
+        self.mySqlApp._enable_mysql_on_boot = Mock()
         self.appStatus.set_next_status(rd_instance.ServiceStatuses.RUNNING)
 
         self.mySqlApp.start_mysql(True)
@@ -579,6 +595,7 @@ class MySqlAppTest(testtools.TestCase):
     def test_start_mysql_runs_forever(self):
 
         dbaas.utils.execute_with_timeout = Mock()
+        self.mySqlApp._enable_mysql_on_boot = Mock()
         self.mySqlApp.state_change_wait_time = 1
         self.appStatus.set_next_status(rd_instance.ServiceStatuses.SHUTDOWN)
 
@@ -634,13 +651,19 @@ class MySqlAppInstallTest(MySqlAppTest):
     def test_install(self):
 
         self.mySqlApp._install_mysql = Mock()
-        self.mySqlApp.is_installed = Mock(return_value=False)
-        self.mySqlApp.install_if_needed()
-        self.assertTrue(self.mySqlApp._install_mysql.called)
+        pkg.Package.pkg_is_installed = Mock(return_value=False)
+        utils.execute_with_timeout = Mock()
+        pkg.Package.pkg_install = Mock()
+        self.mySqlApp._clear_mysql_config = Mock()
+        self.mySqlApp._create_mysql_confd_dir = Mock()
+        self.mySqlApp.start_mysql = Mock()
+        self.mySqlApp.install_if_needed(["package"])
+        self.assertTrue(pkg.Package.pkg_install.called)
         self.assert_reported_status(rd_instance.ServiceStatuses.NEW)
 
     def test_secure(self):
 
+        dbaas.clear_expired_password = Mock()
         self.mySqlApp.start_mysql = Mock()
         self.mySqlApp.stop_db = Mock()
         self.mySqlApp._write_mycnf = Mock()
@@ -660,17 +683,20 @@ class MySqlAppInstallTest(MySqlAppTest):
         from trove.guestagent import pkg
         self.mySqlApp.start_mysql = Mock()
         self.mySqlApp.stop_db = Mock()
-        self.mySqlApp.is_installed = Mock(return_value=False)
-        self.mySqlApp._install_mysql = Mock(
-            side_effect=pkg.PkgPackageStateError("Install error"))
+        pkg.Package.pkg_is_installed = Mock(return_value=False)
+        self.mySqlApp._clear_mysql_config = Mock()
+        self.mySqlApp._create_mysql_confd_dir = Mock()
+        pkg.Package.pkg_install = \
+            Mock(side_effect=pkg.PkgPackageStateError("Install error"))
 
         self.assertRaises(pkg.PkgPackageStateError,
-                          self.mySqlApp.install_if_needed)
+                          self.mySqlApp.install_if_needed, ["package"])
 
         self.assert_reported_status(rd_instance.ServiceStatuses.NEW)
 
     def test_secure_write_conf_error(self):
 
+        dbaas.clear_expired_password = Mock()
         self.mySqlApp.start_mysql = Mock()
         self.mySqlApp.stop_db = Mock()
         self.mySqlApp._write_mycnf = Mock(
@@ -686,18 +712,6 @@ class MySqlAppInstallTest(MySqlAppTest):
         self.assertFalse(self.mySqlApp.start_mysql.called)
         self.assert_reported_status(rd_instance.ServiceStatuses.NEW)
 
-    def test_is_installed(self):
-
-        dbaas.packager.pkg_version = Mock(return_value=True)
-
-        self.assertTrue(self.mySqlApp.is_installed())
-
-    def test_is_installed_not(self):
-
-        dbaas.packager.pkg_version = Mock(return_value=None)
-
-        self.assertFalse(self.mySqlApp.is_installed())
-
 
 class TextClauseMatcher(matchers.Matcher):
     def __init__(self, text):
@@ -772,8 +786,11 @@ class MySqlAppMockTest(testtools.TestCase):
         mock_status = mock()
         when(mock_status).wait_for_real_status_to_change_to(
             any(), any(), any()).thenReturn(True)
+        when(dbaas).clear_expired_password().thenReturn(None)
         app = MySqlApp(mock_status)
         when(app)._write_mycnf(any(), any()).thenReturn(True)
+        when(app).start_mysql().thenReturn(None)
+        when(app).stop_db().thenReturn(None)
         app.secure('foo')
         verify(mock_conn, never).execute(TextClauseMatcher('root'))
 
@@ -883,16 +900,16 @@ class ServiceRegistryTest(testtools.TestCase):
     def tearDown(self):
         super(ServiceRegistryTest, self).tearDown()
 
-    def test_service_registry_with_extra_manager(self):
-        service_registry_ext_test = {
+    def test_datastore_registry_with_extra_manager(self):
+        datastore_registry_ext_test = {
             'test': 'trove.guestagent.datastore.test.manager.Manager',
         }
         dbaas_sr.get_custom_managers = Mock(return_value=
-                                            service_registry_ext_test)
-        test_dict = dbaas_sr.service_registry()
+                                            datastore_registry_ext_test)
+        test_dict = dbaas_sr.datastore_registry()
         self.assertEqual(3, len(test_dict))
         self.assertEqual(test_dict.get('test'),
-                         service_registry_ext_test.get('test', None))
+                         datastore_registry_ext_test.get('test', None))
         self.assertEqual(test_dict.get('mysql'),
                          'trove.guestagent.datastore.mysql.'
                          'manager.Manager')
@@ -900,14 +917,14 @@ class ServiceRegistryTest(testtools.TestCase):
                          'trove.guestagent.datastore.mysql.'
                          'manager.Manager')
 
-    def test_service_registry_with_existing_manager(self):
-        service_registry_ext_test = {
+    def test_datastore_registry_with_existing_manager(self):
+        datastore_registry_ext_test = {
             'mysql': 'trove.guestagent.datastore.mysql.'
                      'manager.Manager123',
         }
         dbaas_sr.get_custom_managers = Mock(return_value=
-                                            service_registry_ext_test)
-        test_dict = dbaas_sr.service_registry()
+                                            datastore_registry_ext_test)
+        test_dict = dbaas_sr.datastore_registry()
         self.assertEqual(2, len(test_dict))
         self.assertEqual(test_dict.get('mysql'),
                          'trove.guestagent.datastore.mysql.'
@@ -916,11 +933,11 @@ class ServiceRegistryTest(testtools.TestCase):
                          'trove.guestagent.datastore.mysql.'
                          'manager.Manager')
 
-    def test_service_registry_with_blank_dict(self):
-        service_registry_ext_test = dict()
+    def test_datastore_registry_with_blank_dict(self):
+        datastore_registry_ext_test = dict()
         dbaas_sr.get_custom_managers = Mock(return_value=
-                                            service_registry_ext_test)
-        test_dict = dbaas_sr.service_registry()
+                                            datastore_registry_ext_test)
+        test_dict = dbaas_sr.datastore_registry()
         self.assertEqual(2, len(test_dict))
         self.assertEqual(test_dict.get('mysql'),
                          'trove.guestagent.datastore.mysql.'

trove/tests/unittests/guestagent/test_manager.py

@@ -24,6 +24,7 @@ from trove.guestagent.datastore.mysql.manager import Manager
 import trove.guestagent.datastore.mysql.service as dbaas
 from trove.guestagent import backup
 from trove.guestagent.volume import VolumeDevice
+from trove.guestagent import pkg
 
 
 class GuestAgentManagerTest(testtools.TestCase):
@@ -37,10 +38,10 @@ class GuestAgentManagerTest(testtools.TestCase):
         self.origin_format = volume.VolumeDevice.format
         self.origin_migrate_data = volume.VolumeDevice.migrate_data
         self.origin_mount = volume.VolumeDevice.mount
-        self.origin_is_installed = dbaas.MySqlApp.is_installed
         self.origin_stop_mysql = dbaas.MySqlApp.stop_db
         self.origin_start_mysql = dbaas.MySqlApp.start_mysql
-        self.origin_install_mysql = dbaas.MySqlApp._install_mysql
+        self.origin_pkg_is_installed = pkg.Package.pkg_is_installed
+        self.origin_os_path_exists = os.path.exists
 
     def tearDown(self):
         super(GuestAgentManagerTest, self).tearDown()
@@ -49,10 +50,10 @@ class GuestAgentManagerTest(testtools.TestCase):
         volume.VolumeDevice.format = self.origin_format
         volume.VolumeDevice.migrate_data = self.origin_migrate_data
         volume.VolumeDevice.mount = self.origin_mount
-        dbaas.MySqlApp.is_installed = self.origin_is_installed
         dbaas.MySqlApp.stop_db = self.origin_stop_mysql
         dbaas.MySqlApp.start_mysql = self.origin_start_mysql
-        dbaas.MySqlApp._install_mysql = self.origin_install_mysql
+        pkg.Package.pkg_is_installed = self.origin_pkg_is_installed
+        os.path.exists = self.origin_os_path_exists
         unstub()
 
     def test_update_status(self):
@@ -139,8 +140,6 @@ class GuestAgentManagerTest(testtools.TestCase):
 
         # covering all outcomes is starting to cause trouble here
         COUNT = 1 if device_path else 0
-        SEC_COUNT = 1 if is_mysql_installed else 0
-        migrate_count = 1 * COUNT if not backup_id else 0
 
         # TODO(juice): this should stub an instance of the MySqlAppStatus
         mock_status = mock()
@@ -155,16 +154,18 @@ class GuestAgentManagerTest(testtools.TestCase):
         when(backup).restore(self.context, backup_id).thenReturn(None)
         when(dbaas.MySqlApp).secure(any()).thenReturn(None)
         when(dbaas.MySqlApp).secure_root(any()).thenReturn(None)
-        when(dbaas.MySqlApp).is_installed().thenReturn(is_mysql_installed)
+        (when(pkg.Package).pkg_is_installed(any()).
+         thenReturn(is_mysql_installed))
         when(dbaas.MySqlAdmin).is_root_enabled().thenReturn(is_root_enabled)
         when(dbaas.MySqlAdmin).create_user().thenReturn(None)
         when(dbaas.MySqlAdmin).create_database().thenReturn(None)
         when(dbaas.MySqlAdmin).report_root_enabled(self.context).thenReturn(
             None)
 
-        when(os.path).exists(any()).thenReturn(is_mysql_installed)
+        when(os.path).exists(any()).thenReturn(True)
         # invocation
-        self.manager.prepare(context=self.context, databases=None,
+        self.manager.prepare(context=self.context, packages=None,
+                             databases=None,
                              memory_mb='2048', users=None,
                              device_path=device_path,
                              mount_point='/var/lib/mysql',
@@ -173,12 +174,11 @@ class GuestAgentManagerTest(testtools.TestCase):
         verify(mock_status).begin_install()
 
         verify(VolumeDevice, times=COUNT).format()
-        verify(dbaas.MySqlApp, times=(COUNT * SEC_COUNT)).stop_db()
-        verify(VolumeDevice, times=(migrate_count * SEC_COUNT)).migrate_data(
+        verify(dbaas.MySqlApp, times=COUNT).stop_db()
+        verify(VolumeDevice, times=COUNT).migrate_data(
             any())
         if backup_id:
             verify(backup).restore(self.context, backup_id, '/var/lib/mysql')
-        verify(dbaas.MySqlApp).install_if_needed()
         # We dont need to make sure the exact contents are there
         verify(dbaas.MySqlApp).secure(any())
         verify(dbaas.MySqlAdmin, never).create_database()

trove/tests/unittests/guestagent/test_pkg.py

@@ -17,6 +17,7 @@
 
 import testtools
 from mock import Mock
+from mockito import when, any
 import pexpect
 from trove.common import utils
 from trove.common import exception
@@ -38,10 +39,12 @@ class PkgDEBInstallTestCase(testtools.TestCase):
         self.pexpect_spawn_closed = pexpect.spawn.close
         self.pkg = pkg.DebianPackagerMixin()
         self.pkg_fix = self.pkg._fix
+        self.pkg_fix_package_selections = self.pkg._fix_package_selections
         utils.execute = Mock()
         pexpect.spawn.__init__ = Mock(return_value=None)
         pexpect.spawn.closed = Mock(return_value=None)
         self.pkg._fix = Mock(return_value=None)
+        self.pkg._fix_package_selections = Mock(return_value=None)
         self.pkgName = 'packageName'
 
     def tearDown(self):
@@ -50,53 +53,78 @@ class PkgDEBInstallTestCase(testtools.TestCase):
         pexpect.spawn.__init__ = self.pexpect_spawn_init
         pexpect.spawn.close = self.pexpect_spawn_closed
         self.pkg._fix = self.pkg_fix
+        self.pkg._fix_package_selections = self.pkg_fix_package_selections
+
+    def test_pkg_is_instaled_no_packages(self):
+        packages = ""
+        self.assertTrue(self.pkg.pkg_is_installed(packages))
+
+    def test_pkg_is_instaled_yes(self):
+        packages = "package1=1.0 package2"
+        when(self.pkg).pkg_version("package1").thenReturn("1.0")
+        when(self.pkg).pkg_version("package2").thenReturn("2.0")
+        self.assertTrue(self.pkg.pkg_is_installed(packages))
+
+    def test_pkg_is_instaled_no(self):
+        packages = "package1=1.0 package2 package3=3.1"
+        when(self.pkg).pkg_version("package1").thenReturn("1.0")
+        when(self.pkg).pkg_version("package2").thenReturn("2.0")
+        when(self.pkg).pkg_version("package3").thenReturn("3.0")
+        self.assertFalse(self.pkg.pkg_is_installed(packages))
 
     def test_success_install(self):
         # test
-        pexpect.spawn.expect = Mock(return_value=5)
-        self.assertTrue(self.pkg.pkg_install(self.pkgName, 5000) is None)
-        # verify
-
-    def test_already_instaled(self):
-        # test happy path
-        pexpect.spawn.expect = Mock(return_value=6)
-        self.pkg.pkg_install(self.pkgName, 5000)
+        pexpect.spawn.expect = Mock(return_value=7)
+        pexpect.spawn.match = False
+        self.assertTrue(self.pkg.pkg_install(self.pkgName, {}, 5000) is None)
 
     def test_permission_error(self):
         # test
         pexpect.spawn.expect = Mock(return_value=0)
+        pexpect.spawn.match = False
         # test and verify
         self.assertRaises(pkg.PkgPermissionError, self.pkg.pkg_install,
-                          self.pkgName, 5000)
+                          self.pkgName, {}, 5000)
 
     def test_package_not_found_1(self):
         # test
         pexpect.spawn.expect = Mock(return_value=1)
+        pexpect.spawn.match = re.match('(.*)', self.pkgName)
         # test and verify
         self.assertRaises(pkg.PkgNotFoundError, self.pkg.pkg_install,
-                          self.pkgName, 5000)
+                          self.pkgName, {}, 5000)
 
     def test_package_not_found_2(self):
         # test
         pexpect.spawn.expect = Mock(return_value=2)
+        pexpect.spawn.match = re.match('(.*)', self.pkgName)
         # test and verify
         self.assertRaises(pkg.PkgNotFoundError, self.pkg.pkg_install,
-                          self.pkgName, 5000)
+                          self.pkgName, {}, 5000)
 
     def test_run_DPKG_bad_State(self):
         # test _fix method is called and PackageStateError is thrown
-        pexpect.spawn.expect = Mock(return_value=3)
+        pexpect.spawn.expect = Mock(return_value=4)
+        pexpect.spawn.match = False
         # test and verify
         self.assertRaises(pkg.PkgPackageStateError, self.pkg.pkg_install,
-                          self.pkgName, 5000)
+                          self.pkgName, {}, 5000)
         self.assertTrue(self.pkg._fix.called)
 
     def test_admin_lock_error(self):
         # test 'Unable to lock the administration directory' error
-        pexpect.spawn.expect = Mock(return_value=4)
+        pexpect.spawn.expect = Mock(return_value=5)
+        pexpect.spawn.match = False
         # test and verify
         self.assertRaises(pkg.PkgAdminLockError, self.pkg.pkg_install,
-                          self.pkgName, 5000)
+                          self.pkgName, {}, 5000)
+
+    def test_package_broken_error(self):
+        pexpect.spawn.expect = Mock(return_value=6)
+        pexpect.spawn.match = False
+        # test and verify
+        self.assertRaises(pkg.PkgBrokenError, self.pkg.pkg_install,
+                          self.pkgName, {}, 5000)
 
     def test_timeout_error(self):
         # test timeout error
@@ -104,7 +132,7 @@ class PkgDEBInstallTestCase(testtools.TestCase):
                                     TIMEOUT('timeout error'))
         # test and verify
         self.assertRaises(pkg.PkgTimeout, self.pkg.pkg_install,
-                          self.pkgName, 5000)
+                          self.pkgName, {}, 5000)
 
 
 class PkgDEBRemoveTestCase(testtools.TestCase):
@@ -140,11 +168,13 @@ class PkgDEBRemoveTestCase(testtools.TestCase):
     def test_success_remove(self):
         # test
         pexpect.spawn.expect = Mock(return_value=6)
+        pexpect.spawn.match = False
         self.assertTrue(self.pkg.pkg_remove(self.pkgName, 5000) is None)
 
     def test_permission_error(self):
         # test
         pexpect.spawn.expect = Mock(return_value=0)
+        pexpect.spawn.match = False
         # test and verify
         self.assertRaises(pkg.PkgPermissionError, self.pkg.pkg_remove,
                           self.pkgName, 5000)
@@ -152,6 +182,7 @@ class PkgDEBRemoveTestCase(testtools.TestCase):
     def test_package_not_found(self):
         # test
         pexpect.spawn.expect = Mock(return_value=1)
+        pexpect.spawn.match = False
         # test and verify
         self.assertRaises(pkg.PkgNotFoundError, self.pkg.pkg_remove,
                           self.pkgName, 5000)
@@ -159,6 +190,7 @@ class PkgDEBRemoveTestCase(testtools.TestCase):
     def test_package_reinstall_first_1(self):
         # test
         pexpect.spawn.expect = Mock(return_value=2)
+        pexpect.spawn.match = False
         # test and verify
         self.assertRaises(pkg.PkgPackageStateError, self.pkg.pkg_remove,
                           self.pkgName, 5000)
@@ -168,6 +200,7 @@ class PkgDEBRemoveTestCase(testtools.TestCase):
     def test_package_reinstall_first_2(self):
         # test
         pexpect.spawn.expect = Mock(return_value=3)
+        pexpect.spawn.match = False
         # test and verify
         self.assertRaises(pkg.PkgPackageStateError, self.pkg.pkg_remove,
                           self.pkgName, 5000)
@@ -177,6 +210,7 @@ class PkgDEBRemoveTestCase(testtools.TestCase):
     def test_package_DPKG_first(self):
         # test
         pexpect.spawn.expect = Mock(return_value=4)
+        pexpect.spawn.match = False
         # test and verify
         self.assertRaises(pkg.PkgPackageStateError, self.pkg.pkg_remove,
                           self.pkgName, 5000)
@@ -186,6 +220,7 @@ class PkgDEBRemoveTestCase(testtools.TestCase):
     def test_admin_lock_error(self):
         # test 'Unable to lock the administration directory' error
         pexpect.spawn.expect = Mock(return_value=5)
+        pexpect.spawn.match = False
         # test and verify
         self.assertRaises(pkg.PkgAdminLockError, self.pkg.pkg_remove,
                           self.pkgName, 5000)
@@ -201,22 +236,6 @@ class PkgDEBRemoveTestCase(testtools.TestCase):
 
 class PkgDEBVersionTestCase(testtools.TestCase):
 
-    @staticmethod
-    def build_output(packageName, packageVersion, parts=None):
-        if parts is None:
-            parts = "ii " + packageName + "  " + packageVersion + \
-                    "  MySQL database server binaries "\
-                    "and system database setup \n"
-        cmd_out = "Desired=Unknown/Install/Remove/Purge/Hold\n" \
-                  "| Status=Not/Inst/Conf-files/Unpacked/halF-conf/"\
-                  "Half-inst/trig-aWait/Trig-pend\n" \
-                  "|/ Err?=(none)/Reinst-required "\
-                  "(Status,Err: uppercase=bad)\n"\
-                  "||/ Name   Version  Description\n" \
-                  "+++-==============-================-=============\n" \
-                  "=================================\n" + parts
-        return cmd_out
-
     def setUp(self):
         super(PkgDEBVersionTestCase, self).setUp()
         self.pkgName = 'mysql-server-5.5'
@@ -228,43 +247,19 @@ class PkgDEBVersionTestCase(testtools.TestCase):
         commands.getstatusoutput = self.commands_output
 
     def test_version_success(self):
-        cmd_out = self.build_output(self.pkgName, self.pkgVersion)
+        cmd_out = "%s:\n  Installed: %s\n" % (self.pkgName, self.pkgVersion)
         commands.getstatusoutput = Mock(return_value=(0, cmd_out))
         version = pkg.DebianPackagerMixin().pkg_version(self.pkgName)
         self.assertTrue(version)
         self.assertEqual(self.pkgVersion, version)
 
-    def test_version_status_error(self):
-        cmd_out = self.build_output(self.pkgName, self.pkgVersion)
-        commands.getstatusoutput = Mock(return_value=(1, cmd_out))
-        self.assertFalse(pkg.DebianPackagerMixin().pkg_version(self.pkgName))
-
-    def test_version_no_output(self):
-        cmd_out = self.build_output(self.pkgName, self.pkgVersion, "")
-        commands.getstatusoutput = Mock(return_value=(0, cmd_out))
-        self.assertIsNone(pkg.DebianPackagerMixin().pkg_version(self.pkgName))
-
-    def test_version_unexpected_parts(self):
-        unexp_parts = "ii   123"
-        cmd_out = self.build_output(self.pkgName, self.pkgVersion, unexp_parts)
-        commands.getstatusoutput = Mock(return_value=(0, cmd_out))
-        self.assertIsNone(pkg.DebianPackagerMixin().pkg_version(self.pkgName))
-
-    def test_version_wrong_package(self):
-        invalid_pkg = "package_invalid_001"
-        cmd_out = self.build_output(invalid_pkg, self.pkgVersion)
-        commands.getstatusoutput = Mock(return_value=(0, cmd_out))
-        self.assertRaises(exception.GuestError,
-                          pkg.DebianPackagerMixin().pkg_version, self.pkgName)
-
     def test_version_unknown_package(self):
-        unk_parts = "un   " + self.pkgName + "   " + self.pkgVersion + "   \n"
-        cmd_out = self.build_output(self.pkgName, self.pkgVersion, unk_parts)
+        cmd_out = "N: Unable to locate package %s" % self.pkgName
         commands.getstatusoutput = Mock(return_value=(0, cmd_out))
         self.assertFalse(pkg.DebianPackagerMixin().pkg_version(self.pkgName))
 
     def test_version_no_version(self):
-        cmd_out = self.build_output(self.pkgName, '<none>')
+        cmd_out = "%s:\n  Installed: %s\n" % (self.pkgName, "(none)")
         commands.getstatusoutput = Mock(return_value=(0, cmd_out))
         self.assertFalse(pkg.DebianPackagerMixin().pkg_version(self.pkgName))
 
@@ -313,73 +308,107 @@ class PkgRPMInstallTestCase(testtools.TestCase):
         pexpect.spawn.__init__ = self.pexpect_spawn_init
         pexpect.spawn.close = self.pexpect_spawn_closed
 
+    def test_pkg_is_instaled_no_packages(self):
+        packages = ""
+        self.assertTrue(self.pkg.pkg_is_installed(packages))
+
+    def test_pkg_is_instaled_yes(self):
+        packages = "package1=1.0 package2"
+        when(commands).getstatusoutput(any()).thenReturn({1: "package1=1.0\n"
+                                                             "package2=2.0"})
+        self.assertTrue(self.pkg.pkg_is_installed(packages))
+
+    def test_pkg_is_instaled_no(self):
+        packages = "package1=1.0 package2 package3=3.0"
+        when(commands).getstatusoutput({1: "package1=1.0\npackage2=2.0"})
+        self.assertFalse(self.pkg.pkg_is_installed(packages))
+
     def test_permission_error(self):
         # test
         pexpect.spawn.expect = Mock(return_value=0)
+        pexpect.spawn.match = False
         # test and verify
         self.assertRaises(pkg.PkgPermissionError, self.pkg.pkg_install,
-                          self.pkgName, 5000)
+                          self.pkgName, {}, 5000)
 
     def test_package_not_found(self):
         # test
         pexpect.spawn.expect = Mock(return_value=1)
+        pexpect.spawn.match = re.match('(.*)', self.pkgName)
         # test and verify
         self.assertRaises(pkg.PkgNotFoundError, self.pkg.pkg_install,
-                          self.pkgName, 5000)
+                          self.pkgName, {}, 5000)
 
-    def test_transaction_check_error(self):
+    def test_package_conflict_remove(self):
         # test
         pexpect.spawn.expect = Mock(return_value=2)
+        pexpect.spawn.match = re.match('(.*)', self.pkgName)
+        self.pkg._rpm_remove_nodeps = Mock()
         # test and verify
-        self.assertRaises(pkg.PkgTransactionCheckError, self.pkg.pkg_install,
-                          self.pkgName, 5000)
+        self.pkg._install(self.pkgName, 5000)
+        self.assertTrue(self.pkg._rpm_remove_nodeps.called)
 
     def test_package_scriptlet_error(self):
         # test
-        pexpect.spawn.expect = Mock(return_value=3)
+        pexpect.spawn.expect = Mock(return_value=5)
+        pexpect.spawn.match = False
         # test and verify
         self.assertRaises(pkg.PkgScriptletError, self.pkg.pkg_install,
-                          self.pkgName, 5000)
+                          self.pkgName, {}, 5000)
 
     def test_package_http_error(self):
         # test
-        pexpect.spawn.expect = Mock(return_value=4)
+        pexpect.spawn.expect = Mock(return_value=6)
+        pexpect.spawn.match = False
         # test and verify
         self.assertRaises(pkg.PkgDownloadError, self.pkg.pkg_install,
-                          self.pkgName, 5000)
+                          self.pkgName, {}, 5000)
 
     def test_package_nomirrors_error(self):
         # test
-        pexpect.spawn.expect = Mock(return_value=5)
+        pexpect.spawn.expect = Mock(return_value=7)
+        pexpect.spawn.match = False
         # test and verify
         self.assertRaises(pkg.PkgDownloadError, self.pkg.pkg_install,
-                          self.pkgName, 5000)
+                          self.pkgName, {}, 5000)
+
+    def test_package_sign_error(self):
+        # test
+        pexpect.spawn.expect = Mock(return_value=8)
+        pexpect.spawn.match = False
+        # test and verify
+        self.assertRaises(pkg.PkgSignError, self.pkg.pkg_install,
+                          self.pkgName, {}, 5000)
 
     def test_package_already_installed(self):
         # test
-        pexpect.spawn.expect = Mock(return_value=6)
+        pexpect.spawn.expect = Mock(return_value=9)
+        pexpect.spawn.match = False
         # test and verify
-        self.assertTrue(self.pkg.pkg_install(self.pkgName, 5000) is None)
+        self.assertTrue(self.pkg.pkg_install(self.pkgName, {}, 5000) is None)
 
     def test_package_success_updated(self):
         # test
-        pexpect.spawn.expect = Mock(return_value=7)
+        pexpect.spawn.expect = Mock(return_value=10)
+        pexpect.spawn.match = False
         # test and verify
-        self.assertTrue(self.pkg.pkg_install(self.pkgName, 5000) is None)
+        self.assertTrue(self.pkg.pkg_install(self.pkgName, {}, 5000) is None)
 
     def test_package_success_installed(self):
         # test
-        pexpect.spawn.expect = Mock(return_value=8)
+        pexpect.spawn.expect = Mock(return_value=11)
+        pexpect.spawn.match = False
         # test and verify
-        self.assertTrue(self.pkg.pkg_install(self.pkgName, 5000) is None)
+        self.assertTrue(self.pkg.pkg_install(self.pkgName, {}, 5000) is None)
 
     def test_timeout_error(self):
         # test timeout error
         pexpect.spawn.expect = Mock(side_effect=pexpect.
                                     TIMEOUT('timeout error'))
+        pexpect.spawn.match = False
         # test and verify
         self.assertRaises(pkg.PkgTimeout, self.pkg.pkg_install,
-                          self.pkgName, 5000)
+                          self.pkgName, {}, 5000)
 
 
 class PkgRPMRemoveTestCase(testtools.TestCase):

trove/tests/unittests/mgmt/test_models.py

@@ -24,6 +24,7 @@ from oslo.config.cfg import ConfigOpts
 from trove.backup.models import Backup
 from trove.common.context import TroveContext
 from trove.common import instance as rd_instance
+from trove.datastore import models as datastore_models
 from trove.db.models import DatabaseModelBase
 from trove.instance.models import DBInstance
 from trove.instance.models import InstanceServiceStatus
@@ -31,6 +32,7 @@ from trove.instance.tasks import InstanceTasks
 import trove.extensions.mgmt.instances.models as mgmtmodels
 from trove.openstack.common.notifier import api as notifier
 from trove.common import remote
+from trove.tests.util import test_config
 
 
 class MockMgmtInstanceTest(TestCase):
@@ -62,11 +64,12 @@ class MockMgmtInstanceTest(TestCase):
                           name='test_name',
                           id='1',
                           flavor_id='flavor_1',
+                          datastore_version_id=
+                          test_config.dbaas_datastore_version,
                           compute_instance_id='compute_id_1',
                           server_id='server_id_1',
                           tenant_id='tenant_id_1',
-                          server_status=status,
-                          service_type='mysql')
+                          server_status=status)
 
 
 class TestNotificationTransformer(MockMgmtInstanceTest):
@@ -78,6 +81,10 @@ class TestNotificationTransformer(MockMgmtInstanceTest):
 
         when(DatabaseModelBase).find_all(deleted=False).thenReturn(
             [db_instance])
+        stub_datastore = mock()
+        stub_datastore.datastore_id = "stub"
+        stub_datastore.manager = "mysql"
+        when(DatabaseModelBase).find_by(id=any()).thenReturn(stub_datastore)
         when(DatabaseModelBase).find_by(instance_id='1').thenReturn(
             InstanceServiceStatus(rd_instance.ServiceStatuses.BUILDING))
 
@@ -165,14 +172,17 @@ class TestNovaNotificationTransformer(MockMgmtInstanceTest):
         self.assertThat(payload['user_id'], Equals('test_user_id'))
         self.assertThat(payload['service_id'], Equals('123'))
 
-    def test_tranformer_invalid_service_type(self):
+    def test_tranformer_invalid_datastore_manager(self):
         status = rd_instance.ServiceStatuses.BUILDING.api_status
         db_instance = MockMgmtInstanceTest.build_db_instance(
             status, task_status=InstanceTasks.BUILDING)
-        db_instance.service_type = 'm0ng0'
 
         server = mock(Server)
         server.user_id = 'test_user_id'
+        stub_datastore = mock()
+        stub_datastore.manager = "m0ng0"
+        when(datastore_models.
+             Datastore).load(any()).thenReturn(stub_datastore)
         mgmt_instance = mgmtmodels.SimpleMgmtInstance(self.context,
                                                       db_instance,
                                                       server,

trove/tests/unittests/taskmanager/test_models.py

@@ -15,6 +15,7 @@ import testtools
 from mock import Mock
 from testtools.matchers import Equals
 from mockito import mock, when, unstub, any, verify, never
+from trove.datastore import models as datastore_models
 from trove.taskmanager import models as taskmanager_models
 import trove.common.remote as remote
 from trove.common.instance import ServiceStatuses
@@ -138,6 +139,10 @@ class FreshInstanceTasksTest(testtools.TestCase):
             "hostname")
         when(taskmanager_models.FreshInstanceTasks).name().thenReturn(
             'name')
+        when(datastore_models.
+             DatastoreVersion).load(any()).thenReturn(mock())
+        when(datastore_models.
+             Datastore).load(any()).thenReturn(mock())
         taskmanager_models.FreshInstanceTasks.nova_client = fake_nova_client()
         taskmanager_models.CONF = mock()
         when(taskmanager_models.CONF).get(any()).thenReturn('')
@@ -152,7 +157,7 @@ class FreshInstanceTasksTest(testtools.TestCase):
             self.guestconfig = f.name
             f.write(self.guestconfig_content)
         self.freshinstancetasks = taskmanager_models.FreshInstanceTasks(
-            None, None, None, None)
+            None, mock(), None, None)
 
     def tearDown(self):
         super(FreshInstanceTasksTest, self).tearDown()
@@ -164,11 +169,12 @@ class FreshInstanceTasksTest(testtools.TestCase):
 
     def test_create_instance_userdata(self):
         cloudinit_location = os.path.dirname(self.cloudinit)
-        service_type = os.path.splitext(os.path.basename(self.cloudinit))[0]
+        datastore_manager = os.path.splitext(os.path.basename(self.
+                                                              cloudinit))[0]
         when(taskmanager_models.CONF).get("cloudinit_location").thenReturn(
             cloudinit_location)
         server = self.freshinstancetasks._create_server(
-            None, None, None, service_type, None, None)
+            None, None, None, datastore_manager, None, None)
         self.assertEqual(server.userdata, self.userdata)
 
     def test_create_instance_guestconfig(self):
@@ -181,23 +187,20 @@ class FreshInstanceTasksTest(testtools.TestCase):
                          self.guestconfig_content)
 
     def test_create_instance_with_az_kwarg(self):
-        service_type = 'mysql'
         server = self.freshinstancetasks._create_server(
-            None, None, None, service_type, None, availability_zone='nova')
+            None, None, None, None, None, availability_zone='nova')
 
         self.assertIsNotNone(server)
 
     def test_create_instance_with_az(self):
-        service_type = 'mysql'
         server = self.freshinstancetasks._create_server(
-            None, None, None, service_type, None, 'nova')
+            None, None, None, None, None, 'nova')
 
         self.assertIsNotNone(server)
 
     def test_create_instance_with_az_none(self):
-        service_type = 'mysql'
         server = self.freshinstancetasks._create_server(
-            None, None, None, service_type, None, None)
+            None, None, None, None, None, None)
 
         self.assertIsNotNone(server)
 

trove/tests/util/client.py

@@ -102,20 +102,6 @@ class TestClient(object):
         flavor_href = self.find_flavor_self_href(flavor)
         return flavor, flavor_href
 
-    def find_image_and_self_href(self, image_id):
-        """Given an ID, returns tuple with image and its self href."""
-        assert_false(image_id is None)
-        image = self.images.get(image_id)
-        assert_true(image is not None)
-        self_links = [link['href'] for link in image.links
-                      if link['rel'] == 'self']
-        assert_true(len(self_links) > 0,
-                    "Found image with ID %s but it had no self link!" %
-                    str(image_id))
-        image_href = self_links[0]
-        assert_false(image_href is None, "Image link self href missing.")
-        return image, image_href
-
     def __getattr__(self, item):
         return getattr(self.real_client, item)