percona image for reddwarf

Implements: blueprint "percona-image-for-reddwarf"
Change-Id: I9160f4ac48ca5824d8176cd7d53b2d4458d84642

contrib/dbaas-mycnf/etc/my.cnf.base

@@ -46,6 +46,7 @@ basedir = /usr
 datadir = /var/lib/mysql
 ####tmpdir = /tmp
 tmpdir = /var/tmp
+pid_file = /var/run/mysqld/mysqld.pid
 skip-external-locking
 #
 # Instead of skip-networking the default is now to listen only on

contrib/dbaas-mycnf/etc/my.cnf.default

@@ -46,6 +46,7 @@ basedir = /usr
 datadir = /var/lib/mysql
 ####tmpdir = /tmp
 tmpdir = /var/tmp
+pid_file = /var/run/mysqld/mysqld.pid
 skip-external-locking
 #
 # Instead of skip-networking the default is now to listen only on

etc/reddwarf/reddwarf-taskmanager.conf.sample

@@ -57,7 +57,7 @@ reddwarf_dns_support = False
 # Guest related conf
 agent_heartbeat_time = 10
 agent_call_low_timeout = 5
-agent_call_high_timeout = 100
+agent_call_high_timeout = 150
 
 # Whether to use nova's contrib api for create server with volume
 use_nova_server_volume = False

etc/reddwarf/reddwarf.conf.sample

@@ -69,7 +69,7 @@ ignore_dbs = lost+found, mysql, information_schema
 # Guest related conf
 agent_heartbeat_time = 10
 agent_call_low_timeout = 5
-agent_call_high_timeout = 100
+agent_call_high_timeout = 150
 
 # Reboot time out for instances
 reboot_time_out = 60

etc/reddwarf/reddwarf.conf.test

@@ -78,7 +78,7 @@ ignore_dbs = lost+found, mysql, information_schema
 # Guest related conf
 agent_heartbeat_time = 10
 agent_call_low_timeout = 5
-agent_call_high_timeout = 100
+agent_call_high_timeout = 150
 
 server_delete_time_out=10
 use_nova_server_volume = False

reddwarf/common/cfg.py

@@ -48,6 +48,7 @@ common_opts = [
     cfg.IntOpt('periodic_interval', default=60),
     cfg.BoolOpt('reddwarf_dns_support', default=False),
     cfg.StrOpt('db_api_implementation', default='reddwarf.db.sqlalchemy.api'),
+    cfg.StrOpt('mysql_pkg', default='mysql-server-5.5'),
     cfg.StrOpt('dns_driver', default='reddwarf.dns.driver.DnsDriver'),
     cfg.StrOpt('dns_instance_entry_factory',
                default='reddwarf.dns.driver.DnsInstanceEntryFactory'),
@@ -68,7 +69,7 @@ common_opts = [
     cfg.IntOpt('agent_call_low_timeout', default=5),
     cfg.IntOpt('agent_call_high_timeout', default=60),
     cfg.StrOpt('guest_id', default=None),
-    cfg.IntOpt('state_change_wait_time', default=2 * 60),
+    cfg.IntOpt('state_change_wait_time', default=3 * 60),
     cfg.IntOpt('agent_heartbeat_time', default=10),
     cfg.IntOpt('num_tries', default=3),
     cfg.StrOpt('volume_fstype', default='ext3'),

reddwarf/common/utils.py

@@ -254,6 +254,7 @@ def execute_with_timeout(*args, **kwargs):
         msg = _("Time out after waiting"
                 " %(time)s seconds when running proc: %(args)s"
                 " %(kwargs)s") % locals()
+        LOG.error(msg)
         raise exception.ProcessExecutionError(msg)
 
     timeout = Timeout(time)
@@ -261,11 +262,13 @@ def execute_with_timeout(*args, **kwargs):
         return execute(*args, **kwargs)
     except Timeout as t:
         if t is not timeout:
+            LOG.error("Timeout reached but not from our timeout. This is bad!")
             raise
         else:
             msg = _("Time out after waiting "
                     "%(time)s seconds when running proc: %(args)s"
                     " %(kwargs)s") % locals()
+            LOG.error(msg)
             raise exception.ProcessExecutionError(msg)
     finally:
         timeout.cancel()

reddwarf/guestagent/api.py

@@ -47,7 +47,7 @@ class API(proxy.RpcProxy):
                                   RPC_API_VERSION)
 
     def _call(self, method_name, timeout_sec, **kwargs):
-        LOG.debug("Calling %s" % method_name)
+        LOG.debug("Calling %s with timeout %s" % (method_name, timeout_sec))
         try:
             result = self.call(self.context,
                                self.make_msg(method_name, **kwargs),
@@ -212,7 +212,8 @@ class API(proxy.RpcProxy):
 
     def start_mysql_with_conf_changes(self, updated_memory_size):
         """Start the MySQL server."""
-        LOG.debug(_("Sending the call to start MySQL on the Guest."))
+        LOG.debug(_("Sending the call to start MySQL on the Guest with "
+                    "a timeout of %s.") % AGENT_HIGH_TIMEOUT)
         self._call("start_mysql_with_conf_changes", AGENT_HIGH_TIMEOUT,
                    updated_memory_size=updated_memory_size)
 

reddwarf/guestagent/dbaas.py

@@ -161,7 +161,7 @@ class MySqlAppStatus(object):
 
         Updates the database with the actual MySQL status.
         """
-        LOG.info("Ending install or restart.")
+        LOG.info("Ending install_if_needed or restart.")
         self.restart_mode = False
         real_status = self._get_actual_db_status()
         LOG.info("Updating status to %s" % real_status)
@@ -599,7 +599,7 @@ class MySqlApp(object):
     """Prepares DBaaS on a Guest container."""
 
     TIME_OUT = 1000
-    MYSQL_PACKAGE_VERSION = "mysql-server-5.5"
+    MYSQL_PACKAGE_VERSION = CONF.mysql_pkg
 
     def __init__(self, status):
         """ By default login with root no password for initial setup. """
@@ -632,13 +632,14 @@ class MySqlApp(object):
         t = text(str(uu))
         client.execute(t)
 
-    def install_and_secure(self, memory_mb):
+    def install_if_needed(self):
         """Prepare the guest machine with a secure mysql server installation"""
         LOG.info(_("Preparing Guest as MySQL Server"))
+        if not self.is_installed():
+            self._install_mysql()
+        LOG.info(_("Dbaas install_if_needed complete"))
 
-        #TODO(tim.simpson): Check that MySQL is not already installed.
+    def secure(self, memory_mb):
-        self.status.begin_mysql_install()
-        self._install_mysql()
         LOG.info(_("Generating root password..."))
         admin_password = generate_random_password()
 
@@ -654,10 +655,10 @@ class MySqlApp(object):
         self.start_mysql()
 
         self.status.end_install_or_restart()
-        LOG.info(_("Dbaas install_and_secure complete."))
+        LOG.info(_("Dbaas secure complete."))
 
     def _install_mysql(self):
-        """Install mysql server. The current version is 5.1"""
+        """Install mysql server. The current version is 5.5"""
         LOG.debug(_("Installing mysql server"))
         pkg.pkg_install(self.MYSQL_PACKAGE_VERSION, self.TIME_OUT)
         LOG.debug(_("Finished installing mysql server"))
@@ -665,24 +666,38 @@ class MySqlApp(object):
 
     def _enable_mysql_on_boot(self):
         '''
-        # This works in Debian Squeeze, but Ubuntu Precise has other plans.
+        There is a difference between the init.d mechanism and the upstart
-        # Use update-rc.d to enable or disable mysql at boot.
+        The stock mysql uses the upstart mechanism, therefore, there is a
-        # update-rc.d is idempotent; any substitute method should be, too.
+        mysql.conf file responsible for the job. to toggle enable/disable
-        flag = "enable" if enabled else "disable"
+        on boot one needs to modify this file. Percona uses the init.d
-        LOG.info("Setting mysql to '%s' in rc.d" % flag)
+        mechanism and there is no mysql.conf file. Instead, the update-rc.d
-        utils.execute_with_timeout("sudo", "update-rc.d", "mysql", flag)
+        command needs to be used to modify the /etc/rc#.d/[S/K]##mysql links
         '''
         LOG.info("Enabling mysql on boot.")
         conf = "/etc/init/mysql.conf"
-        command = "sudo sed -i '/^manual$/d' %(conf)s"
+        if os.path.isfile(conf):
-        command = command % locals()
+            command = "sudo sed -i '/^manual$/d' %(conf)s"
+            command = command % locals()
+        else:
+            command = "sudo update-rc.d mysql enable"
         utils.execute_with_timeout(command, with_shell=True)
 
     def _disable_mysql_on_boot(self):
+        '''
+        There is a difference between the init.d mechanism and the upstart
+        The stock mysql uses the upstart mechanism, therefore, there is a
+        mysql.conf file responsible for the job. to toggle enable/disable
+        on boot one needs to modify this file. Percona uses the init.d
+        mechanism and there is no mysql.conf file. Instead, the update-rc.d
+        command needs to be used to modify the /etc/rc#.d/[S/K]##mysql links
+        '''
         LOG.info("Disabling mysql on boot.")
         conf = "/etc/init/mysql.conf"
-        command = '''sudo sh -c "echo manual >> %(conf)s"'''
+        if os.path.isfile(conf):
-        command = command % locals()
+            command = '''sudo sh -c "echo manual >> %(conf)s"'''
+            command = command % locals()
+        else:
+            command = "sudo update-rc.d mysql disable"
         utils.execute_with_timeout(command, with_shell=True)
 
     def stop_mysql(self, update_db=False, do_not_start_on_reboot=False):
@@ -718,10 +733,12 @@ class MySqlApp(object):
         LOG.debug("template_path(%s) original_path(%s)"
                   % (template_path, original_path))
         if os.path.isfile(template_path):
-            utils.execute_with_timeout(
+            if os.path.isfile(original_path):
-                "sudo", "mv", original_path,
+                utils.execute_with_timeout(
-                "%(name)s.%(date)s" % {'name': original_path,
+                    "sudo", "mv", original_path,
-                                       'date': date.today().isoformat()})
+                    "%(name)s.%(date)s" %
+                    {'name': original_path, 'date':
+                    date.today().isoformat()})
             utils.execute_with_timeout("sudo", "cp", template_path,
                                        original_path)
 
@@ -807,6 +824,16 @@ class MySqlApp(object):
         try:
             utils.execute_with_timeout("sudo", "/etc/init.d/mysql", "start")
         except ProcessExecutionError:
+            # it seems mysql (percona, at least) might come back with [Fail]
+            # but actually come up ok. we're looking into the timing issue on
+            # parallel, but for now, we'd like to give it one more chance to
+            # come up. so regardless of the execute_with_timeout() respose,
+            # we'll assume mysql comes up and check it's status for a while.
+            pass
+        if not self.status.wait_for_real_status_to_change_to(
+                rd_models.ServiceStatuses.RUNNING,
+                self.state_change_wait_time, update_db):
+            LOG.error(_("Start up of MySQL failed!"))
             # If it won't start, but won't die either, kill it by hand so we
             # don't let a rouge process wander around.
             try:
@@ -815,12 +842,6 @@ class MySqlApp(object):
                 LOG.error("Error killing stalled mysql start command.")
                 LOG.error(p)
             # There's nothing more we can do...
-            raise RuntimeError("Can't start MySQL!")
-
-        if not self.status.wait_for_real_status_to_change_to(
-                rd_models.ServiceStatuses.RUNNING,
-                self.state_change_wait_time, update_db):
-            LOG.error(_("Start up of MySQL failed!"))
             self.status.end_install_or_restart()
             raise RuntimeError("Could not start MySQL!")
 

reddwarf/guestagent/manager.py

@@ -3,6 +3,8 @@ from reddwarf.guestagent import volume
 from reddwarf.openstack.common import log as logging
 from reddwarf.openstack.common import periodic_task
 from reddwarf.openstack.common.gettextutils import _
+from reddwarf.instance import models as rd_models
+import os
 LOG = logging.getLogger(__name__)
 
 MYSQL_BASE_DIR = "/var/lib/mysql"
@@ -62,13 +64,14 @@ class Manager(periodic_task.PeriodicTasks):
                 mount_point=None):
         """Makes ready DBAAS on a Guest container."""
         dbaas.MySqlAppStatus.get().begin_mysql_install()
-        # status end_mysql_install set with install_and_secure()
+        # status end_mysql_install set with secure()
         app = dbaas.MySqlApp(dbaas.MySqlAppStatus.get())
         restart_mysql = False
         if device_path:
             device = volume.VolumeDevice(device_path)
             device.format()
-            if app.is_installed():
+            #if a /var/lib/mysql folder exists, back it up.
+            if os.path.exists(MYSQL_BASE_DIR):
                 #stop and do not update database
                 app.stop_mysql()
                 restart_mysql = True
@@ -80,9 +83,9 @@ class Manager(periodic_task.PeriodicTasks):
             #check mysql was installed and stopped
             if restart_mysql:
                 app.start_mysql()
-        app.install_and_secure(memory_mb)
+        app.install_if_needed()
-        LOG.info("Creating initial databases and users following successful "
+        LOG.info("Securing mysql now.")
-                 "prepare.")
+        app.secure(memory_mb)
         self.create_database(context, databases)
         self.create_user(context, users)
         LOG.info('"prepare" call has finished.')

reddwarf/instance/service.py

@@ -173,7 +173,8 @@ class InstanceController(wsgi.Controller):
         LOG.info(_("body : '%s'\n\n") % body)
         context = req.environ[wsgi.CONTEXT_KEY]
         # Set the service type to mysql if its not in the request
-        service_type = body['instance'].get('service_type') or 'mysql'
+        service_type = (body['instance'].get('service_type') or
+                        CONF.service_type)
         service = models.ServiceImage.find_by(service_name=service_type)
         image_id = service['image_id']
         name = body['instance']['name']

reddwarf/tests/api/instances_actions.py

@@ -82,7 +82,7 @@ class MySqlConnection(object):
             raise ex
 
 
-TIME_OUT_TIME = 10 * 60
+TIME_OUT_TIME = 15 * 60
 USER_WAS_DELETED = False
 
 
@@ -480,11 +480,17 @@ class ResizeInstanceTest(ActionTestBase):
         if CONFIG.simulate_events:
             raise SkipTest("Cannot simulate this test.")
         self.ensure_mysql_is_running()
+
+    @test(depends_on=[test_instance_returns_to_active_after_resize],
+          runs_after=[test_make_sure_mysql_is_running_after_resize])
+    def test_instance_has_new_flavor_after_resize(self):
+        if CONFIG.simulate_events:
+            raise SkipTest("Cannot simulate this test.")
         actual = self.get_flavor_href(self.instance.flavor['id'])
         expected = self.get_flavor_href(flavor_id=self.expected_new_flavor_id)
         assert_equal(actual, expected)
 
-    @test(depends_on=[test_make_sure_mysql_is_running_after_resize])
+    @test(depends_on=[test_instance_has_new_flavor_after_resize])
     @time_out(TIME_OUT_TIME)
     def test_resize_down(self):
         if CONFIG.simulate_events:

reddwarf/tests/unittests/guestagent/test_dbaas.py

@@ -601,43 +601,49 @@ class MySqlAppInstallTest(MySqlAppTest):
         dbaas.create_engine = self.orig_create_engine
         dbaas.pkg.pkg_version = self.orig_pkg_version
 
-    def test_install_and_secure(self):
+    def test_install(self):
+
+        self.mySqlApp._install_mysql = Mock()
+        self.mySqlApp.is_installed = Mock(return_value=False)
+        self.mySqlApp.install_if_needed()
+        self.assertTrue(self.mySqlApp._install_mysql.called)
+        self.assert_reported_status(ServiceStatuses.NEW)
+
+    def test_secure(self):
 
         self.mySqlApp.start_mysql = Mock()
         self.mySqlApp.stop_mysql = Mock()
-        self.mySqlApp._install_mysql = Mock()
         self.mySqlApp._write_mycnf = Mock()
         self.mysql_stops_successfully()
         self.mysql_starts_successfully()
         dbaas.create_engine = Mock()
 
-        self.mySqlApp.install_and_secure(100)
+        self.mySqlApp.secure(100)
 
-        self.assertTrue(self.mySqlApp._install_mysql.called)
         self.assertTrue(self.mySqlApp.stop_mysql.called)
         self.assertTrue(self.mySqlApp._write_mycnf.called)
         self.assertTrue(self.mySqlApp.start_mysql.called)
         self.assert_reported_status(ServiceStatuses.RUNNING)
 
-    def test_install_and_secure_install_error(self):
+    def test_install_install_error(self):
 
         from reddwarf.guestagent import pkg
         self.mySqlApp.start_mysql = Mock()
         self.mySqlApp.stop_mysql = Mock()
+        self.mySqlApp.is_installed = Mock(return_value=False)
         self.mySqlApp._install_mysql = \
             Mock(side_effect=pkg.PkgPackageStateError("Install error"))
 
         self.assertRaises(pkg.PkgPackageStateError,
-                          self.mySqlApp.install_and_secure, 100)
+                          self.mySqlApp.install_if_needed)
 
-        self.assert_reported_status(ServiceStatuses.BUILDING)
+        self.assert_reported_status(ServiceStatuses.NEW)
 
-    def test_install_and_secure_write_conf_error(self):
+    def test_secure_write_conf_error(self):
 
         from reddwarf.guestagent import pkg
         self.mySqlApp.start_mysql = Mock()
         self.mySqlApp.stop_mysql = Mock()
-        self.mySqlApp._install_mysql = Mock()
         self.mySqlApp._write_mycnf = \
             Mock(side_effect=pkg.PkgPackageStateError("Install error"))
         self.mysql_stops_successfully()
@@ -645,12 +651,11 @@ class MySqlAppInstallTest(MySqlAppTest):
         dbaas.create_engine = Mock()
 
         self.assertRaises(pkg.PkgPackageStateError,
-                          self.mySqlApp.install_and_secure, 100)
+                          self.mySqlApp.secure, 100)
 
-        self.assertTrue(self.mySqlApp._install_mysql.called)
         self.assertTrue(self.mySqlApp.stop_mysql.called)
         self.assertTrue(self.mySqlApp._write_mycnf.called)
-        self.assert_reported_status(ServiceStatuses.BUILDING)
+        self.assert_reported_status(ServiceStatuses.NEW)
 
     def test_is_installed(self):
 

reddwarf/tests/unittests/guestagent/test_manager.py

@@ -16,6 +16,8 @@ from reddwarf.guestagent.manager import Manager
 from reddwarf.guestagent import dbaas
 from reddwarf.guestagent import volume
 import testtools
+from reddwarf.instance import models as rd_models
+import os
 from mock import Mock, MagicMock
 
 
@@ -25,9 +27,27 @@ class GuestAgentManagerTest(testtools.TestCase):
         super(GuestAgentManagerTest, self).setUp()
         self.context = Mock()
         self.manager = Manager()
+        self.origin_MySqlAppStatus = dbaas.MySqlAppStatus
+        self.origin_os_path_exists = os.path.exists
+        self.origin_format = volume.VolumeDevice.format
+        self.origin_migrate_data = volume.VolumeDevice.migrate_data
+        self.origin_mount = volume.VolumeDevice.mount
+        self.origin_is_installed = dbaas.MySqlApp.is_installed
+        self.origin_stop_mysql = dbaas.MySqlApp.stop_mysql
+        self.origin_start_mysql = dbaas.MySqlApp.start_mysql
+        self.origin_install_mysql = dbaas.MySqlApp._install_mysql
 
     def tearDown(self):
         super(GuestAgentManagerTest, self).tearDown()
+        dbaas.MySqlAppStatus = self.origin_MySqlAppStatus
+        os.path.exists = self.origin_os_path_exists
+        volume.VolumeDevice.format = self.origin_format
+        volume.VolumeDevice.migrate_data = self.origin_migrate_data
+        volume.VolumeDevice.mount = self.origin_mount
+        dbaas.MySqlApp.is_installed = self.origin_is_installed
+        dbaas.MySqlApp.stop_mysql = self.origin_stop_mysql
+        dbaas.MySqlApp.start_mysql = self.origin_start_mysql
+        dbaas.MySqlApp._install_mysql = self.origin_install_mysql
 
     def test_update_status(self):
         dbaas.MySqlAppStatus.get = MagicMock()
@@ -107,17 +127,14 @@ class GuestAgentManagerTest(testtools.TestCase):
 
         self._setUp_MySqlAppStatus_get()
         dbaas.MySqlAppStatus.begin_mysql_install = MagicMock()
-        origin_format = volume.VolumeDevice.format
         volume.VolumeDevice.format = MagicMock()
-
+        volume.VolumeDevice.migrate_data = MagicMock()
-        origin_is_installed, origin_stop_mysql, origin_migrate_data =\
-            self._prepare_mysql_is_installed(is_mysql_installed)
-
-        origin_mount = volume.VolumeDevice.mount
         volume.VolumeDevice.mount = MagicMock()
-
+        dbaas.MySqlApp.stop_mysql = MagicMock()
         dbaas.MySqlApp.start_mysql = MagicMock()
-        dbaas.MySqlApp.install_and_secure = MagicMock()
+        dbaas.MySqlApp.install_if_needed = MagicMock()
+        dbaas.MySqlApp.secure = MagicMock()
+        self._prepare_mysql_is_installed(is_mysql_installed)
 
         Manager.create_database = MagicMock()
         Manager.create_user = MagicMock()
@@ -127,7 +144,8 @@ class GuestAgentManagerTest(testtools.TestCase):
                          dbaas.MySqlAppStatus.begin_mysql_install.call_count)
 
         self.assertEqual(COUNT, volume.VolumeDevice.format.call_count)
-        self.assertEqual(COUNT, dbaas.MySqlApp.is_installed.call_count)
+        # now called internally in install_if_needed() which is a mock
+        #self.assertEqual(1, dbaas.MySqlApp.is_installed.call_count)
 
         self.assertEqual(COUNT * SEC_COUNT,
                          dbaas.MySqlApp.stop_mysql.call_count)
@@ -138,24 +156,32 @@ class GuestAgentManagerTest(testtools.TestCase):
         self.assertEqual(COUNT * SEC_COUNT,
                          dbaas.MySqlApp.start_mysql.call_count)
 
-        self.assertEqual(1, dbaas.MySqlApp.install_and_secure.call_count)
+        self.assertEqual(1,
+                         dbaas.MySqlApp.install_if_needed.call_count)
+        self.assertEqual(1, dbaas.MySqlApp.secure.call_count)
         self.assertEqual(1, Manager.create_database.call_count)
         self.assertEqual(1, Manager.create_user.call_count)
 
-        volume.VolumeDevice.format = origin_format
-        volume.VolumeDevice.migrate_data = origin_migrate_data
-        dbaas.MySqlApp.is_installed = origin_is_installed
-        dbaas.MySqlApp.stop_mysql = origin_stop_mysql
-        volume.VolumeDevice.mount = origin_mount
-
     def _prepare_mysql_is_installed(self, is_installed=True):
-        origin_is_installed = dbaas.MySqlApp.is_installed
-        origin_stop_mysql = dbaas.MySqlApp.stop_mysql
-        origin_migrate_data = volume.VolumeDevice.migrate_data
         dbaas.MySqlApp.is_installed = MagicMock(return_value=is_installed)
-        dbaas.MySqlApp.stop_mysql = MagicMock()
+        os.path.exists = MagicMock()
-        volume.VolumeDevice.migrate_data = MagicMock()
+        dbaas.MySqlAppStatus._get_actual_db_status = MagicMock()
-        return origin_is_installed, origin_stop_mysql, origin_migrate_data
+
+        def path_exists_true(path):
+            if path == "/var/lib/mysql":
+                return True
+            else:
+                return False
+
+        def path_exists_false(path):
+            if path == "/var/lib/mysql":
+                return False
+            else:
+                return False
+        if is_installed:
+            os.path.exists.side_effect = path_exists_true
+        else:
+            os.path.exists.side_effect = path_exists_false
 
     def test_restart(self):
         self._setUp_MySqlAppStatus_get()
@@ -179,5 +205,5 @@ class GuestAgentManagerTest(testtools.TestCase):
         self.assertEqual(1, dbaas.MySqlApp.stop_mysql.call_count)
 
     def _setUp_MySqlAppStatus_get(self):
-        dbaas.MySqlAppStatus = Mock
+        dbaas.MySqlAppStatus = Mock()
         dbaas.MySqlAppStatus.get = MagicMock(return_value=dbaas.MySqlAppStatus)