fix int-tests running with out volume

fixed up the instance create and user/database level tests
fixed up the resize up test
fixed up and able to run through the entire suite of tests
fixed issues with the guest not writing the my.cnf file correctly

Fixes Bug #1095929

Change-Id: I2f971a072728380f83d82fdcb7595604a78a0511

reddwarf/guestagent/api.py

@@ -201,7 +201,7 @@ class API(proxy.RpcProxy):
     def get_volume_info(self):
         """Make a synchronous call to get volume info for the container"""
         LOG.debug(_("Check Volume Info on Instance %s"), self.id)
-        self._check_for_hearbeat()
+        # self._check_for_hearbeat()
         return self._call("get_filesystem_stats", AGENT_LOW_TIMEOUT,
                           fs_path="/var/lib/mysql")
 

reddwarf/guestagent/dbaas.py

@@ -82,7 +82,7 @@ def get_auth_password():
     pwd, err = utils.execute_with_timeout(
         "sudo",
         "awk",
-        "/password\\t=/{print $3}",
+        "/password\\t=/{print $3; exit}",
         "/etc/mysql/my.cnf")
     if err:
         LOG.err(err)
@@ -224,7 +224,7 @@ class MySqlAppStatus(object):
     @property
     def is_mysql_running(self):
         """True if MySQL is running."""
-        return (self.status is not None,
+        return (self.status is not None and
                 self.status == rd_models.ServiceStatuses.RUNNING)
 
     @staticmethod
@@ -624,6 +624,9 @@ class MySqlApp(object):
             self.status.end_install_or_restart()
 
     def _replace_mycnf_with_template(self, template_path, original_path):
+        LOG.debug("replacing the mycnf with template")
+        LOG.debug("template_path(%s) original_path(%s)"
+                  % (template_path, original_path))
         if os.path.isfile(template_path):
             utils.execute_with_timeout(
                 "sudo", "mv", original_path,
@@ -634,6 +637,7 @@ class MySqlApp(object):
 
     def _write_temp_mycnf_with_admin_account(self, original_file_path,
                                              temp_file_path, password):
+        utils.execute_with_timeout("sudo", "chmod", "0711", MYSQL_BASE_DIR)
         mycnf_file = open(original_file_path, 'r')
         tmp_file = open(temp_file_path, 'w')
         for line in mycnf_file:
@@ -729,7 +733,10 @@ class MySqlApp(object):
             raise RuntimeError("Could not start MySQL!")
 
     def start_mysql_with_conf_changes(self, updated_memory_mb):
-        LOG.info(_("Starting mysql with conf changes..."))
+        LOG.info(_("Starting mysql with conf changes to memory(%s)...")
+                 % updated_memory_mb)
+        LOG.info(_("inside the guest - self.status.is_mysql_running(%s)...")
+                 % self.status.is_mysql_running)
         if self.status.is_mysql_running:
             LOG.error(_("Cannot execute start_mysql_with_conf_changes because "
                         "MySQL state == %s!") % self.status)
@@ -742,3 +749,25 @@ class MySqlApp(object):
         #(cp16net) could raise an exception, does it need to be handled here?
         version = pkg.pkg_version(self.MYSQL_PACKAGE_VERSION)
         return not version is None
+
+
+class Interrogator(object):
+    def get_filesystem_volume_stats(self, fs_path):
+        out, err = utils.execute_with_timeout(
+            "stat",
+            "-f",
+            "-t",
+            fs_path)
+        if err:
+            LOG.err(err)
+            raise RuntimeError("Filesystem not found (%s) : %s"
+                               % (fs_path, err))
+        stats = out.split()
+        output = {}
+        output['block_size'] = int(stats[4])
+        output['total_blocks'] = int(stats[6])
+        output['free_blocks'] = int(stats[7])
+        output['total'] = int(stats[6]) * int(stats[4])
+        output['free'] = int(stats[7]) * int(stats[4])
+        output['used'] = int(output['total']) - int(output['free'])
+        return output

reddwarf/guestagent/manager.py

@@ -83,3 +83,7 @@ class Manager(periodic_task.PeriodicTasks):
     def stop_mysql(self, context):
         app = dbaas.MySqlApp(dbaas.MySqlAppStatus.get())
         app.stop_mysql()
+
+    def get_filesystem_stats(self, context, fs_path):
+        """ Gets the filesystem stats for the path given """
+        return dbaas.Interrogator().get_filesystem_volume_stats(fs_path)

reddwarf/instance/models.py

@@ -178,7 +178,7 @@ class SimpleInstance(object):
 
         ### Report as Shutdown while deleting, unless there's an error.
         if 'DELETING' == ACTION:
-            if self.db_info.server_status in ["ACTIVE", "SHUTDOWN"]:
+            if self.db_info.server_status in ["ACTIVE", "SHUTDOWN", "DELETED"]:
                 return InstanceStatus.SHUTDOWN
             else:
                 msg = _("While shutting down instance (%s): server had "

reddwarf/taskmanager/models.py

@@ -12,6 +12,7 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import time
 from eventlet import greenthread
 from datetime import datetime
 import traceback
@@ -353,7 +354,7 @@ class BuiltInstanceTasks(BuiltInstance):
             return "instance_id=%s, status=%s, flavor_id=%s, "\
                    "dest. flavor id=%s)" % (self.db_info.id,
                                             self.server.status,
-                                            str(self.flavor['id']),
+                                            str(self.server.flavor['id']),
                                             str(new_flavor_id))
 
         try:
@@ -363,12 +364,14 @@ class BuiltInstanceTasks(BuiltInstance):
                 LOG.debug("Instance %s calling Compute resize..."
                           % self.db_info.id)
                 if new_flavor_id:
+                    LOG.debug("Instance with new flavor id")
                     self.server.resize(new_flavor_id)
                 else:
                     LOG.debug("Migrating instance %s without flavor change ..."
                               % self.db_info.id)
                     self.server.migrate()
 
+                LOG.debug("refreshing the compute status of the instance")
                 # Do initial check and confirm the status is appropriate.
                 self._refresh_compute_server_info()
                 if (self.server.status != "RESIZE" and
@@ -376,31 +379,46 @@ class BuiltInstanceTasks(BuiltInstance):
                     msg = "Unexpected status after call to resize! : %s"
                     raise ReddwarfError(msg % resize_status_msg())
 
+                LOG.debug("the compute status of the instance : (%s)"
+                          % self.server.status)
+
                 # Wait for the flavor to change.
                 def update_server_info():
                     self._refresh_compute_server_info()
+                    LOG.debug("refreshed... compute status (%s)"
+                              % self.server.status)
                     return self.server.status != 'RESIZE'
 
+                LOG.debug("polling the server until its not RESIZE")
                 utils.poll_until(
                     update_server_info,
                     sleep_time=2,
-                    time_out=60 * 2)
+                    time_out=60 * 10)
+
+                LOG.debug("compute status should not be RESIZE now")
+                LOG.debug("instance_id=%s, status=%s, "
+                          "dest. flavor id=%s)" % (self.db_info.id,
+                                                   self.server.status,
+                                                   str(new_flavor_id)))
 
                 # Do check to make sure the status and flavor id are correct.
                 if new_flavor_id:
                     if str(self.server.flavor['id']) != str(new_flavor_id):
-                        msg = "Assertion failed! flavor_id=%s and not %s" \
-                              % (self.server.flavor['id'], new_flavor_id)
+                        msg = ("Assertion failed! flavor_id=%s and not %s"
+                               % (self.server.flavor['id'], new_flavor_id))
                         raise ReddwarfError(msg)
                 if (self.server.status != "VERIFY_RESIZE"):
-                    msg = "Assertion failed! status=%s and not %s" \
-                          % (self.server.status, 'VERIFY_RESIZE')
+                    msg = ("Assertion failed! status=%s and not %s"
+                           % (self.server.status, 'VERIFY_RESIZE'))
                     raise ReddwarfError(msg)
 
+                LOG.debug("wait a sec man!!!")
+                time.sleep(5)
                 # Confirm the resize with Nova.
                 LOG.debug("Instance %s calling Compute confirm resize..."
                           % self.db_info.id)
                 self.server.confirm_resize()
+                LOG.debug("Compute confirm resize DONE ...")
                 if new_flavor_id:
                     # Record the new flavor_id in our database.
                     LOG.debug("Updating instance %s to flavor_id %s."
@@ -409,8 +427,7 @@ class BuiltInstanceTasks(BuiltInstance):
             except Exception as ex:
                 new_memory_size = old_memory_size
                 new_flavor_id = None
-                LOG.error("Error resizing instance %s." % self.db_info.id)
-                LOG.error(ex)
+                LOG.exception("Error resizing instance %s." % self.db_info.id)
             finally:
                 # Tell the guest to restart MySQL with the new RAM size.
                 # This is in the finally because we have to call this, or

reddwarf/tests/api/databases.py

@@ -50,14 +50,14 @@ class TestMysqlAccess(object):
     @test
     def test_mysql_admin(self):
         """Ensure we aren't allowed access with os_admin and wrong password."""
-        assert_mysql_connection_fails("os_admin", "asdfd-asdf234",
-                                      instance_info.get_address())
+        util.mysql_connection().assert_fails(
+            "os_admin", "asdfd-asdf234", instance_info.get_address())
 
     @test
     def test_mysql_root(self):
         """Ensure we aren't allowed access with root and wrong password."""
-        assert_mysql_connection_fails("root", "dsfgnear",
-                                      instance_info.get_address())
+        util.mysql_connection().assert_fails(
+            "root", "dsfgnear", instance_info.get_address())
 
 
 @test(depends_on_groups=[GROUP_START],

reddwarf/tests/api/instances.py

@@ -97,9 +97,7 @@ class InstanceTestInfo(object):
 
     def get_address(self):
         result = self.dbaas_admin.mgmt.instances.show(self.id)
-        addresses = result.server['addresses']
-        address = addresses[test_config.visible_address_group][0]
-        return address['addr']
+        return result.ip[0]
 
     def get_local_id(self):
         mgmt_instance = self.dbaas_admin.management.show(self.id)
@@ -244,8 +242,6 @@ class CreateInstance(unittest.TestCase):
                           "way_too_large", instance_info.dbaas_flavor_href,
                           {'size': too_big + 1}, [])
             assert_equal(413, dbaas.last_http_code)
-        #else:
-        #    raise SkipTest("N/A: No max accepted volume size defined.")
 
     def test_create(self):
         databases = []
@@ -290,7 +286,6 @@ class CreateInstance(unittest.TestCase):
         else:
             report.log("Test was invoked with TESTS_USE_INSTANCE_ID=%s, so no "
                        "instance was actually created." % id)
-            report.log("Local id = %d" % instance_info.get_local_id())
 
         # Check these attrs only are returned in create response
         expected_attrs = ['created', 'flavor', 'addresses', 'id', 'links',
@@ -665,7 +660,7 @@ class TestInstanceListing(object):
         if create_new_instance():
             assert_true(0.12 < instance.volume['used'] < 0.25)
 
-    @test
+    @test(enabled=do_not_delete_instance())
     def test_instance_not_shown_to_other_user(self):
         daffy_ids = [instance.id for instance in
                      self.other_client.instances.list()]
@@ -679,7 +674,7 @@ class TestInstanceListing(object):
         for id in admin_ids:
             assert_equal(daffy_ids.count(id), 0)
 
-    @test
+    @test(enabled=do_not_delete_instance())
     def test_instance_not_deleted_by_other_user(self):
         assert_raises(exceptions.NotFound,
                       self.other_client.instances.get, instance_info.id)

reddwarf/tests/api/instances_actions.py

@@ -66,8 +66,9 @@ class MySqlConnection(object):
         """Connect to MySQL database."""
         print("Connecting to MySQL, mysql --host %s -u %s -p%s"
               % (self.host, MYSQL_USERNAME, MYSQL_PASSWORD))
-        self.client = LocalSqlClient(util.init_engine(
-            MYSQL_USERNAME, MYSQL_PASSWORD, self.host), use_flush=False)
+        sql_engine = LocalSqlClient.init_engine(MYSQL_USERNAME, MYSQL_PASSWORD,
+                                                self.host)
+        self.client = LocalSqlClient(sql_engine, use_flush=False)
 
     def is_connected(self):
         try:
@@ -108,8 +109,8 @@ class ActionTestBase(object):
         return self.dbaas.instances.get(self.instance_id)
 
     @property
-    def instance_local_id(self):
-        return instance_info.get_local_id()
+    def instance_address(self):
+        return instance_info.get_address()
 
     @property
     def instance_id(self):
@@ -146,7 +147,7 @@ class ActionTestBase(object):
             check.equal(instance.status, "ACTIVE")
 
     def find_mysql_proc_on_instance(self):
-        return util.find_mysql_procid_on_instance(self.instance_local_id)
+        return util.find_mysql_procid_on_instance(self.instance_address)
 
     def log_current_users(self):
         users = self.dbaas.users.list(self.instance_id)
@@ -217,7 +218,7 @@ class RebootTestBase(ActionTestBase):
         self.fix_mysql()  # kill files
         cmd = """ssh %s 'sudo cp /dev/null /var/lib/mysql/ib_logfile%d'"""
         for index in range(2):
-            full_cmd = cmd % (self.instance_local_id, index)
+            full_cmd = cmd % (self.instance_address, index)
             print("RUNNING COMMAND: %s" % full_cmd)
             util.process(full_cmd)
 
@@ -226,13 +227,13 @@ class RebootTestBase(ActionTestBase):
         if not FAKE_MODE:
             cmd = "ssh %s 'sudo rm /var/lib/mysql/ib_logfile%d'"
             for index in range(2):
-                util.process(cmd % (self.instance_local_id, index))
+                util.process(cmd % (self.instance_address, index))
 
     def wait_for_failure_status(self):
         """Wait until status becomes running."""
         def is_finished_rebooting():
             instance = self.instance
-            if instance.status == "REBOOT":
+            if instance.status == "REBOOT" or instance.status == "ACTIVE":
                 return False
             assert_equal("SHUTDOWN", instance.status)
             return True

reddwarf/tests/api/users.py

@@ -158,8 +158,8 @@ class TestUsers(object):
     def show_databases(self, user, password):
         print("Going to connect to %s, %s, %s"
               % (instance_info.get_address(), user, password))
-        with create_mysql_connection(instance_info.get_address(),
-                                     user, password) as db:
+        with util.mysql_connection().create(instance_info.get_address(),
+                                            user, password) as db:
             print(db)
             dbs = db.execute("show databases")
             return [row['Database'] for row in dbs]
@@ -264,8 +264,8 @@ class TestUsers(object):
 
     def _check_connection(self, username, password):
         if not FAKE:
-            util.assert_mysql_connection_fails(username, password,
-                                               instance_info.get_address())
+            util.mysql_connection().assert_fails(username, password,
+                                                 instance_info.get_address())
         # Also determine the db is gone via API.
         result = self.dbaas.users.list(instance_info.id)
         assert_equal(200, self.dbaas.last_http_code)

reddwarf/tests/util/__init__.py

@@ -59,6 +59,7 @@ from reddwarf.tests.util import test_config as CONFIG
 from reddwarf.tests.util.client import TestClient as TestClient
 from reddwarf.tests.util.users import Requirements
 from reddwarf.common.exception import PollTimeOut
+from reddwarf.common.utils import import_object
 
 
 WHITE_BOX = test_config.white_box
@@ -221,6 +222,45 @@ else:
     from reddwarf.common.utils import poll_until
 
 
+def mysql_connection():
+    cls = CONFIG.get('mysql_connection',
+                     "local.MySqlConnection")
+    if cls == "local.MySqlConnection":
+        return MySqlConnection()
+    return import_object(cls)()
+
+
+def find_mysql_procid_on_instance(ip_address):
+    """Returns the process id of MySql on an instance if running, or None."""
+    cmd = "ssh %s ps aux | grep /usr/sbin/mysqld " \
+          "| awk '{print $2}'" % ip_address
+    stdout, stderr = process(cmd)
+    try:
+        return int(stdout)
+    except ValueError:
+        return None
+
+
+class MySqlConnection(object):
+
+    def assert_fails(self, user_name, password, ip):
+        from reddwarf.tests.util import mysql
+        try:
+            with mysql.create_mysql_connection(ip, user_name, password) as db:
+                pass
+            fail("Should have failed to connect: mysql --host %s -u %s -p%s"
+                 % (ip, user_name, password))
+        except mysql.MySqlPermissionsFailure:
+            return  # Good, this is what we wanted.
+        except mysql.MySqlConnectionFailure as mcf:
+            fail("Expected to see permissions failure. Instead got message:"
+                 "%s" % mcf.message)
+
+    def create(self, ip, user_name, password):
+        from reddwarf.tests.util import mysql
+        return mysql.create_mysql_connection(ip, user_name, password)
+
+
 class LocalSqlClient(object):
     """A sqlalchemy wrapper to manage transactions"""
 
@@ -250,3 +290,11 @@ class LocalSqlClient(object):
             self.trans.rollback()
             self.trans = None
             raise
+
+    @staticmethod
+    def init_engine(user, password, host):
+        return create_engine("mysql://%s:%s@%s:3306" %
+                             (user, password, host),
+                             pool_recycle=1800, echo=True)
+        self.engine = engine
+        self.use_flush = use_flush

reddwarf/tests/util/mysql.py

@@ -0,0 +1,159 @@
+import pexpect
+import re
+from sqlalchemy import create_engine
+from reddwarf.tests.config import CONFIG
+from sqlalchemy.exc import OperationalError
+try:
+    from sqlalchemy.exc import ResourceClosedError
+except ImportError:
+    ResourceClosedError = Exception
+
+
+def create_mysql_connection(host, user, password):
+    connection = CONFIG.mysql_connection_method
+    if connection['type'] == "direct":
+        return SqlAlchemyConnection(host, user, password)
+    elif connection['type'] == "tunnel":
+        if 'ssh' not in connection:
+            raise RuntimeError("If connection type is 'tunnel' then a "
+                               "property 'ssh' is expected.")
+        return PexpectMySqlConnection(connection['ssh'], host, user, password)
+    else:
+        raise RuntimeError("Unknown Bad test configuration for "
+                           "mysql_connection_method")
+
+
+class MySqlConnectionFailure(RuntimeError):
+
+    def __init__(self, msg):
+        super(MySqlConnectionFailure, self).__init__(msg)
+
+
+class MySqlPermissionsFailure(RuntimeError):
+
+    def __init__(self, msg):
+        super(MySqlPermissionsFailure, self).__init__(msg)
+
+
+class SqlAlchemyConnection(object):
+
+    def __init__(self, host, user, password):
+        self.host = host
+        self.user = user
+        self.password = password
+        try:
+            self.engine = self._init_engine(user, password, host)
+        except OperationalError as oe:
+            if self._exception_is_permissions_issue(oe.message):
+                raise MySqlPermissionsFailure(oe)
+            else:
+                raise MySqlConnectionFailure(oe)
+
+    @staticmethod
+    def _exception_is_permissions_issue(msg):
+        """Assert message cited a permissions issue and not something else."""
+        pos_error = re.compile(".*Host '[\w\.]*' is not allowed to connect to "
+                               "this MySQL server.*")
+        pos_error1 = re.compile(".*Access denied for user "
+                                "'[\w\*\!\@\#\^\&]*'@'[\w\.]*'.*")
+        if (pos_error.match(msg) or pos_error1.match(msg)):
+            return True
+
+    def __enter__(self):
+        try:
+            self.conn = self.engine.connect()
+        except OperationalError as oe:
+            if self._exception_is_permissions_issue(oe.message):
+                raise MySqlPermissionsFailure(oe)
+            else:
+                raise MySqlConnectionFailure(oe)
+        self.trans = self.conn.begin()
+        return self
+
+    def execute(self, cmd):
+        """Execute some code."""
+        cmd = cmd.replace("%", "%%")
+        try:
+            return self.conn.execute(cmd).fetchall()
+        except:
+            self.trans.rollback()
+            self.trans = None
+            try:
+                raise
+            except ResourceClosedError:
+                return []
+
+    def __exit__(self, type, value, traceback):
+        if self.trans:
+            if type is not None:  # An error occurred
+                self.trans.rollback()
+            else:
+                self.trans.commit()
+        self.conn.close()
+
+    @staticmethod
+    def _init_engine(user, password, host):
+        return create_engine("mysql://%s:%s@%s:3306" % (user, password, host),
+                             pool_recycle=1800, echo=True)
+
+
+class PexpectMySqlConnection(object):
+
+    TIME_OUT = 30
+
+    def __init__(self, ssh_args, host, user, password):
+        self.host = host
+        self.user = user
+        self.password = password
+        cmd = 'ssh %s' % ssh_args
+        self.proc = pexpect.spawn(cmd)
+        print(cmd)
+        self.proc.expect(":~\$", timeout=self.TIME_OUT)
+        cmd2 = "mysql --host '%s' -u '%s' '-p%s'\n" % \
+               (self.host, self.user, self.password)
+        print(cmd2)
+        self.proc.send(cmd2)
+        result = self.proc.expect([
+            'mysql>',
+            'Access denied',
+            "Can't connect to MySQL server"],
+            timeout=self.TIME_OUT)
+        if result == 1:
+            raise MySqlPermissionsFailure(self.proc.before)
+        elif result == 2:
+            raise MySqlConnectionFailure(self.proc.before)
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, type, value, traceback):
+        self.proc.close()
+
+    def execute(self, cmd):
+        self.proc.send(cmd + "\G\n")
+        outcome = self.proc.expect(['Empty set', 'mysql>'],
+                                   timeout=self.TIME_OUT)
+        if outcome == 0:
+            return []
+        else:
+            # This next line might be invaluable for long test runs.
+            print("Interpreting output: %s" % self.proc.before)
+            lines = self.proc.before.split("\r\n")
+            result = []
+            row = None
+            for line in lines:
+                plural_s = "s" if len(result) != 0 else ""
+                end_line = "%d row%s in set" % ((len(result) + 1), plural_s)
+                if len(result) == 0:
+                    end_line = "1 row in set"
+                if (line.startswith("***************************") or
+                    line.startswith(end_line)):
+                    if row is not None:
+                        result.append(row)
+                    row = {}
+                elif row is not None:
+                    colon = line.find(": ")
+                    field = line[:colon]
+                    value = line[colon + 2:]
+                    row[field] = value
+            return result