From 0a0da897a34635f719f7343b4ec1ac04cae9db91 Mon Sep 17 00:00:00 2001 From: Mark Biciunas Date: Tue, 13 Jan 2015 14:33:35 -0500 Subject: [PATCH] MySQL restore wait for shutdown before killing Problem: MySQL restory strategy in function '_spawn_with_init_file' of '/trove/trove/guestagent/strategies/restore/mysql_impl.py' issues a shutdown command to the mysql process immediatelly followed by 'killall'. The kill call is however necessary only if the soft shutdown fails. The current code potentially executes 'kill' unnecessarily after a successful shutdown. This implementation also opens up a timing dependent defect in case of non-blocking shutdown when it may lead to database corruption by killing the process prematurely in the middle of shutdown procedure. Solution: Poll database status after executing shutdown and follow by 'killall' only if soft shutdown fails. Change-Id: Iaba7baa2c4357cc6f2d28d8178e2946721e068e6 Closes-bug: bug/1414186 --- .../strategies/restore/mysql_impl.py | 22 ++++++++++++++----- 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/trove/guestagent/strategies/restore/mysql_impl.py b/trove/guestagent/strategies/restore/mysql_impl.py index d67581cbc7..9d6cdffc72 100644 --- a/trove/guestagent/strategies/restore/mysql_impl.py +++ b/trove/guestagent/strategies/restore/mysql_impl.py @@ -82,12 +82,22 @@ class MySQLRestoreMixin(object): LOG.debug("Cleaning up the temp mysqld process.") utils.execute_with_timeout("mysqladmin", "-uroot", "--protocol=tcp", "shutdown") - utils.execute_with_timeout("killall", "mysqld_safe", - root_helper="sudo", run_as_root=True) - self.poll_until_then_raise( - self.mysql_is_not_running, - base.RestoreError("Reset root password failed: " - "mysqld did not stop!")) + LOG.debug("Polling for shutdown to complete.") + try: + utils.poll_until(self.mysql_is_not_running, + sleep_time=self.RESET_ROOT_SLEEP_INTERVAL, + time_out=self.RESET_ROOT_RETRY_TIMEOUT) + LOG.debug("Database successfully shutdown") + except exception.PollTimeOut: + LOG.debug("Timeout shutting down database " + "- performing killall on mysqld_safe.") + utils.execute_with_timeout("killall", "mysqld_safe", + root_helper="sudo", + run_as_root=True) + self.poll_until_then_raise( + self.mysql_is_not_running, + base.RestoreError("Reset root password failed: " + "mysqld did not stop!")) def reset_root_password(self): #Create temp file with reset root password