From b938999f7dd4678adef9efad88d48e82f6721c43 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 26 May 2016 17:16:28 +0200 Subject: [PATCH] Port more guestagent unit tests to Python 3 * mysql: open temporary file in text mode, not in binary mode * operating_system: open files in text mode, not in binary mode, and flush explicitly the file rather than using unbuffered file. On Python 3, it's no more possible to open a text file in unbuffered mode (it would be very inefficient). * CouchbaseRootAccess.write_password_to_file(): encode password to UTF-8 if it's an unicode string. * Replace ConfigParser import with six.moves.configparser * Replace exc.message with exception_to_unicode(exc) * tox: run the following guestagent unit tests on Python 3.4 - test_backups.py - test_configuration.py - test_couchbase_manager.py - test_couchdb_manager.py - test_db2_manager.py - test_dbmodels.py - test_galera_manager.py - test_manager.py - test_mariadb_manager.py - test_mongodb_cluster_manager.py - test_mysql_manager.py - test_redis_manager.py - test_vertica_manager.py Partially implements: blueprint trove-python3 Change-Id: I7bf477b328dfe15c5339cda1b80e44358019bfff --- tox.ini | 13 +++++++++++++ trove/guestagent/common/operating_system.py | 8 +++++--- .../datastore/experimental/couchbase/service.py | 3 +++ .../datastore/experimental/vertica/service.py | 6 +++--- trove/guestagent/datastore/manager.py | 3 ++- trove/guestagent/strategies/restore/mysql_impl.py | 2 +- trove/tests/unittests/guestagent/test_manager.py | 4 +++- 7 files changed, 30 insertions(+), 9 deletions(-) diff --git a/tox.ini b/tox.ini index e882658e8e..5562d2ef08 100644 --- a/tox.ini +++ b/tox.ini @@ -69,13 +69,26 @@ commands = trove/tests/unittests/dns/test_designate_driver.py \ trove/tests/unittests/guestagent/test_agent_heartbeats_models.py \ trove/tests/unittests/guestagent/test_api.py \ + trove/tests/unittests/guestagent/test_backups.py \ + trove/tests/unittests/guestagent/test_configuration.py \ + trove/tests/unittests/guestagent/test_couchbase_manager.py \ + trove/tests/unittests/guestagent/test_couchdb_manager.py \ + trove/tests/unittests/guestagent/test_db2_manager.py \ + trove/tests/unittests/guestagent/test_dbmodels.py \ trove/tests/unittests/guestagent/test_galera_cluster_api.py \ + trove/tests/unittests/guestagent/test_galera_manager.py \ trove/tests/unittests/guestagent/test_guestagent_utils.py \ + trove/tests/unittests/guestagent/test_manager.py \ + trove/tests/unittests/guestagent/test_mariadb_manager.py \ trove/tests/unittests/guestagent/test_models.py \ + trove/tests/unittests/guestagent/test_mongodb_cluster_manager.py \ + trove/tests/unittests/guestagent/test_mysql_manager.py \ trove/tests/unittests/guestagent/test_pkg.py \ trove/tests/unittests/guestagent/test_query.py \ + trove/tests/unittests/guestagent/test_redis_manager.py \ trove/tests/unittests/guestagent/test_service.py \ trove/tests/unittests/guestagent/test_vertica_api.py \ + trove/tests/unittests/guestagent/test_vertica_manager.py \ trove/tests/unittests/instance/test_instance_controller.py \ trove/tests/unittests/instance/test_instance_models.py \ trove/tests/unittests/instance/test_instance_status.py \ diff --git a/trove/guestagent/common/operating_system.py b/trove/guestagent/common/operating_system.py index 3191eab4b9..2a9f2aadd9 100644 --- a/trove/guestagent/common/operating_system.py +++ b/trove/guestagent/common/operating_system.py @@ -59,7 +59,7 @@ def read_file(path, codec=IdentityCodec(), as_root=False, decode=True): if as_root: return _read_file_as_root(path, codec, decode=decode) - with open(path, 'rb') as fp: + with open(path, 'r') as fp: if decode: return codec.deserialize(fp.read()) return codec.serialize(fp.read()) @@ -144,11 +144,12 @@ def write_file(path, data, codec=IdentityCodec(), as_root=False, encode=True): if as_root: _write_file_as_root(path, data, codec, encode=encode) else: - with open(path, 'wb', 0) as fp: + with open(path, 'w') as fp: if encode: fp.write(codec.serialize(data)) else: fp.write(codec.deserialize(data)) + fp.flush() else: raise exception.UnprocessableEntity(_("Invalid path: %s") % path) @@ -170,11 +171,12 @@ def _write_file_as_root(path, data, codec, encode=True): """ # The files gets removed automatically once the managing object goes # out of scope. - with tempfile.NamedTemporaryFile('wb', 0, delete=False) as fp: + with tempfile.NamedTemporaryFile('w', delete=False) as fp: if encode: fp.write(codec.serialize(data)) else: fp.write(codec.deserialize(data)) + fp.flush() fp.close() # Release the resource before proceeding. copy(fp.name, path, force=True, as_root=True) diff --git a/trove/guestagent/datastore/experimental/couchbase/service.py b/trove/guestagent/datastore/experimental/couchbase/service.py index 3c21cbc17e..516d24b1cd 100644 --- a/trove/guestagent/datastore/experimental/couchbase/service.py +++ b/trove/guestagent/datastore/experimental/couchbase/service.py @@ -22,6 +22,7 @@ import tempfile from oslo_log import log as logging from oslo_utils import netutils import pexpect +import six from trove.common import cfg from trove.common import exception @@ -246,6 +247,8 @@ class CouchbaseRootAccess(object): try: tempfd, tempname = tempfile.mkstemp() os.fchmod(tempfd, stat.S_IRUSR | stat.S_IWUSR) + if isinstance(root_password, six.text_type): + root_password = root_password.encode('utf-8') os.write(tempfd, root_password) os.fchmod(tempfd, stat.S_IRUSR) os.close(tempfd) diff --git a/trove/guestagent/datastore/experimental/vertica/service.py b/trove/guestagent/datastore/experimental/vertica/service.py index c626703e23..dea3a43120 100644 --- a/trove/guestagent/datastore/experimental/vertica/service.py +++ b/trove/guestagent/datastore/experimental/vertica/service.py @@ -11,13 +11,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -import ConfigParser import os import subprocess import tempfile from oslo_log import log as logging from oslo_utils import netutils +from six.moves import configparser from trove.common import cfg from trove.common import exception @@ -361,7 +361,7 @@ class VerticaApp(object): def _generate_database_password(self): """Generate and write the password to vertica.cnf file.""" - config = ConfigParser.ConfigParser() + config = configparser.ConfigParser() config.add_section('credentials') config.set('credentials', 'dbadmin_password', utils.generate_random_password()) @@ -388,7 +388,7 @@ class VerticaApp(object): def read_config(self): """Reads and returns the Vertica config.""" try: - config = ConfigParser.ConfigParser() + config = configparser.ConfigParser() config.read(system.VERTICA_CONF) return config except Exception: diff --git a/trove/guestagent/datastore/manager.py b/trove/guestagent/datastore/manager.py index 6f6618a6df..36273212d0 100644 --- a/trove/guestagent/datastore/manager.py +++ b/trove/guestagent/datastore/manager.py @@ -19,6 +19,7 @@ import abc from oslo_config import cfg as oslo_cfg from oslo_log import log as logging from oslo_service import periodic_task +from oslo_utils import encodeutils from trove.common import cfg from trove.common import exception @@ -285,7 +286,7 @@ class Manager(periodic_task.PeriodicTasks): except Exception as ex: self.prepare_error = True LOG.exception(_("An error occurred preparing datastore: %s") % - ex.message) + encodeutils.exception_to_unicode(ex)) raise finally: LOG.info(_("Ending datastore prepare for '%s'.") % self.manager) diff --git a/trove/guestagent/strategies/restore/mysql_impl.py b/trove/guestagent/strategies/restore/mysql_impl.py index 61c8018bc7..e21e04b385 100644 --- a/trove/guestagent/strategies/restore/mysql_impl.py +++ b/trove/guestagent/strategies/restore/mysql_impl.py @@ -132,7 +132,7 @@ class MySQLRestoreMixin(object): for initial datastore configuration. """ - with tempfile.NamedTemporaryFile() as init_file: + with tempfile.NamedTemporaryFile(mode='w') as init_file: operating_system.chmod(init_file.name, FileMode.ADD_READ_ALL, as_root=True) self._writelines_one_per_line(init_file, diff --git a/trove/tests/unittests/guestagent/test_manager.py b/trove/tests/unittests/guestagent/test_manager.py index 4e7f4072db..ec3a0ed0d7 100644 --- a/trove/tests/unittests/guestagent/test_manager.py +++ b/trove/tests/unittests/guestagent/test_manager.py @@ -22,6 +22,7 @@ from mock import DEFAULT from mock import MagicMock from mock import Mock from mock import patch +from oslo_utils import encodeutils from proboscis.asserts import assert_equal from proboscis.asserts import assert_true @@ -462,8 +463,9 @@ class ManagerTest(trove_testtools.TestCase): apply_overrides_on_prepare=MagicMock( side_effect=expected_failure )): + expected_msg = encodeutils.exception_to_unicode(expected_failure) self.assertRaisesRegexp( - Exception, expected_failure.message, + Exception, expected_msg, self.manager.prepare, self.context, packages, databases, memory_mb, users, device_path, mount_point, backup_info, config_contents,