From d9b9ac893dd0e55676c6c2455236005dc352f2e0 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 12 Jun 2015 13:28:38 +0200 Subject: [PATCH] Fix Python 3 issues in utils * Replace string.split(key_file, ',') with key_file.split(','): the string.split() function was removed in Python 3, use the str.split() method instead * Fix conf_fixture: import backup_enable_progress_timer and backup_swift_enable_progress_timer options. This change is required to run tests using testtools (it's now possible yet to run tests on Python 3 using testr). * hash_file(): use b'' (bytes) for the sentinel, not '' (unicode), when reading the input binary file * test_hash_file() requires a binary file (bytes), not a text file (unicode) * Get the builtin open() function from six.moves.builtins instead of getting it from __builtin__ to mock the open() function. The __builtin__ module was renamed to builtins in Python 3. * sanitize_hostname(): on Python 3, decode back the hostname from Latin1 to work on a native string (Unicode) * replace reduce() with six.moves.reduce(): the Python 2 builtin reduce() function was moved to the functools module in Python 3 * convert_version_to_str(): fix integer division, use a//b, not a/b * convert_version_to_str(): replace reduce() with ''.join() with map(str) Blueprint cinder-python3 Change-Id: If7b8f50c6a8b0a5044c2c7108b2b0293dddafff3 --- cinder/ssh_utils.py | 3 +-- cinder/tests/unit/conf_fixture.py | 4 ++++ cinder/tests/unit/test_utils.py | 22 +++++++++++----------- cinder/utils.py | 14 +++++++++----- 4 files changed, 25 insertions(+), 18 deletions(-) diff --git a/cinder/ssh_utils.py b/cinder/ssh_utils.py index e6b736374..22f29a52d 100644 --- a/cinder/ssh_utils.py +++ b/cinder/ssh_utils.py @@ -19,7 +19,6 @@ """Utilities related to SSH connection management.""" import os -import string from eventlet import pools from oslo_config import cfg @@ -103,7 +102,7 @@ class SSHPool(pools.Pool): try: ssh = paramiko.SSHClient() if ',' in self.hosts_key_file: - files = string.split(self.hosts_key_file, ',') + files = self.hosts_key_file.split(',') for f in files: ssh.load_host_keys(f) else: diff --git a/cinder/tests/unit/conf_fixture.py b/cinder/tests/unit/conf_fixture.py index 2898eb1cc..2685b15f7 100644 --- a/cinder/tests/unit/conf_fixture.py +++ b/cinder/tests/unit/conf_fixture.py @@ -21,6 +21,10 @@ from oslo_config import cfg CONF = cfg.CONF +CONF.import_opt('backup_enable_progress_timer', + 'cinder.backup.drivers.nfs') +CONF.import_opt('backup_swift_enable_progress_timer', + 'cinder.backup.drivers.swift') CONF.import_opt('policy_file', 'cinder.policy') CONF.import_opt('volume_driver', 'cinder.volume.manager') CONF.import_opt('xiv_ds8k_proxy', diff --git a/cinder/tests/unit/test_utils.py b/cinder/tests/unit/test_utils.py index 4631a723b..e117bbe90 100644 --- a/cinder/tests/unit/test_utils.py +++ b/cinder/tests/unit/test_utils.py @@ -416,8 +416,8 @@ class GenericUtilsTestCase(test.TestCase): self.assertEqual(''foo'', utils.xhtml_escape("'foo'")) def test_hash_file(self): - data = 'Mary had a little lamb, its fleece as white as snow' - flo = six.StringIO(data) + data = b'Mary had a little lamb, its fleece as white as snow' + flo = six.BytesIO(data) h1 = utils.hash_file(flo) h2 = hashlib.sha1(data).hexdigest() self.assertEqual(h1, h2) @@ -1029,7 +1029,7 @@ class FakeTransport(object): class SSHPoolTestCase(test.TestCase): """Unit test for SSH Connection Pool.""" @mock.patch('cinder.ssh_utils.CONF') - @mock.patch('__builtin__.open') + @mock.patch('six.moves.builtins.open') @mock.patch('paramiko.SSHClient') @mock.patch('os.path.isfile', return_value=True) def test_ssh_default_hosts_key_file(self, mock_isfile, mock_sshclient, @@ -1053,7 +1053,7 @@ class SSHPoolTestCase(test.TestCase): '/var/lib/cinder/ssh_known_hosts') @mock.patch('cinder.ssh_utils.CONF') - @mock.patch('__builtin__.open') + @mock.patch('six.moves.builtins.open') @mock.patch('paramiko.SSHClient') @mock.patch('os.path.isfile', return_value=True) def test_ssh_host_key_file_kwargs(self, mock_isfile, mock_sshclient, @@ -1082,7 +1082,7 @@ class SSHPoolTestCase(test.TestCase): mock_ssh.assert_has_calls(expected, any_order=True) @mock.patch('cinder.ssh_utils.CONF') - @mock.patch('__builtin__.open') + @mock.patch('six.moves.builtins.open') @mock.patch('os.path.isfile', return_value=True) @mock.patch('paramiko.RSAKey.from_private_key_file') @mock.patch('paramiko.SSHClient') @@ -1121,7 +1121,7 @@ class SSHPoolTestCase(test.TestCase): min_size=1, max_size=1) - @mock.patch('__builtin__.open') + @mock.patch('six.moves.builtins.open') @mock.patch('paramiko.SSHClient') def test_closed_reopened_ssh_connections(self, mock_sshclient, mock_open): mock_sshclient.return_value = eval('FakeSSHClient')() @@ -1149,7 +1149,7 @@ class SSHPoolTestCase(test.TestCase): self.assertNotEqual(first_id, third_id) @mock.patch('cinder.ssh_utils.CONF') - @mock.patch('__builtin__.open') + @mock.patch('six.moves.builtins.open') @mock.patch('paramiko.SSHClient') def test_missing_ssh_hosts_key_config(self, mock_sshclient, mock_open, mock_conf): @@ -1165,7 +1165,7 @@ class SSHPoolTestCase(test.TestCase): min_size=1, max_size=1) - @mock.patch('__builtin__.open') + @mock.patch('six.moves.builtins.open') @mock.patch('paramiko.SSHClient') def test_create_default_known_hosts_file(self, mock_sshclient, mock_open): @@ -1187,7 +1187,7 @@ class SSHPoolTestCase(test.TestCase): ssh_pool.remove(ssh) @mock.patch('os.path.isfile', return_value=False) - @mock.patch('__builtin__.open') + @mock.patch('six.moves.builtins.open') @mock.patch('paramiko.SSHClient') def test_ssh_missing_hosts_key_file(self, mock_sshclient, mock_open, mock_isfile): @@ -1207,7 +1207,7 @@ class SSHPoolTestCase(test.TestCase): @mock.patch.multiple('cinder.ssh_utils.CONF', strict_ssh_host_key_policy=True, ssh_hosts_key_file='/var/lib/cinder/ssh_known_hosts') - @mock.patch('__builtin__.open') + @mock.patch('six.moves.builtins.open') @mock.patch('paramiko.SSHClient') @mock.patch('os.path.isfile', return_value=True) def test_ssh_strict_host_key_policy(self, mock_isfile, mock_sshclient, @@ -1225,7 +1225,7 @@ class SSHPoolTestCase(test.TestCase): self.assertTrue(isinstance(ssh.get_policy(), paramiko.RejectPolicy)) - @mock.patch('__builtin__.open') + @mock.patch('six.moves.builtins.open') @mock.patch('paramiko.SSHClient') @mock.patch('os.path.isfile', return_value=True) def test_ssh_not_strict_host_key_policy(self, mock_isfile, mock_sshclient, diff --git a/cinder/utils.py b/cinder/utils.py index 1b2baa59d..ce1757ec5 100644 --- a/cinder/utils.py +++ b/cinder/utils.py @@ -459,8 +459,12 @@ def make_dev_path(dev, partition=None, base='/dev'): def sanitize_hostname(hostname): """Return a hostname which conforms to RFC-952 and RFC-1123 specs.""" - if isinstance(hostname, six.text_type): + if six.PY3: hostname = hostname.encode('latin-1', 'ignore') + hostname = hostname.decode('latin-1') + else: + if isinstance(hostname, six.text_type): + hostname = hostname.encode('latin-1', 'ignore') hostname = re.sub('[ _]', '-', hostname) hostname = re.sub('[^\w.-]+', '', hostname) @@ -473,7 +477,7 @@ def sanitize_hostname(hostname): def hash_file(file_like_object): """Generate a hash for the contents of a file.""" checksum = hashlib.sha1() - any(map(checksum.update, iter(lambda: file_like_object.read(32768), ''))) + any(map(checksum.update, iter(lambda: file_like_object.read(32768), b''))) return checksum.hexdigest() @@ -793,7 +797,7 @@ def convert_version_to_int(version): if isinstance(version, six.string_types): version = convert_version_to_tuple(version) if isinstance(version, tuple): - return reduce(lambda x, y: (x * 1000) + y, version) + return six.moves.reduce(lambda x, y: (x * 1000) + y, version) except Exception: msg = _("Version %s is invalid.") % version raise exception.CinderException(msg) @@ -805,9 +809,9 @@ def convert_version_to_str(version_int): while version_int != 0: version_number = version_int - (version_int // factor * factor) version_numbers.insert(0, six.text_type(version_number)) - version_int = version_int / factor + version_int = version_int // factor - return reduce(lambda x, y: "%s.%s" % (x, y), version_numbers) + return '.'.join(map(str, version_numbers)) def convert_version_to_tuple(version_str):