From 02e435f0ed1aede7a2da68cf99d315e9551b642d Mon Sep 17 00:00:00 2001 From: caihui Date: Wed, 22 Jan 2020 03:52:35 -0800 Subject: [PATCH] Remove six usage from freezer package We don't support Python 2 anymore so we don't need this compatibility library. This patch remove six usage. Change-Id: I661e9cdaf33e89bdb905d1f34a001f3853c4c366 --- freezer/engine/engine.py | 11 +++-------- freezer/engine/rsync/pyrsync.py | 11 ----------- freezer/engine/rsync/rsync.py | 6 +++--- freezer/engine/rsyncv2/rsyncv2.py | 14 +++++++------- freezer/job.py | 4 +--- freezer/lib/pep3143daemon/daemon.py | 3 +-- freezer/mode/mode.py | 4 +--- freezer/scheduler/scheduler_job.py | 2 +- freezer/storage/base.py | 4 +--- freezer/storage/fslike.py | 4 +--- freezer/storage/multiple.py | 2 +- freezer/storage/physical.py | 4 +--- freezer/tests/integration/common.py | 1 - .../tests/unit/engines/rsync/test_pyrsync.py | 18 +++++++++--------- freezer/tests/unit/utils/test_checksum.py | 4 ++-- freezer/tests/unit/utils/test_config.py | 5 ++--- freezer/utils/checksum.py | 11 +++-------- freezer/utils/config.py | 8 ++------ freezer/utils/streaming.py | 2 +- freezer/utils/utils.py | 2 +- lower-constraints.txt | 1 - requirements.txt | 1 - 22 files changed, 41 insertions(+), 81 deletions(-) diff --git a/freezer/engine/engine.py b/freezer/engine/engine.py index b9d90881..e45b61ce 100644 --- a/freezer/engine/engine.py +++ b/freezer/engine/engine.py @@ -21,25 +21,20 @@ import time from oslo_log import log from oslo_serialization import jsonutils as json -import six # PyCharm will not recognize queue. Puts red squiggle line under it. That's OK. -from six.moves import queue +import queue from freezer.exceptions import engine as engine_exceptions from freezer.storage import base from freezer.utils import streaming from freezer.utils import utils -if six.PY3: - from multiprocessing import SimpleQueue -else: - from multiprocessing.queues import SimpleQueue +from multiprocessing import SimpleQueue LOG = log.getLogger(__name__) -@six.add_metaclass(abc.ABCMeta) -class BackupEngine(object): +class BackupEngine(metaclass=abc.ABCMeta): """ The main part of making a backup and making a restore is the mechanism of implementing it. For a long time Freezer had only one mechanism of diff --git a/freezer/engine/rsync/pyrsync.py b/freezer/engine/rsync/pyrsync.py index 0fbf0c23..f54ab450 100644 --- a/freezer/engine/rsync/pyrsync.py +++ b/freezer/engine/rsync/pyrsync.py @@ -27,17 +27,6 @@ https://samba.anu.edu.au/rsync/. import collections import hashlib -import six -from six.moves import range - - -if six.PY2: - # Python 2.x compatibility - def bytes(var, *args): - try: - return ''.join(map(chr, var)) - except TypeError: - return map(ord, var) __all__ = ["rollingchecksum", "weakchecksum", "rsyncdelta", "blockchecksums"] diff --git a/freezer/engine/rsync/rsync.py b/freezer/engine/rsync/rsync.py index f1a073f5..9d66b971 100644 --- a/freezer/engine/rsync/rsync.py +++ b/freezer/engine/rsync/rsync.py @@ -19,15 +19,15 @@ import getpass import grp import os import pwd +import queue import re import stat import sys import threading +from io import StringIO from oslo_log import log from oslo_serialization import jsonutils as json -from six.moves import cStringIO -from six.moves import queue from freezer.engine import engine from freezer.engine.rsync import pyrsync @@ -256,7 +256,7 @@ class RsyncEngine(engine.BackupEngine): len_deltas = 0 old_signature = old_file_meta['signature'] # Get changed blocks index only - all_changed_indexes = cStringIO() + all_changed_indexes = StringIO() file_path_fd.seek(0) previous_index = -1 modified_blocks = [] diff --git a/freezer/engine/rsyncv2/rsyncv2.py b/freezer/engine/rsyncv2/rsyncv2.py index a743e603..989523f9 100644 --- a/freezer/engine/rsyncv2/rsyncv2.py +++ b/freezer/engine/rsyncv2/rsyncv2.py @@ -18,8 +18,10 @@ Freezer rsync incremental engine import fnmatch import getpass import grp +import io import os import pwd +import queue import shutil import stat import sys @@ -27,9 +29,7 @@ import threading import msgpack from oslo_log import log -import six -from six.moves import queue from freezer.engine import engine from freezer.engine.rsyncv2 import pyrsync @@ -222,7 +222,7 @@ class Rsyncv2Engine(engine.BackupEngine): files_meta = msgpack.load(data_stream) except msgpack.ExtraData as e: files_meta = e.unpacked - data_stream = six.BytesIO(e.extra) + data_stream = io.BytesIO(e.extra) break except msgpack.OutOfData: data_stream.write(data_gen.next().read()) @@ -349,14 +349,14 @@ class Rsyncv2Engine(engine.BackupEngine): data_chunk += read_pipe.recv_bytes() continue if data_chunk: - yield six.BytesIO(data_chunk) + yield io.BytesIO(data_chunk) data_chunk = read_pipe.recv_bytes() except EOFError: LOG.info("[*] EOF from pipe. Flushing buffer.") data_chunk = decompressor.flush() if data_chunk: - yield six.BytesIO(data_chunk) + yield io.BytesIO(data_chunk) @staticmethod def _process_backup_data(data, compressor, encryptor, do_compress=True): @@ -574,7 +574,7 @@ class Rsyncv2Engine(engine.BackupEngine): """ file_name = os.path.basename(file_path) - for fn in six.iterkeys(old_files): + for fn in iter(old_files.keys()): base_name = os.path.basename(fn) if fnmatch.fnmatch(base_name, '*' + file_name + '*'): return base_name @@ -704,7 +704,7 @@ class Rsyncv2Engine(engine.BackupEngine): counts['total_files'] += 1 # Check for deleted files - for del_file in (f for f in six.iterkeys(old_fs_meta_struct) if + for del_file in (f for f in iter(old_fs_meta_struct.keys()) if f not in files_meta['files']): backup_header.append({'path': del_file, 'deleted': True}) diff --git a/freezer/job.py b/freezer/job.py index 04331431..37c6b586 100644 --- a/freezer/job.py +++ b/freezer/job.py @@ -24,7 +24,6 @@ import time from oslo_config import cfg from oslo_log import log from oslo_utils import importutils -import six from freezer.common import client_manager from freezer.openstack import admin @@ -39,8 +38,7 @@ CONF = cfg.CONF LOG = log.getLogger(__name__) -@six.add_metaclass(abc.ABCMeta) -class Job(object): +class Job(metaclass=abc.ABCMeta): """ :type storage: freezer.storage.base.Storage :type engine: freezer.engine.engine.BackupEngine diff --git a/freezer/lib/pep3143daemon/daemon.py b/freezer/lib/pep3143daemon/daemon.py index 3cfd6d6f..fa0728a0 100644 --- a/freezer/lib/pep3143daemon/daemon.py +++ b/freezer/lib/pep3143daemon/daemon.py @@ -32,7 +32,6 @@ import signal import socket import sys -import six class DaemonError(Exception): @@ -171,7 +170,7 @@ class DaemonContext(object): """ if not handler: result = signal.SIG_IGN - elif isinstance(handler, six.string_types): + elif isinstance(handler, str): result = getattr(self, handler) else: result = handler diff --git a/freezer/mode/mode.py b/freezer/mode/mode.py index b9e6def5..a6681b40 100644 --- a/freezer/mode/mode.py +++ b/freezer/mode/mode.py @@ -13,11 +13,9 @@ # limitations under the License. import abc -import six -@six.add_metaclass(abc.ABCMeta) -class Mode(object): +class Mode(metaclass=abc.ABCMeta): @abc.abstractproperty def name(self): pass diff --git a/freezer/scheduler/scheduler_job.py b/freezer/scheduler/scheduler_job.py index d3bb43d8..1ddf0524 100644 --- a/freezer/scheduler/scheduler_job.py +++ b/freezer/scheduler/scheduler_job.py @@ -13,6 +13,7 @@ # limitations under the License. +import configparser import datetime import os import subprocess @@ -23,7 +24,6 @@ from freezer.utils import utils from oslo_config import cfg from oslo_log import log from oslo_serialization import jsonutils as json -from six.moves import configparser CONF = cfg.CONF diff --git a/freezer/storage/base.py b/freezer/storage/base.py index 95bd8fde..aae9f9a9 100644 --- a/freezer/storage/base.py +++ b/freezer/storage/base.py @@ -19,15 +19,13 @@ import tempfile from oslo_log import log from oslo_serialization import jsonutils as json -import six from freezer.utils import utils LOG = log.getLogger(__name__) -@six.add_metaclass(abc.ABCMeta) -class Storage(object): +class Storage(metaclass=abc.ABCMeta): """ Any freezer storage implementation should be inherited from this abstract class. diff --git a/freezer/storage/fslike.py b/freezer/storage/fslike.py index 32f98902..814a3026 100644 --- a/freezer/storage/fslike.py +++ b/freezer/storage/fslike.py @@ -13,15 +13,13 @@ # limitations under the License. import abc -import six from oslo_serialization import jsonutils as json from freezer.storage import physical -@six.add_metaclass(abc.ABCMeta) -class FsLikeStorage(physical.PhysicalStorage): +class FsLikeStorage(physical.PhysicalStorage, metaclass=abc.ABCMeta): _type = 'fslike' def __init__(self, storage_path, diff --git a/freezer/storage/multiple.py b/freezer/storage/multiple.py index 69fa845e..103e1b89 100644 --- a/freezer/storage/multiple.py +++ b/freezer/storage/multiple.py @@ -14,7 +14,7 @@ from oslo_log import log # PyCharm will not recognize queue. Puts red squiggle line under it. That's OK. -from six.moves import queue +import queue from freezer.storage import base from freezer.storage import exceptions diff --git a/freezer/storage/physical.py b/freezer/storage/physical.py index e5e14a7a..ee295ad9 100644 --- a/freezer/storage/physical.py +++ b/freezer/storage/physical.py @@ -16,14 +16,12 @@ import abc import os -import six from freezer.storage import base from freezer.utils import utils -@six.add_metaclass(abc.ABCMeta) -class PhysicalStorage(base.Storage): +class PhysicalStorage(base.Storage, metaclass=abc.ABCMeta): """ Backup like Swift, SSH or Local. Something that represents real storage. For example MultipleStorage is not physical. diff --git a/freezer/tests/integration/common.py b/freezer/tests/integration/common.py index 649da51b..e7478b5c 100644 --- a/freezer/tests/integration/common.py +++ b/freezer/tests/integration/common.py @@ -27,7 +27,6 @@ from oslo_serialization import jsonutils as json import paramiko -from six.moves import range FREEZERC = distutils.spawn.find_executable('freezer-agent') diff --git a/freezer/tests/unit/engines/rsync/test_pyrsync.py b/freezer/tests/unit/engines/rsync/test_pyrsync.py index b8bbb8ed..beae2450 100644 --- a/freezer/tests/unit/engines/rsync/test_pyrsync.py +++ b/freezer/tests/unit/engines/rsync/test_pyrsync.py @@ -14,17 +14,17 @@ import unittest -import six +import io from freezer.engine.rsync import pyrsync class TestPyrsync(unittest.TestCase): def test_blockcheksum(self): - instream = six.BytesIO(b'aae9dd83aa45f906' - b'a4629f42e97eac99' - b'b9882284dc7030ca' - b'427ad365fedd2a55') + instream = io.BytesIO(b'aae9dd83aa45f906' + b'a4629f42e97eac99' + b'b9882284dc7030ca' + b'427ad365fedd2a55') weak, strong = pyrsync.blockchecksums(instream, 16) exp_weak = [736756931, 616825970, 577963056, 633341072] exp_strong = ['0f923c37c14f648de4065d4666c2429231a923bc', @@ -34,10 +34,10 @@ class TestPyrsync(unittest.TestCase): self.assertEqual((weak, strong), (exp_weak, exp_strong)) def test_rsyncdelta(self): - datastream = six.BytesIO(b'addc830058f917ae' - b'a1be5ab4d899b570' - b'85c9534c64d8d71c' - b'1f32cde9c71e5b6d') + datastream = io.BytesIO(b'addc830058f917ae' + b'a1be5ab4d899b570' + b'85c9534c64d8d71c' + b'1f32cde9c71e5b6d') old_weak = [675087508, 698025105, 579470394, 667092162] old_strong = ['e72251cb70a1b918ee43876896ebb4c8a7225f78', diff --git a/freezer/tests/unit/utils/test_checksum.py b/freezer/tests/unit/utils/test_checksum.py index d1842904..ecee5e94 100644 --- a/freezer/tests/unit/utils/test_checksum.py +++ b/freezer/tests/unit/utils/test_checksum.py @@ -12,12 +12,12 @@ # License for the specific language governing permissions and limitations # under the License. +import io import sys import unittest import mock from mock import patch -from six import moves from freezer.utils.checksum import CheckSum @@ -31,7 +31,7 @@ class TestChecksum(unittest.TestCase): self.hello_world_sha256sum = ('17b949eb67acf16bbf2605d57a01f7af4ff4b5' '7e200259de63fcebf20e75bbf5') - self.fake_file = moves.StringIO(u"hello world\n") + self.fake_file = io.StringIO(u"hello world\n") self.increment_hash_one = self.hello_world_sha256sum self.increment_hash_multi = ('1b4bc4ff41172a5f29eaeffb7e9fc24c683c693' '9ab30132ad5d93a1e4a6b16e8') diff --git a/freezer/tests/unit/utils/test_config.py b/freezer/tests/unit/utils/test_config.py index 0beae32c..9a3a5970 100644 --- a/freezer/tests/unit/utils/test_config.py +++ b/freezer/tests/unit/utils/test_config.py @@ -12,10 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +import io import unittest -from six import moves - from freezer.utils import config @@ -47,7 +46,7 @@ password = 'aNiceQuotedPassword' password2 = "aNiceQuotedPassword" spaced = value""" - fd = moves.cStringIO(string) + fd = io.StringIO(string) res = config.ini_parse(fd) self.assertEqual('127.0.0.1', res['host']) self.assertEqual('openstack', res['user']) diff --git a/freezer/utils/checksum.py b/freezer/utils/checksum.py index 563309c9..aa1129b1 100644 --- a/freezer/utils/checksum.py +++ b/freezer/utils/checksum.py @@ -15,8 +15,7 @@ import hashlib import os -import six -from six import moves +import io from freezer.utils import utils @@ -119,11 +118,7 @@ class CheckSum(object): # Need to use string-escape for Python 2 non-unicode strings. For # Python 2 unicode strings and all Python 3 strings, we need to use # unicode-escape. The effect of them is the same. - if six.PY2 and isinstance(buf, str): - buf = buf.encode('string-escape') - elif six.PY2 and not isinstance(buf, str): - buf = buf.encode('unicode-escape') - elif six.PY3 and isinstance(buf, six.string_types): + if isinstance(buf, str): buf = buf.encode('unicode-escape') self.hasher.update(buf) @@ -134,7 +129,7 @@ class CheckSum(object): """ :return: the hash for a given string """ - fd = moves.StringIO(string) + fd = io.StringIO(string) return self.hashfile(fd) def compute(self): diff --git a/freezer/utils/config.py b/freezer/utils/config.py index 06dcd4cd..70550baf 100644 --- a/freezer/utils/config.py +++ b/freezer/utils/config.py @@ -12,12 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +import configparser import os import re from oslo_log import log -import six -from six.moves import configparser from freezer.utils import utils @@ -41,10 +40,7 @@ class Config(object): raise Exception("Configuration file {0} not found !".format( config_path)) # SafeConfigParser was deprecated in Python 3.2 - if six.PY3: - config = configparser.ConfigParser() - else: - config = configparser.SafeConfigParser() + config = configparser.ConfigParser() config.read([config_path]) sections = config.sections() storages = [] diff --git a/freezer/utils/streaming.py b/freezer/utils/streaming.py index 87dd5e51..395c3a93 100644 --- a/freezer/utils/streaming.py +++ b/freezer/utils/streaming.py @@ -20,7 +20,7 @@ Freezer general utils functions import threading from oslo_log import log -from six.moves import queue +import queue LOG = log.getLogger(__name__) diff --git a/freezer/utils/utils.py b/freezer/utils/utils.py index a22baabf..29b3f05e 100644 --- a/freezer/utils/utils.py +++ b/freezer/utils/utils.py @@ -18,6 +18,7 @@ Freezer general utils functions """ +import configparser import datetime import errno import fnmatch as fn @@ -31,7 +32,6 @@ from distutils import spawn as distspawn from freezer.exceptions import utils from functools import wraps from oslo_log import log -from six.moves import configparser logging.getLogger('botocore').setLevel(logging.WARNING) diff --git a/lower-constraints.txt b/lower-constraints.txt index e20d27aa..a4aa5930 100644 --- a/lower-constraints.txt +++ b/lower-constraints.txt @@ -107,7 +107,6 @@ rfc3986==1.1.0 Routes==2.4.1 setuptools==21.0.0 simplejson==3.13.2 -six==1.10.0 snowballstemmer==1.2.1 Sphinx==1.6.2 sphinxcontrib-websupport==1.0.1 diff --git a/requirements.txt b/requirements.txt index cc249946..e55e4cb8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -23,7 +23,6 @@ cryptography>=2.1 # Apache-2.0 PyMySQL>=0.7.6 # MIT License pymongo!=3.1,>=3.0.2 # Apache-2.0 paramiko>=2.0.0 # LGPLv2.1+ -six>=1.10.0 # MIT apscheduler>=3.0.5 # MIT License psutil>=3.2.2 # BSD