diff --git a/glanceclient/common/http.py b/glanceclient/common/http.py index 67fde15..f163dac 100644 --- a/glanceclient/common/http.py +++ b/glanceclient/common/http.py @@ -36,11 +36,12 @@ if not hasattr(parse, 'parse_qsl'): import cgi parse.parse_qsl = cgi.parse_qsl +from oslo.utils import encodeutils + from glanceclient.common import https from glanceclient.common.utils import safe_header from glanceclient import exc from glanceclient.openstack.common import importutils -from glanceclient.openstack.common import strutils osprofiler_web = importutils.try_import("osprofiler.web") @@ -117,7 +118,7 @@ class HTTPClient(object): curl.append(url) - msg = ' '.join([strutils.safe_encode(item, errors='ignore') + msg = ' '.join([encodeutils.safe_decode(item, errors='ignore') for item in curl]) LOG.debug(msg) @@ -129,9 +130,9 @@ class HTTPClient(object): dump.extend(['%s: %s' % safe_header(k, v) for k, v in headers]) dump.append('') if body: - body = strutils.safe_decode(body) + body = encodeutils.safe_decode(body) dump.extend([body, '']) - LOG.debug('\n'.join([strutils.safe_encode(x, errors='ignore') + LOG.debug('\n'.join([encodeutils.safe_decode(x, errors='ignore') for x in dump])) @staticmethod @@ -145,7 +146,7 @@ class HTTPClient(object): :returns: Dictionary with encoded headers' names and values """ - return dict((strutils.safe_encode(h), strutils.safe_encode(v)) + return dict((encodeutils.safe_encode(h), encodeutils.safe_encode(v)) for h, v in six.iteritems(headers)) def _request(self, method, url, **kwargs): diff --git a/glanceclient/common/https.py b/glanceclient/common/https.py index 6baa6af..186b3ac 100644 --- a/glanceclient/common/https.py +++ b/glanceclient/common/https.py @@ -14,6 +14,7 @@ # under the License. import socket +import ssl import struct import OpenSSL @@ -25,8 +26,8 @@ except ImportError: from urllib3 import connectionpool from urllib3 import poolmanager +from oslo.utils import encodeutils import six -import ssl from glanceclient.common import utils @@ -50,7 +51,6 @@ except ImportError: from glanceclient import exc -from glanceclient.openstack.common import strutils def to_bytes(s): @@ -81,7 +81,7 @@ class HTTPSAdapter(adapters.HTTPAdapter): # NOTE(flaper87): Make sure the url is encoded, otherwise # python's standard httplib will fail with a TypeError. url = super(HTTPSAdapter, self).request_url(request, proxies) - return strutils.safe_encode(url) + return encodeutils.safe_encode(url) def cert_verify(self, conn, url, verify, cert): super(HTTPSAdapter, self).cert_verify(conn, url, verify, cert) diff --git a/glanceclient/common/utils.py b/glanceclient/common/utils.py index 61c526e..d99d4ae 100644 --- a/glanceclient/common/utils.py +++ b/glanceclient/common/utils.py @@ -31,11 +31,12 @@ if os.name == 'nt': else: msvcrt = None +from oslo.utils import encodeutils +from oslo.utils import strutils import prettytable from glanceclient import exc from glanceclient.openstack.common import importutils -from glanceclient.openstack.common import strutils _memoized_property_lock = threading.Lock() @@ -150,7 +151,7 @@ def print_list(objs, fields, formatters=None, field_settings=None): row.append(data) pt.add_row(row) - print(strutils.safe_encode(pt.get_string())) + print(encodeutils.safe_decode(pt.get_string())) def print_dict(d, max_column_width=80): @@ -161,7 +162,7 @@ def print_dict(d, max_column_width=80): if isinstance(v, (dict, list)): v = json.dumps(v) pt.add_row([k, v]) - print(strutils.safe_encode(pt.get_string(sortby='Property'))) + print(encodeutils.safe_decode(pt.get_string(sortby='Property'))) def find_resource(manager, name_or_id): @@ -175,7 +176,9 @@ def find_resource(manager, name_or_id): # now try to get entity as uuid try: - uuid.UUID(strutils.safe_encode(name_or_id)) + # This must be unicode for Python 3 compatibility. + # If you pass a bytestring to uuid.UUID, you will get a TypeError + uuid.UUID(encodeutils.safe_decode(name_or_id)) return manager.get(name_or_id) except (ValueError, exc.NotFound): pass @@ -233,7 +236,7 @@ def import_versioned_module(version, submodule=None): def exit(msg=''): if msg: - print(strutils.safe_encode(msg), file=sys.stderr) + print(encodeutils.safe_decode(msg), file=sys.stderr) sys.exit(1) @@ -291,7 +294,7 @@ def exception_to_str(exc): except UnicodeError: error = ("Caught '%(exception)s' exception." % {"exception": exc.__class__.__name__}) - return strutils.safe_encode(error, errors='ignore') + return encodeutils.safe_decode(error, errors='ignore') def get_file_size(file_obj): diff --git a/glanceclient/openstack/common/strutils.py b/glanceclient/openstack/common/strutils.py deleted file mode 100644 index b8c44a8..0000000 --- a/glanceclient/openstack/common/strutils.py +++ /dev/null @@ -1,245 +0,0 @@ -# Copyright 2011 OpenStack Foundation. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -System-level utilities and helper functions. -""" - -import math -import re -import sys -import unicodedata - -import six - -from glanceclient.openstack.common.gettextutils import _ - - -UNIT_PREFIX_EXPONENT = { - 'k': 1, - 'K': 1, - 'Ki': 1, - 'M': 2, - 'Mi': 2, - 'G': 3, - 'Gi': 3, - 'T': 4, - 'Ti': 4, -} -UNIT_SYSTEM_INFO = { - 'IEC': (1024, re.compile(r'(^[-+]?\d*\.?\d+)([KMGT]i?)?(b|bit|B)$')), - 'SI': (1000, re.compile(r'(^[-+]?\d*\.?\d+)([kMGT])?(b|bit|B)$')), -} - -TRUE_STRINGS = ('1', 't', 'true', 'on', 'y', 'yes') -FALSE_STRINGS = ('0', 'f', 'false', 'off', 'n', 'no') - -SLUGIFY_STRIP_RE = re.compile(r"[^\w\s-]") -SLUGIFY_HYPHENATE_RE = re.compile(r"[-\s]+") - - -def int_from_bool_as_string(subject): - """Interpret a string as a boolean and return either 1 or 0. - - Any string value in: - - ('True', 'true', 'On', 'on', '1') - - is interpreted as a boolean True. - - Useful for JSON-decoded stuff and config file parsing - """ - return bool_from_string(subject) and 1 or 0 - - -def bool_from_string(subject, strict=False, default=False): - """Interpret a string as a boolean. - - A case-insensitive match is performed such that strings matching 't', - 'true', 'on', 'y', 'yes', or '1' are considered True and, when - `strict=False`, anything else returns the value specified by 'default'. - - Useful for JSON-decoded stuff and config file parsing. - - If `strict=True`, unrecognized values, including None, will raise a - ValueError which is useful when parsing values passed in from an API call. - Strings yielding False are 'f', 'false', 'off', 'n', 'no', or '0'. - """ - if not isinstance(subject, six.string_types): - subject = str(subject) - - lowered = subject.strip().lower() - - if lowered in TRUE_STRINGS: - return True - elif lowered in FALSE_STRINGS: - return False - elif strict: - acceptable = ', '.join( - "'%s'" % s for s in sorted(TRUE_STRINGS + FALSE_STRINGS)) - msg = _("Unrecognized value '%(val)s', acceptable values are:" - " %(acceptable)s") % {'val': subject, - 'acceptable': acceptable} - raise ValueError(msg) - else: - return default - - -def safe_decode(text, incoming=None, errors='strict'): - """Decodes incoming text/bytes string using `incoming` if they're not - already unicode. - - :param incoming: Text's current encoding - :param errors: Errors handling policy. See here for valid - values http://docs.python.org/2/library/codecs.html - :returns: text or a unicode `incoming` encoded - representation of it. - :raises TypeError: If text is not an instance of str - """ - if not isinstance(text, (six.string_types, six.binary_type)): - raise TypeError("%s can't be decoded" % type(text)) - - if isinstance(text, six.text_type): - return text - - if not incoming: - incoming = (sys.stdin.encoding or - sys.getdefaultencoding()) - - try: - return text.decode(incoming, errors) - except UnicodeDecodeError: - # Note(flaper87) If we get here, it means that - # sys.stdin.encoding / sys.getdefaultencoding - # didn't return a suitable encoding to decode - # text. This happens mostly when global LANG - # var is not set correctly and there's no - # default encoding. In this case, most likely - # python will use ASCII or ANSI encoders as - # default encodings but they won't be capable - # of decoding non-ASCII characters. - # - # Also, UTF-8 is being used since it's an ASCII - # extension. - return text.decode('utf-8', errors) - - -def safe_encode(text, incoming=None, - encoding='utf-8', errors='strict'): - """Encodes incoming text/bytes string using `encoding`. - - If incoming is not specified, text is expected to be encoded with - current python's default encoding. (`sys.getdefaultencoding`) - - :param incoming: Text's current encoding - :param encoding: Expected encoding for text (Default UTF-8) - :param errors: Errors handling policy. See here for valid - values http://docs.python.org/2/library/codecs.html - :returns: text or a bytestring `encoding` encoded - representation of it. - :raises TypeError: If text is not an instance of str - """ - if not isinstance(text, (six.string_types, six.binary_type)): - raise TypeError("%s can't be encoded" % type(text)) - - if not incoming: - incoming = (sys.stdin.encoding or - sys.getdefaultencoding()) - - if isinstance(text, six.text_type): - if six.PY3: - return text.encode(encoding, errors).decode(incoming) - else: - return text.encode(encoding, errors) - elif text and encoding != incoming: - # Decode text before encoding it with `encoding` - text = safe_decode(text, incoming, errors) - if six.PY3: - return text.encode(encoding, errors).decode(incoming) - else: - return text.encode(encoding, errors) - - return text - - -def string_to_bytes(text, unit_system='IEC', return_int=False): - """Converts a string into an float representation of bytes. - - The units supported for IEC :: - - Kb(it), Kib(it), Mb(it), Mib(it), Gb(it), Gib(it), Tb(it), Tib(it) - KB, KiB, MB, MiB, GB, GiB, TB, TiB - - The units supported for SI :: - - kb(it), Mb(it), Gb(it), Tb(it) - kB, MB, GB, TB - - Note that the SI unit system does not support capital letter 'K' - - :param text: String input for bytes size conversion. - :param unit_system: Unit system for byte size conversion. - :param return_int: If True, returns integer representation of text - in bytes. (default: decimal) - :returns: Numerical representation of text in bytes. - :raises ValueError: If text has an invalid value. - - """ - try: - base, reg_ex = UNIT_SYSTEM_INFO[unit_system] - except KeyError: - msg = _('Invalid unit system: "%s"') % unit_system - raise ValueError(msg) - match = reg_ex.match(text) - if match: - magnitude = float(match.group(1)) - unit_prefix = match.group(2) - if match.group(3) in ['b', 'bit']: - magnitude /= 8 - else: - msg = _('Invalid string format: %s') % text - raise ValueError(msg) - if not unit_prefix: - res = magnitude - else: - res = magnitude * pow(base, UNIT_PREFIX_EXPONENT[unit_prefix]) - if return_int: - return int(math.ceil(res)) - return res - - -def to_slug(value, incoming=None, errors="strict"): - """Normalize string. - - Convert to lowercase, remove non-word characters, and convert spaces - to hyphens. - - Inspired by Django's `slugify` filter. - - :param value: Text to slugify - :param incoming: Text's current encoding - :param errors: Errors handling policy. See here for valid - values http://docs.python.org/2/library/codecs.html - :returns: slugified unicode representation of `value` - :raises TypeError: If text is not an instance of str - """ - value = safe_decode(value, incoming, errors) - # NOTE(aababilov): no need to use safe_(encode|decode) here: - # encodings are always "ascii", error handling is always "ignore" - # and types are always known (first: unicode; second: str) - value = unicodedata.normalize("NFKD", value).encode( - "ascii", "ignore").decode("ascii") - value = SLUGIFY_STRIP_RE.sub("", value).strip().lower() - return SLUGIFY_HYPHENATE_RE.sub("-", value) diff --git a/glanceclient/shell.py b/glanceclient/shell.py index 4eb174e..bca2450 100644 --- a/glanceclient/shell.py +++ b/glanceclient/shell.py @@ -29,6 +29,7 @@ from os.path import expanduser import sys import traceback +from oslo.utils import encodeutils import six.moves.urllib.parse as urlparse import glanceclient @@ -36,7 +37,6 @@ from glanceclient.common import utils from glanceclient import exc from glanceclient.openstack.common.gettextutils import _ from glanceclient.openstack.common import importutils -from glanceclient.openstack.common import strutils from keystoneclient.auth.identity import v2 as v2_auth from keystoneclient.auth.identity import v3 as v3_auth @@ -695,7 +695,7 @@ class HelpFormatter(argparse.HelpFormatter): def main(): try: - OpenStackImagesShell().main(map(strutils.safe_decode, sys.argv[1:])) + OpenStackImagesShell().main(map(encodeutils.safe_decode, sys.argv[1:])) except KeyboardInterrupt: print('... terminating glance client', file=sys.stderr) sys.exit(1) diff --git a/glanceclient/v1/images.py b/glanceclient/v1/images.py index e3a6e69..857b546 100644 --- a/glanceclient/v1/images.py +++ b/glanceclient/v1/images.py @@ -15,12 +15,13 @@ import copy +from oslo.utils import encodeutils +from oslo.utils import strutils import six import six.moves.urllib.parse as urlparse from glanceclient.common import utils from glanceclient.openstack.common.apiclient import base -from glanceclient.openstack.common import strutils UPDATE_PARAMS = ('name', 'disk_format', 'container_format', 'min_disk', 'min_ram', 'owner', 'size', 'is_public', 'protected', @@ -70,7 +71,7 @@ class ImageManager(base.ManagerWithFind): def _image_meta_from_headers(self, headers): meta = {'properties': {}} - safe_decode = strutils.safe_decode + safe_decode = encodeutils.safe_decode for key, value in six.iteritems(headers): value = safe_decode(value, incoming='utf-8') if key.startswith('x-image-meta-property-'): @@ -191,7 +192,7 @@ class ImageManager(base.ManagerWithFind): # # Making sure all params are str before # trying to encode them - qp[param] = strutils.safe_encode(value) + qp[param] = encodeutils.safe_decode(value) url = '/v1/images/detail?%s' % urlparse.urlencode(qp) images, resp = self._list(url, "images") diff --git a/glanceclient/v1/shell.py b/glanceclient/v1/shell.py index 5d5452e..7cd8081 100644 --- a/glanceclient/v1/shell.py +++ b/glanceclient/v1/shell.py @@ -20,10 +20,12 @@ import functools import six import sys +from oslo.utils import encodeutils +from oslo.utils import strutils + from glanceclient.common import progressbar from glanceclient.common import utils from glanceclient import exc -from glanceclient.openstack.common import strutils import glanceclient.v1.images CONTAINER_FORMATS = 'Acceptable formats: ami, ari, aki, bare, and ovf.' @@ -327,7 +329,7 @@ def do_image_delete(gc, args): try: if args.verbose: print('Requesting image delete for %s ...' % - strutils.safe_encode(args_image), end=' ') + encodeutils.safe_decode(args_image), end=' ') gc.images.delete(image) diff --git a/glanceclient/v2/images.py b/glanceclient/v2/images.py index b49c5ee..75115a4 100644 --- a/glanceclient/v2/images.py +++ b/glanceclient/v2/images.py @@ -14,13 +14,14 @@ # under the License. import json + +from oslo.utils import encodeutils import six from six.moves.urllib import parse import warlock from glanceclient.common import utils from glanceclient import exc -from glanceclient.openstack.common import strutils from glanceclient.v2 import schemas DEFAULT_PAGE_SIZE = 20 @@ -83,11 +84,11 @@ class Controller(object): for tag in tags: if isinstance(tag, six.string_types): - tags_url_params.append({'tag': strutils.safe_encode(tag)}) + tags_url_params.append({'tag': encodeutils.safe_encode(tag)}) for param, value in six.iteritems(filters): if isinstance(value, six.string_types): - filters[param] = strutils.safe_encode(value) + filters[param] = encodeutils.safe_encode(value) url = '/v2/images?%s' % parse.urlencode(filters) diff --git a/glanceclient/v2/metadefs.py b/glanceclient/v2/metadefs.py index 8d9512b..56468fa 100644 --- a/glanceclient/v2/metadefs.py +++ b/glanceclient/v2/metadefs.py @@ -13,12 +13,12 @@ # License for the specific language governing permissions and limitations # under the License. +from oslo.utils import encodeutils import six from six.moves.urllib import parse import warlock from glanceclient.common import utils -from glanceclient.openstack.common import strutils from glanceclient.v2 import schemas DEFAULT_PAGE_SIZE = 20 @@ -161,9 +161,9 @@ class NamespaceController(object): for param, value in six.iteritems(filters): if isinstance(value, list): - filters[param] = strutils.safe_encode(','.join(value)) + filters[param] = encodeutils.safe_encode(','.join(value)) elif isinstance(value, six.string_types): - filters[param] = strutils.safe_encode(value) + filters[param] = encodeutils.safe_encode(value) url = '/v2/metadefs/namespaces?%s' % parse.urlencode(filters) diff --git a/glanceclient/v2/tasks.py b/glanceclient/v2/tasks.py index 730e130..51217ea 100644 --- a/glanceclient/v2/tasks.py +++ b/glanceclient/v2/tasks.py @@ -14,12 +14,11 @@ # License for the specific language governing permissions and limitations # under the License. +from oslo.utils import encodeutils import six - import warlock from glanceclient.common import utils -from glanceclient.openstack.common import strutils from glanceclient.v2 import schemas DEFAULT_PAGE_SIZE = 20 @@ -84,7 +83,7 @@ class Controller(object): for param, value in filters.items(): if isinstance(value, six.string_types): - filters[param] = strutils.safe_encode(value) + filters[param] = encodeutils.safe_encode(value) url = '/v2/tasks?%s' % six.moves.urllib.parse.urlencode(filters) for task in paginate(url): diff --git a/openstack-common.conf b/openstack-common.conf index b8c1da2..c77808b 100644 --- a/openstack-common.conf +++ b/openstack-common.conf @@ -4,7 +4,7 @@ module=apiclient module=gettextutils module=importutils -module=strutils +module=uuidutils # The base module to hold the copy of openstack.common base=glanceclient diff --git a/tests/test_http.py b/tests/test_http.py index 985284e..7356ca4 100644 --- a/tests/test_http.py +++ b/tests/test_http.py @@ -97,11 +97,11 @@ class TestClient(testtools.TestCase): # when creating the http client, the session headers don't contain # the X-Auth-Token key. identity_headers = { - 'X-User-Id': 'user', - 'X-Tenant-Id': 'tenant', - 'X-Roles': 'roles', - 'X-Identity-Status': 'Confirmed', - 'X-Service-Catalog': 'service_catalog', + b'X-User-Id': b'user', + b'X-Tenant-Id': b'tenant', + b'X-Roles': b'roles', + b'X-Identity-Status': b'Confirmed', + b'X-Service-Catalog': b'service_catalog', } kwargs = {'identity_headers': identity_headers} http_client = http.HTTPClient(self.endpoint, **kwargs) @@ -165,10 +165,7 @@ class TestClient(testtools.TestCase): value = u'ni\xf1o' headers = {"test": value} encoded = self.client.encode_headers(headers) - if six.PY2: - self.assertEqual("ni\xc3\xb1o", encoded["test"]) - else: - self.assertEqual(value, encoded["test"]) + self.assertEqual(b"ni\xc3\xb1o", encoded[b"test"]) def test_raw_request(self): " Verify the path being used for HTTP requests reflects accurately. " diff --git a/tests/v2/test_images.py b/tests/v2/test_images.py index ed6e75f..9cb6cec 100644 --- a/tests/v2/test_images.py +++ b/tests/v2/test_images.py @@ -15,7 +15,6 @@ import errno -import six import testtools from glanceclient import exc @@ -488,10 +487,7 @@ class TestController(testtools.TestCase): # /v2/images?owner=ni%C3%B1o&limit=20 # We just want to make sure filters are correctly encoded. pass - if six.PY2: - self.assertEqual("ni\xc3\xb1o", filters["owner"]) - else: - self.assertEqual("ni\xf1o", filters["owner"]) + self.assertEqual(b"ni\xc3\xb1o", filters["owner"]) def test_list_images_for_tag_single_image(self): img_id = '3a4560a1-e585-443e-9b39-553b46ec92d1' diff --git a/tests/v2/test_tasks.py b/tests/v2/test_tasks.py index 82ac763..a5491a3 100644 --- a/tests/v2/test_tasks.py +++ b/tests/v2/test_tasks.py @@ -14,7 +14,6 @@ # License for the specific language governing permissions and limitations # under the License. -import six import testtools from glanceclient.v2 import tasks @@ -260,10 +259,7 @@ class TestController(testtools.TestCase): # We just want to make sure filters are correctly encoded. pass - if six.PY2: - self.assertEqual("ni\xc3\xb1o", filters["owner"]) - else: - self.assertEqual("ni\xf1o", filters["owner"]) + self.assertEqual(b"ni\xc3\xb1o", filters["owner"]) def test_get_task(self): task = self.controller.get('3a4560a1-e585-443e-9b39-553b46ec92d1')