From 57200e0fa6ba86273a77db4e6d542067a59a87c7 Mon Sep 17 00:00:00 2001 From: Andrey Kurilin Date: Fri, 16 Mar 2018 17:15:43 +0200 Subject: [PATCH] get rid of oslo.utils we are using just several simple methods from oslo.utils which requires a lot of other dependecies. To make rally a lightweight framework, we need to abandon usage of such things Change-Id: I6e6170c7dabc97a089d030ee6b05fc07993e8c32 --- rally/cli/cliutils.py | 2 +- rally/cli/commands/task.py | 8 +- rally/cli/envutils.py | 2 +- ...ew_db_schema_for_verification_component.py | 8 +- ...2_bc908ac9a1fc_move_deployment_to_env_2.py | 8 +- rally/common/io/subunit_v2.py | 2 +- rally/common/utils.py | 24 +-- .../plugins/common/exporters/json_exporter.py | 4 +- .../common/sla/performance_degradation.py | 7 +- .../openstack/scenarios/sahara/utils.py | 4 +- rally/task/engine.py | 8 +- rally/utils/__init__.py | 0 rally/utils/encodeutils.py | 94 +++++++++ rally/utils/strutils.py | 105 ++++++++++ rally/verification/utils.py | 2 +- requirements.txt | 1 - tests/functional/utils.py | 4 +- tests/unit/common/test_utils.py | 39 ---- .../common/exporters/test_json_exporter.py | 14 +- .../openstack/scenarios/sahara/test_utils.py | 7 +- tests/unit/test_resources.py | 3 +- tests/unit/utils/__init__.py | 0 tests/unit/utils/test_encodeutils.py | 103 ++++++++++ tests/unit/utils/test_strutils.py | 188 ++++++++++++++++++ 24 files changed, 536 insertions(+), 101 deletions(-) create mode 100644 rally/utils/__init__.py create mode 100644 rally/utils/encodeutils.py create mode 100644 rally/utils/strutils.py create mode 100644 tests/unit/utils/__init__.py create mode 100644 tests/unit/utils/test_encodeutils.py create mode 100644 tests/unit/utils/test_strutils.py diff --git a/rally/cli/cliutils.py b/rally/cli/cliutils.py index 74b9c7b967..45170deeaf 100644 --- a/rally/cli/cliutils.py +++ b/rally/cli/cliutils.py @@ -25,7 +25,6 @@ import warnings import decorator import jsonschema -from oslo_utils import encodeutils import prettytable import six import sqlalchemy.exc @@ -35,6 +34,7 @@ from rally.common import cfg from rally.common import logging from rally.common.plugin import info from rally import exceptions +from rally.utils import encodeutils CONF = cfg.CONF diff --git a/rally/cli/commands/task.py b/rally/cli/commands/task.py index 31ce4ac259..331375045a 100644 --- a/rally/cli/commands/task.py +++ b/rally/cli/commands/task.py @@ -25,7 +25,6 @@ import sys import webbrowser import jsonschema -from oslo_utils import uuidutils import six from rally.cli import cliutils @@ -42,6 +41,7 @@ from rally.task import atomic from rally.task.processing import charts from rally.task.processing import plot from rally.task import utils as tutils +from rally.utils import strutils LOG = logging.getLogger(__name__) @@ -505,9 +505,9 @@ class TaskCommands(object): print() print("Load duration: %s" - % rutils.format_float_to_str(workload["load_duration"])) + % strutils.format_float_to_str(workload["load_duration"])) print("Full duration: %s" - % rutils.format_float_to_str(workload["full_duration"])) + % strutils.format_float_to_str(workload["full_duration"])) print("\nHINTS:") print("* To plot HTML graphics with this data, run:") @@ -811,7 +811,7 @@ class TaskCommands(object): for task_id in tasks: if os.path.exists(os.path.expanduser(task_id)): results.extend(self._load_task_results_file(api, task_id)) - elif uuidutils.is_uuid_like(task_id): + elif strutils.is_uuid_like(task_id): results.append(api.task.get(task_id=task_id, detailed=True)) else: print("ERROR: Invalid UUID or file name passed: %s" % task_id, diff --git a/rally/cli/envutils.py b/rally/cli/envutils.py index 6a4d1c4838..39bf5b5d05 100644 --- a/rally/cli/envutils.py +++ b/rally/cli/envutils.py @@ -16,10 +16,10 @@ import os import decorator -from oslo_utils import strutils from rally.common import fileutils from rally import exceptions +from rally.utils import strutils PATH_GLOBALS = "~/.rally/globals" ENV_ENV = "RALLY_ENV" diff --git a/rally/common/db/migrations/versions/2016_11_484cd9413e66_new_db_schema_for_verification_component.py b/rally/common/db/migrations/versions/2016_11_484cd9413e66_new_db_schema_for_verification_component.py index 5c777f834b..055f413178 100644 --- a/rally/common/db/migrations/versions/2016_11_484cd9413e66_new_db_schema_for_verification_component.py +++ b/rally/common/db/migrations/versions/2016_11_484cd9413e66_new_db_schema_for_verification_component.py @@ -19,8 +19,10 @@ Revises: e654a0648db0 Create Date: 2016-11-04 17:04:24.614075 """ + +import datetime as dt + from alembic import op -from oslo_utils import timeutils import sqlalchemy as sa from rally.common.db import models @@ -151,8 +153,8 @@ def upgrade(): "version": "n/a", "system_wide": False, "status": "init", - "created_at": timeutils.utcnow(), - "updated_at": timeutils.utcnow() + "created_at": dt.datetime.utcnow(), + "updated_at": dt.datetime.utcnow() }] ) default_verifier = connection.execute( diff --git a/rally/common/db/migrations/versions/2018_02_bc908ac9a1fc_move_deployment_to_env_2.py b/rally/common/db/migrations/versions/2018_02_bc908ac9a1fc_move_deployment_to_env_2.py index 456126190e..5f1a988adc 100644 --- a/rally/common/db/migrations/versions/2018_02_bc908ac9a1fc_move_deployment_to_env_2.py +++ b/rally/common/db/migrations/versions/2018_02_bc908ac9a1fc_move_deployment_to_env_2.py @@ -25,10 +25,10 @@ Create Date: 2018-02-22 21:37:21.258560 """ import copy +import datetime as dt import uuid from alembic import op -from oslo_utils import timeutils import sqlalchemy as sa from sqlalchemy.engine import reflection @@ -178,7 +178,7 @@ def upgrade(): "spec": spec, "extras": extras, "created_at": deployment.created_at, - "updated_at": timeutils.utcnow() + "updated_at": dt.datetime.utcnow() }] ) if platform_data: @@ -193,8 +193,8 @@ def upgrade(): "plugin_data": {}, "platform_name": "openstack", "platform_data": platform_data, - "created_at": timeutils.utcnow(), - "updated_at": timeutils.utcnow() + "created_at": dt.datetime.utcnow(), + "updated_at": dt.datetime.utcnow() }] ) diff --git a/rally/common/io/subunit_v2.py b/rally/common/io/subunit_v2.py index 9bed80b550..933aea14b1 100644 --- a/rally/common/io/subunit_v2.py +++ b/rally/common/io/subunit_v2.py @@ -14,10 +14,10 @@ # under the License. # -from oslo_utils import encodeutils from subunit import v2 from rally.common import logging +from rally.utils import encodeutils def prepare_input_args(func): diff --git a/rally/common/utils.py b/rally/common/utils.py index 8146e43855..c6ed952d82 100644 --- a/rally/common/utils.py +++ b/rally/common/utils.py @@ -34,6 +34,7 @@ from six import moves from rally.common import logging from rally import exceptions +from rally.utils import strutils LOG = logging.getLogger(__name__) @@ -685,26 +686,11 @@ class LockedDict(dict): return super(LockedDict, self).clear(*args, **kwargs) +@logging.log_deprecated(message="Its not used elsewhere in Rally already.", + rally_version="0.11.2") def format_float_to_str(num): - """Format number into human-readable float format. - - More precise it convert float into the string and remove redundant - zeros from the floating part. - It will format the number by the following examples: - 0.0000001 -> 0.0 - 0.000000 -> 0.0 - 37 -> 37.0 - 1.0000001 -> 1.0 - 1.0000011 -> 1.000001 - 1.0000019 -> 1.000002 - - :param num: Number to be formatted - :return: string representation of the number - """ - - num_str = "%f" % num - float_part = num_str.split(".")[1].rstrip("0") or "0" - return num_str.split(".")[0] + "." + float_part + """DEPRECATED. Use rally.utils.strutils.format_float_to_str instead.""" + return strutils.format_float_to_str(num) class DequeAsQueue(object): diff --git a/rally/plugins/common/exporters/json_exporter.py b/rally/plugins/common/exporters/json_exporter.py index 944d90ee98..aabcee3257 100644 --- a/rally/plugins/common/exporters/json_exporter.py +++ b/rally/plugins/common/exporters/json_exporter.py @@ -16,8 +16,6 @@ import collections import datetime as dt import json -from oslo_utils import timeutils - from rally.common import version as rally_version from rally.task import exporter @@ -108,7 +106,7 @@ class JSONExporter(exporter.TaskExporter): def generate(self): results = {"info": {"rally_version": rally_version.version_string(), "generated_at": dt.datetime.strftime( - timeutils.utcnow(), TIMEFORMAT), + dt.datetime.utcnow(), TIMEFORMAT), "format_version": self.REVISION}, "tasks": self._generate_tasks()} diff --git a/rally/plugins/common/sla/performance_degradation.py b/rally/plugins/common/sla/performance_degradation.py index 76412d5d17..0d89e81d93 100644 --- a/rally/plugins/common/sla/performance_degradation.py +++ b/rally/plugins/common/sla/performance_degradation.py @@ -22,9 +22,9 @@ with contracted values such as maximum error rate or minimum response time. from __future__ import division from rally.common import streaming_algorithms -from rally.common import utils from rally import consts from rally.task import sla +from rally.utils import strutils @sla.configure(name="performance_degradation") @@ -68,6 +68,5 @@ class PerformanceDegradation(sla.SLA): return self.success def details(self): - return ("Current degradation: %s%% - %s" % - (utils.format_float_to_str(self.degradation.result() or 0.0), - self.status())) + res = strutils.format_float_to_str(self.degradation.result() or 0.0) + return "Current degradation: %s%% - %s" % (res, self.status()) diff --git a/rally/plugins/openstack/scenarios/sahara/utils.py b/rally/plugins/openstack/scenarios/sahara/utils.py index 4ca8e408ca..3f150f8d7c 100644 --- a/rally/plugins/openstack/scenarios/sahara/utils.py +++ b/rally/plugins/openstack/scenarios/sahara/utils.py @@ -15,7 +15,6 @@ import random -from oslo_utils import uuidutils from saharaclient.api import base as sahara_base from rally.common import cfg @@ -27,6 +26,7 @@ from rally.plugins.openstack import scenario from rally.plugins.openstack.scenarios.sahara import consts as sahara_consts from rally.task import atomic from rally.task import utils +from rally.utils import strutils LOG = logging.getLogger(__name__) @@ -115,7 +115,7 @@ class SaharaScenario(scenario.OpenStackScenario): def _setup_neutron_floating_ip_pool(self, name_or_id): if name_or_id: - if uuidutils.is_uuid_like(name_or_id): + if strutils.is_uuid_like(name_or_id): # Looks like an id is provided Return as is. return name_or_id else: diff --git a/rally/task/engine.py b/rally/task/engine.py index 222cd85a16..8be2e4f36c 100644 --- a/rally/task/engine.py +++ b/rally/task/engine.py @@ -24,7 +24,6 @@ import jsonschema from rally.common import cfg from rally.common import logging from rally.common import objects -from rally.common import utils from rally import consts from rally import exceptions from rally.task import context @@ -32,6 +31,7 @@ from rally.task import hook from rally.task import runner from rally.task import scenario from rally.task import sla +from rally.utils import strutils LOG = logging.getLogger(__name__) @@ -155,11 +155,11 @@ class ResultConsumer(object): load_duration = max(self.load_finished_at - self.load_started_at, 0) - LOG.info("Load duration is: %s" % utils.format_float_to_str( + LOG.info("Load duration is: %s" % strutils.format_float_to_str( load_duration)) LOG.info("Full runner duration is: %s" % - utils.format_float_to_str(self.runner.run_duration)) - LOG.info("Full duration is: %s" % utils.format_float_to_str( + strutils.format_float_to_str(self.runner.run_duration)) + LOG.info("Full duration is: %s" % strutils.format_float_to_str( self.finish - self.start)) results = {} diff --git a/rally/utils/__init__.py b/rally/utils/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/rally/utils/encodeutils.py b/rally/utils/encodeutils.py new file mode 100644 index 0000000000..1f1ae819bc --- /dev/null +++ b/rally/utils/encodeutils.py @@ -0,0 +1,94 @@ +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import sys + +import six + + +def safe_decode(text, incoming=None, errors="strict"): + """Decodes incoming string using `incoming` if they're not already unicode. + + :param text: text/bytes string to decode + :param incoming: Text's current encoding + :param errors: Errors handling policy. See here for valid + values http://docs.python.org/2/library/codecs.html + :returns: text or a unicode `incoming` encoded representation of it. + :raises TypeError: If text is not an instance of str + """ + if not isinstance(text, (six.string_types, six.binary_type)): + raise TypeError("%s can't be decoded" % type(text)) + + if isinstance(text, six.text_type): + return text + + if not incoming: + incoming = (sys.stdin.encoding or + sys.getdefaultencoding()) + + try: + return text.decode(incoming, errors) + except UnicodeDecodeError: + # Note(flaper87) If we get here, it means that + # sys.stdin.encoding / sys.getdefaultencoding + # didn't return a suitable encoding to decode + # text. This happens mostly when global LANG + # var is not set correctly and there's no + # default encoding. In this case, most likely + # python will use ASCII or ANSI encoders as + # default encodings but they won't be capable + # of decoding non-ASCII characters. + # + # Also, UTF-8 is being used since it's an ASCII + # extension. + return text.decode("utf-8", errors) + + +def safe_encode(text, incoming=None, encoding="utf-8", errors="strict"): + """Encodes incoming text/bytes string using `encoding`. + + If incoming is not specified, text is expected to be encoded with + current python's default encoding. (`sys.getdefaultencoding`) + + :param text: Incoming text/bytes string + :param incoming: Text's current encoding + :param encoding: Expected encoding for text (Default UTF-8) + :param errors: Errors handling policy. See here for valid + values http://docs.python.org/2/library/codecs.html + :returns: text or a bytestring `encoding` encoded representation of it. + :raises TypeError: If text is not an instance of str + See also to_utf8() function which is simpler and don't depend on + the locale encoding. + """ + if not isinstance(text, (six.string_types, six.binary_type)): + raise TypeError("%s can't be encoded" % type(text)) + + if not incoming: + incoming = (sys.stdin.encoding or + sys.getdefaultencoding()) + + # Avoid case issues in comparisons + if hasattr(incoming, "lower"): + incoming = incoming.lower() + if hasattr(encoding, "lower"): + encoding = encoding.lower() + + if isinstance(text, six.text_type): + return text.encode(encoding, errors) + elif text and encoding != incoming: + # Decode text before encoding it with `encoding` + text = safe_decode(text, incoming, errors) + return text.encode(encoding, errors) + else: + return text diff --git a/rally/utils/strutils.py b/rally/utils/strutils.py new file mode 100644 index 0000000000..baf2db15cc --- /dev/null +++ b/rally/utils/strutils.py @@ -0,0 +1,105 @@ +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import uuid + +import six + + +def _format_uuid_string(string): + return (string.replace("urn:", "") + .replace("uuid:", "") + .strip("{}") + .replace("-", "") + .lower()) + + +def is_uuid_like(val): + """Returns validation of a value as a UUID. + + :param val: Value to verify + :type val: string + :returns: bool + + .. versionchanged:: 1.1.1 + Support non-lowercase UUIDs. + """ + try: + return str(uuid.UUID(val)).replace("-", "") == _format_uuid_string(val) + except (TypeError, ValueError, AttributeError): + return False + + +TRUE_STRINGS = ("1", "t", "true", "on", "y", "yes") +FALSE_STRINGS = ("0", "f", "false", "off", "n", "no") + + +def bool_from_string(subject, strict=False, default=False): + """Interpret a subject as a boolean. + + A subject can be a boolean, a string or an integer. Boolean type value + will be returned directly, otherwise the subject will be converted to + a string. A case-insensitive match is performed such that strings + matching 't','true', 'on', 'y', 'yes', or '1' are considered True and, + when `strict=False`, anything else returns the value specified by + 'default'. + + Useful for JSON-decoded stuff and config file parsing. + + If `strict=True`, unrecognized values, including None, will raise a + ValueError which is useful when parsing values passed in from an API call. + Strings yielding False are 'f', 'false', 'off', 'n', 'no', or '0'. + """ + if isinstance(subject, bool): + return subject + if not isinstance(subject, six.string_types): + subject = six.text_type(subject) + + lowered = subject.strip().lower() + + if lowered in TRUE_STRINGS: + return True + elif lowered in FALSE_STRINGS: + return False + elif strict: + acceptable = ", ".join( + "'%s'" % s for s in sorted(TRUE_STRINGS + FALSE_STRINGS)) + msg = ("Unrecognized value '%(val)s', acceptable values are:" + " %(acceptable)s") % {"val": subject, + "acceptable": acceptable} + raise ValueError(msg) + else: + return default + + +def format_float_to_str(num): + """Format number into human-readable float format. + + More precise it convert float into the string and remove redundant + zeros from the floating part. + It will format the number by the following examples: + 0.0000001 -> 0.0 + 0.000000 -> 0.0 + 37 -> 37.0 + 1.0000001 -> 1.0 + 1.0000011 -> 1.000001 + 1.0000019 -> 1.000002 + + :param num: Number to be formatted + :return: string representation of the number + """ + + num_str = "%f" % num + float_part = num_str.split(".")[1].rstrip("0") or "0" + return num_str.split(".")[0] + "." + float_part diff --git a/rally/verification/utils.py b/rally/verification/utils.py index d5add24de4..cdf823afdb 100644 --- a/rally/verification/utils.py +++ b/rally/verification/utils.py @@ -15,11 +15,11 @@ import os import subprocess -from oslo_utils import encodeutils import six from six.moves import configparser from rally.common import logging +from rally.utils import encodeutils LOG = logging.getLogger(__name__) diff --git a/requirements.txt b/requirements.txt index c7488fd07d..4c7fd7a52b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,6 @@ netaddr>=0.7.18 # BSD oslo.config>=5.1.0 # Apache Software License oslo.db>=4.27.0 # Apache Software License oslo.log>=3.36.0 # Apache Software License -oslo.utils>=3.33.0 # Apache Software License paramiko>=2.0.0 # LGPL pbr>=2.0.0,!=2.1.0 # Apache Software License PrettyTable>=0.7.1,<0.8 # BSD diff --git a/tests/functional/utils.py b/tests/functional/utils.py index fc15d7e3fa..07174ba221 100644 --- a/tests/functional/utils.py +++ b/tests/functional/utils.py @@ -22,10 +22,10 @@ import shutil import subprocess import tempfile - -from oslo_utils import encodeutils from six.moves import configparser +from rally.utils import encodeutils + class RallyCliError(Exception): diff --git a/tests/unit/common/test_utils.py b/tests/unit/common/test_utils.py index e21fa50f83..edcf0b92f1 100644 --- a/tests/unit/common/test_utils.py +++ b/tests/unit/common/test_utils.py @@ -559,45 +559,6 @@ class LockedDictTestCase(test.TestCase): self.assertEqual({"memo": "foo_memo"}, kw) -@ddt.ddt -class FloatFormatterTestCase(test.TestCase): - - @ddt.data( - { - "num_float": 0, - "num_str": "0.0" - }, - { - "num_float": 37, - "num_str": "37.0" - }, - { - "num_float": 0.0000001, - "num_str": "0.0" - }, - { - "num_float": 0.000000, - "num_str": "0.0" - }, - { - "num_float": 1.0000001, - "num_str": "1.0" - }, - { - "num_float": 1.0000011, - "num_str": "1.000001" - }, - { - "num_float": 1.0000019, - "num_str": "1.000002" - } - - ) - @ddt.unpack - def test_format_float_to_str(self, num_float, num_str): - self.assertEqual(num_str, utils.format_float_to_str(num_float)) - - class DequeAsQueueTestCase(test.TestCase): def setUp(self): diff --git a/tests/unit/plugins/common/exporters/test_json_exporter.py b/tests/unit/plugins/common/exporters/test_json_exporter.py index 25659c8dad..068210fb11 100644 --- a/tests/unit/plugins/common/exporters/test_json_exporter.py +++ b/tests/unit/plugins/common/exporters/test_json_exporter.py @@ -16,7 +16,6 @@ import collections import datetime as dt import mock -from oslo_utils import timeutils from rally.common import version as rally_version from rally.plugins.common.exporters import json_exporter @@ -83,9 +82,9 @@ class JSONExporterTestCase(test.TestCase): ])], reporter._generate_tasks()) @mock.patch("%s.json.dumps" % PATH, return_value="json") - @mock.patch("%s.timeutils.utcnow" % PATH, - return_value=timeutils.utcnow()) - def test_generate(self, mock_utcnow, mock_json_dumps): + @mock.patch("%s.dt" % PATH) + def test_generate(self, mock_dt, mock_json_dumps): + mock_dt.datetime.utcnow.return_value = dt.datetime.utcnow() tasks_results = test_html.get_tasks_results() # print @@ -94,12 +93,13 @@ class JSONExporterTestCase(test.TestCase): self.assertEqual({"print": "json"}, reporter.generate()) results = { "info": {"rally_version": rally_version.version_string(), - "generated_at": dt.datetime.strftime( - mock_utcnow.return_value, - json_exporter.TIMEFORMAT), + "generated_at": mock_dt.datetime.strftime.return_value, "format_version": "1.1"}, "tasks": reporter._generate_tasks.return_value } + mock_dt.datetime.strftime.assert_called_once_with( + mock_dt.datetime.utcnow.return_value, + json_exporter.TIMEFORMAT) reporter._generate_tasks.assert_called_once_with() mock_json_dumps.assert_called_once_with(results, sort_keys=False, diff --git a/tests/unit/plugins/openstack/scenarios/sahara/test_utils.py b/tests/unit/plugins/openstack/scenarios/sahara/test_utils.py index 65b850706e..8476f393e3 100644 --- a/tests/unit/plugins/openstack/scenarios/sahara/test_utils.py +++ b/tests/unit/plugins/openstack/scenarios/sahara/test_utils.py @@ -13,8 +13,9 @@ # License for the specific language governing permissions and limitations # under the License. +import uuid + import mock -from oslo_utils import uuidutils from saharaclient.api import base as sahara_base from rally.common import cfg @@ -166,7 +167,7 @@ class SaharaScenarioTestCase(test.ScenarioTestCase): } } - floating_ip_pool_uuid = uuidutils.generate_uuid() + floating_ip_pool_uuid = str(uuid.uuid4()) node_groups = [ { "name": "master-ng", @@ -274,7 +275,7 @@ class SaharaScenarioTestCase(test.ScenarioTestCase): } } - floating_ip_pool_uuid = uuidutils.generate_uuid() + floating_ip_pool_uuid = str(uuid.uuid4()) node_groups = [ { "name": "master-ng", diff --git a/tests/unit/test_resources.py b/tests/unit/test_resources.py index 2ea7abe534..406c2c5ecb 100644 --- a/tests/unit/test_resources.py +++ b/tests/unit/test_resources.py @@ -16,10 +16,9 @@ import difflib import os -from oslo_utils import encodeutils - import rally from rally.cli import cliutils +from rally.utils import encodeutils from tests.unit import test RES_PATH = os.path.join(os.path.dirname(rally.__file__), os.pardir, "etc") diff --git a/tests/unit/utils/__init__.py b/tests/unit/utils/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/utils/test_encodeutils.py b/tests/unit/utils/test_encodeutils.py new file mode 100644 index 0000000000..672111903f --- /dev/null +++ b/tests/unit/utils/test_encodeutils.py @@ -0,0 +1,103 @@ +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import mock +import six + +from rally.utils import encodeutils +from tests.unit import test + + +class EncodeUtilsTestCase(test.TestCase): + + def test_safe_decode(self): + safe_decode = encodeutils.safe_decode + self.assertRaises(TypeError, safe_decode, True) + self.assertEqual(six.u("ni\xf1o"), safe_decode(six.b("ni\xc3\xb1o"), + incoming="utf-8")) + if six.PY2: + # In Python 3, bytes.decode() doesn"t support anymore + # bytes => bytes encodings like base64 + self.assertEqual(six.u("test"), safe_decode("dGVzdA==", + incoming="base64")) + + self.assertEqual(six.u("strange"), safe_decode(six.b("\x80strange"), + errors="ignore")) + + self.assertEqual(six.u("\xc0"), safe_decode(six.b("\xc0"), + incoming="iso-8859-1")) + + # Forcing incoming to ascii so it falls back to utf-8 + self.assertEqual(six.u("ni\xf1o"), safe_decode(six.b("ni\xc3\xb1o"), + incoming="ascii")) + + self.assertEqual(six.u("foo"), safe_decode(b"foo")) + + def test_safe_encode_none_instead_of_text(self): + self.assertRaises(TypeError, encodeutils.safe_encode, None) + + def test_safe_encode_bool_instead_of_text(self): + self.assertRaises(TypeError, encodeutils.safe_encode, True) + + def test_safe_encode_int_instead_of_text(self): + self.assertRaises(TypeError, encodeutils.safe_encode, 1) + + def test_safe_encode_list_instead_of_text(self): + self.assertRaises(TypeError, encodeutils.safe_encode, []) + + def test_safe_encode_dict_instead_of_text(self): + self.assertRaises(TypeError, encodeutils.safe_encode, {}) + + def test_safe_encode_tuple_instead_of_text(self): + self.assertRaises(TypeError, encodeutils.safe_encode, ("foo", "bar",)) + + def test_safe_encode_py2(self): + if six.PY2: + # In Python 3, str.encode() doesn"t support anymore + # text => text encodings like base64 + self.assertEqual( + six.b("dGVzdA==\n"), + encodeutils.safe_encode("test", encoding="base64"), + ) + else: + self.skipTest("Requires py2.x") + + def test_safe_encode_force_incoming_utf8_to_ascii(self): + # Forcing incoming to ascii so it falls back to utf-8 + self.assertEqual( + six.b("ni\xc3\xb1o"), + encodeutils.safe_encode(six.b("ni\xc3\xb1o"), incoming="ascii"), + ) + + def test_safe_encode_same_encoding_different_cases(self): + with mock.patch.object(encodeutils, "safe_decode", mock.Mock()): + utf8 = encodeutils.safe_encode( + six.u("foo\xf1bar"), encoding="utf-8") + self.assertEqual( + encodeutils.safe_encode(utf8, "UTF-8", "utf-8"), + encodeutils.safe_encode(utf8, "utf-8", "UTF-8"), + ) + self.assertEqual( + encodeutils.safe_encode(utf8, "UTF-8", "utf-8"), + encodeutils.safe_encode(utf8, "utf-8", "utf-8"), + ) + encodeutils.safe_decode.assert_has_calls([]) + + def test_safe_encode_different_encodings(self): + text = six.u("foo\xc3\xb1bar") + result = encodeutils.safe_encode( + text=text, incoming="utf-8", encoding="iso-8859-1") + self.assertNotEqual(text, result) + + self.assertNotEqual(six.b("foo\xf1bar"), result) diff --git a/tests/unit/utils/test_strutils.py b/tests/unit/utils/test_strutils.py new file mode 100644 index 0000000000..221dc0340f --- /dev/null +++ b/tests/unit/utils/test_strutils.py @@ -0,0 +1,188 @@ +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import uuid + +import ddt +import mock + +from rally.utils import strutils +from tests.unit import test + + +@ddt.ddt +class StrUtilsTestCase(test.TestCase): + + def test_is_uuid_like(self): + self.assertTrue(strutils.is_uuid_like(str(uuid.uuid4()))) + self.assertTrue(strutils.is_uuid_like( + "{12345678-1234-5678-1234-567812345678}")) + self.assertTrue(strutils.is_uuid_like( + "12345678123456781234567812345678")) + self.assertTrue(strutils.is_uuid_like( + "urn:uuid:12345678-1234-5678-1234-567812345678")) + self.assertTrue(strutils.is_uuid_like( + "urn:bbbaaaaa-aaaa-aaaa-aabb-bbbbbbbbbbbb")) + self.assertTrue(strutils.is_uuid_like( + "uuid:bbbaaaaa-aaaa-aaaa-aabb-bbbbbbbbbbbb")) + self.assertTrue(strutils.is_uuid_like( + "{}---bbb---aaa--aaa--aaa-----aaa---aaa--bbb-bbb---bbb-bbb-bb-{}")) + + def test_is_uuid_like_insensitive(self): + self.assertTrue(strutils.is_uuid_like(str(uuid.uuid4()).upper())) + + def test_id_is_uuid_like(self): + self.assertFalse(strutils.is_uuid_like(1234567)) + + def test_name_is_uuid_like(self): + self.assertFalse(strutils.is_uuid_like("asdasdasd")) + + @mock.patch("six.text_type") + def test_bool_bool_from_string_no_text(self, mock_text_type): + self.assertTrue(strutils.bool_from_string(True)) + self.assertFalse(strutils.bool_from_string(False)) + self.assertEqual(0, mock_text_type.call_count) + + def test_bool_bool_from_string(self): + self.assertTrue(strutils.bool_from_string(True)) + self.assertFalse(strutils.bool_from_string(False)) + + def test_bool_bool_from_string_default(self): + self.assertTrue(strutils.bool_from_string("", default=True)) + self.assertFalse(strutils.bool_from_string("wibble", default=False)) + + def _test_bool_from_string(self, c): + self.assertTrue(strutils.bool_from_string(c("true"))) + self.assertTrue(strutils.bool_from_string(c("TRUE"))) + self.assertTrue(strutils.bool_from_string(c("on"))) + self.assertTrue(strutils.bool_from_string(c("On"))) + self.assertTrue(strutils.bool_from_string(c("yes"))) + self.assertTrue(strutils.bool_from_string(c("YES"))) + self.assertTrue(strutils.bool_from_string(c("yEs"))) + self.assertTrue(strutils.bool_from_string(c("1"))) + self.assertTrue(strutils.bool_from_string(c("T"))) + self.assertTrue(strutils.bool_from_string(c("t"))) + self.assertTrue(strutils.bool_from_string(c("Y"))) + self.assertTrue(strutils.bool_from_string(c("y"))) + + self.assertFalse(strutils.bool_from_string(c("false"))) + self.assertFalse(strutils.bool_from_string(c("FALSE"))) + self.assertFalse(strutils.bool_from_string(c("off"))) + self.assertFalse(strutils.bool_from_string(c("OFF"))) + self.assertFalse(strutils.bool_from_string(c("no"))) + self.assertFalse(strutils.bool_from_string(c("0"))) + self.assertFalse(strutils.bool_from_string(c("42"))) + self.assertFalse(strutils.bool_from_string(c( + "This should not be True"))) + self.assertFalse(strutils.bool_from_string(c("F"))) + self.assertFalse(strutils.bool_from_string(c("f"))) + self.assertFalse(strutils.bool_from_string(c("N"))) + self.assertFalse(strutils.bool_from_string(c("n"))) + + # Whitespace should be stripped + self.assertTrue(strutils.bool_from_string(c(" 1 "))) + self.assertTrue(strutils.bool_from_string(c(" true "))) + self.assertFalse(strutils.bool_from_string(c(" 0 "))) + self.assertFalse(strutils.bool_from_string(c(" false "))) + + def test_bool_from_string(self): + self._test_bool_from_string(lambda s: s) + + def test_other_bool_from_string(self): + self.assertFalse(strutils.bool_from_string(None)) + self.assertFalse(strutils.bool_from_string(mock.Mock())) + + def test_int_bool_from_string(self): + self.assertTrue(strutils.bool_from_string(1)) + + self.assertFalse(strutils.bool_from_string(-1)) + self.assertFalse(strutils.bool_from_string(0)) + self.assertFalse(strutils.bool_from_string(2)) + + def test_strict_bool_from_string(self): + # None isn"t allowed in strict mode + exc = self.assertRaises(ValueError, strutils.bool_from_string, None, + strict=True) + expected_msg = ("Unrecognized value 'None', acceptable values are:" + " '0', '1', 'f', 'false', 'n', 'no', 'off', 'on'," + " 't', 'true', 'y', 'yes'") + self.assertEqual(expected_msg, str(exc)) + + # Unrecognized strings aren't allowed + self.assertFalse(strutils.bool_from_string("Other", strict=False)) + exc = self.assertRaises(ValueError, strutils.bool_from_string, "Other", + strict=True) + expected_msg = ("Unrecognized value 'Other', acceptable values are:" + " '0', '1', 'f', 'false', 'n', 'no', 'off', 'on'," + " 't', 'true', 'y', 'yes'") + self.assertEqual(expected_msg, str(exc)) + + # Unrecognized numbers aren't allowed + exc = self.assertRaises(ValueError, strutils.bool_from_string, 2, + strict=True) + expected_msg = ("Unrecognized value '2', acceptable values are:" + " '0', '1', 'f', 'false', 'n', 'no', 'off', 'on'," + " 't', 'true', 'y', 'yes'") + self.assertEqual(expected_msg, str(exc)) + + # False-like values are allowed + self.assertFalse(strutils.bool_from_string("f", strict=True)) + self.assertFalse(strutils.bool_from_string("false", strict=True)) + self.assertFalse(strutils.bool_from_string("off", strict=True)) + self.assertFalse(strutils.bool_from_string("n", strict=True)) + self.assertFalse(strutils.bool_from_string("no", strict=True)) + self.assertFalse(strutils.bool_from_string("0", strict=True)) + + self.assertTrue(strutils.bool_from_string("1", strict=True)) + + # Avoid font-similarity issues (one looks like lowercase-el, zero like + # oh, etc...) + for char in ("O", "o", "L", "l", "I", "i"): + self.assertRaises(ValueError, strutils.bool_from_string, char, + strict=True) + + @ddt.data( + { + "num_float": 0, + "num_str": "0.0" + }, + { + "num_float": 37, + "num_str": "37.0" + }, + { + "num_float": 0.0000001, + "num_str": "0.0" + }, + { + "num_float": 0.000000, + "num_str": "0.0" + }, + { + "num_float": 1.0000001, + "num_str": "1.0" + }, + { + "num_float": 1.0000011, + "num_str": "1.000001" + }, + { + "num_float": 1.0000019, + "num_str": "1.000002" + } + + ) + @ddt.unpack + def test_format_float_to_str(self, num_float, num_str): + self.assertEqual(num_str, strutils.format_float_to_str(num_float))