From 3a009031be349fb20738ba295a63c2fc2cefa4fd Mon Sep 17 00:00:00 2001 From: Chris Dent Date: Mon, 28 Jul 2014 12:34:35 +0100 Subject: [PATCH] Fix dict and set order related issues in tests There are several tests where the expected output relies on order of strings generated from the arbitrary ordering of dictionaries or sets. The new version of tox causes that ordering to be different between tests runs so tests must be resilient and test either the content of the dicts or sets or their sorted output. Note: These fixes are only of those cases where a failure was generated. If there are issues in skipped tests, this patch has not found nor fixed them. The vast majority of skipped tests are related to availability of functionality so there's little that can be done about that with the current testing setup. The best we can do is be on the lookout for failures that could be related PYTHONHASHSEED down the road and fix them as they happen. Closes-Bug: 1348818 Change-Id: I6b27ca2597c51b0656f441f325f9ffd0e31a606d --- ceilometer/api/controllers/v2.py | 2 +- ceilometer/tests/alarm/test_notifier.py | 56 +++++++++++++------ .../tests/api/v2/test_alarm_scenarios.py | 20 +++---- ceilometer/tests/test_utils.py | 13 +++-- tox.ini | 3 - 5 files changed, 58 insertions(+), 36 deletions(-) diff --git a/ceilometer/api/controllers/v2.py b/ceilometer/api/controllers/v2.py index de56a527f8..be5900fef8 100644 --- a/ceilometer/api/controllers/v2.py +++ b/ceilometer/api/controllers/v2.py @@ -411,7 +411,7 @@ def _validate_query(query, db_func, internal_keys=None, '%s' % i.field) else: msg = ("unrecognized field in query: %s, " - "valid keys: %s") % (query, valid_keys) + "valid keys: %s") % (query, sorted(valid_keys)) raise wsme.exc.UnknownArgument(key, msg) diff --git a/ceilometer/tests/alarm/test_notifier.py b/ceilometer/tests/alarm/test_notifier.py index a91366f75a..9ef1d5480c 100644 --- a/ceilometer/tests/alarm/test_notifier.py +++ b/ceilometer/tests/alarm/test_notifier.py @@ -15,6 +15,7 @@ # License for the specific language governing permissions and limitations # under the License. +import anyjson import mock import requests import six.moves.urllib.parse as urlparse @@ -26,9 +27,11 @@ from ceilometer.openstack.common.fixture import mockpatch from ceilometer.tests import base as tests_base -DATA_JSON = ('{"current": "ALARM", "alarm_id": "foobar",' - ' "reason": "what ?", "reason_data": {"test": "test"},' - ' "previous": "OK"}') +DATA_JSON = anyjson.loads( + '{"current": "ALARM", "alarm_id": "foobar",' + ' "reason": "what ?", "reason_data": {"test": "test"},' + ' "previous": "OK"}' +) NOTIFICATION = dict(alarm_id='foobar', condition=dict(threshold=42), reason='what ?', @@ -110,8 +113,11 @@ class TestAlarmNotifier(tests_base.BaseTestCase): with mock.patch.object(requests.Session, 'post') as poster: self.service.notify_alarm(context.get_admin_context(), self._notification(action)) - poster.assert_called_with(action, data=DATA_JSON, - headers=self.HTTP_HEADERS) + poster.assert_called_with(action, data=mock.ANY, + headers=mock.ANY) + args, kwargs = poster.call_args + self.assertEqual(self.HTTP_HEADERS, kwargs['headers']) + self.assertEqual(DATA_JSON, anyjson.loads(kwargs['data'])) def test_notify_alarm_rest_action_with_ssl_client_cert(self): action = 'https://host/action' @@ -124,9 +130,12 @@ class TestAlarmNotifier(tests_base.BaseTestCase): with mock.patch.object(requests.Session, 'post') as poster: self.service.notify_alarm(context.get_admin_context(), self._notification(action)) - poster.assert_called_with(action, data=DATA_JSON, - headers=self.HTTP_HEADERS, + poster.assert_called_with(action, data=mock.ANY, + headers=mock.ANY, cert=certificate, verify=True) + args, kwargs = poster.call_args + self.assertEqual(self.HTTP_HEADERS, kwargs['headers']) + self.assertEqual(DATA_JSON, anyjson.loads(kwargs['data'])) def test_notify_alarm_rest_action_with_ssl_client_cert_and_key(self): action = 'https://host/action' @@ -142,9 +151,12 @@ class TestAlarmNotifier(tests_base.BaseTestCase): with mock.patch.object(requests.Session, 'post') as poster: self.service.notify_alarm(context.get_admin_context(), self._notification(action)) - poster.assert_called_with(action, data=DATA_JSON, - headers=self.HTTP_HEADERS, + poster.assert_called_with(action, data=mock.ANY, + headers=mock.ANY, cert=(certificate, key), verify=True) + args, kwargs = poster.call_args + self.assertEqual(self.HTTP_HEADERS, kwargs['headers']) + self.assertEqual(DATA_JSON, anyjson.loads(kwargs['data'])) def test_notify_alarm_rest_action_with_ssl_verify_disable_by_cfg(self): action = 'https://host/action' @@ -156,9 +168,12 @@ class TestAlarmNotifier(tests_base.BaseTestCase): with mock.patch.object(requests.Session, 'post') as poster: self.service.notify_alarm(context.get_admin_context(), self._notification(action)) - poster.assert_called_with(action, data=DATA_JSON, - headers=self.HTTP_HEADERS, + poster.assert_called_with(action, data=mock.ANY, + headers=mock.ANY, verify=False) + args, kwargs = poster.call_args + self.assertEqual(self.HTTP_HEADERS, kwargs['headers']) + self.assertEqual(DATA_JSON, anyjson.loads(kwargs['data'])) def test_notify_alarm_rest_action_with_ssl_verify_disable(self): action = 'https://host/action?ceilometer-alarm-ssl-verify=0' @@ -167,9 +182,12 @@ class TestAlarmNotifier(tests_base.BaseTestCase): with mock.patch.object(requests.Session, 'post') as poster: self.service.notify_alarm(context.get_admin_context(), self._notification(action)) - poster.assert_called_with(action, data=DATA_JSON, - headers=self.HTTP_HEADERS, + poster.assert_called_with(action, data=mock.ANY, + headers=mock.ANY, verify=False) + args, kwargs = poster.call_args + self.assertEqual(self.HTTP_HEADERS, kwargs['headers']) + self.assertEqual(DATA_JSON, anyjson.loads(kwargs['data'])) def test_notify_alarm_rest_action_with_ssl_verify_enable_by_user(self): action = 'https://host/action?ceilometer-alarm-ssl-verify=1' @@ -181,9 +199,12 @@ class TestAlarmNotifier(tests_base.BaseTestCase): with mock.patch.object(requests.Session, 'post') as poster: self.service.notify_alarm(context.get_admin_context(), self._notification(action)) - poster.assert_called_with(action, data=DATA_JSON, - headers=self.HTTP_HEADERS, + poster.assert_called_with(action, data=mock.ANY, + headers=mock.ANY, verify=True) + args, kwargs = poster.call_args + self.assertEqual(self.HTTP_HEADERS, kwargs['headers']) + self.assertEqual(DATA_JSON, anyjson.loads(kwargs['data'])) @staticmethod def _fake_urlsplit(*args, **kwargs): @@ -234,4 +255,7 @@ class TestAlarmNotifier(tests_base.BaseTestCase): headers = {'X-Auth-Token': 'token_1234'} headers.update(self.HTTP_HEADERS) poster.assert_called_with( - url, data=DATA_JSON, headers=headers) + url, data=mock.ANY, headers=mock.ANY) + args, kwargs = poster.call_args + self.assertEqual(headers, kwargs['headers']) + self.assertEqual(DATA_JSON, anyjson.loads(kwargs['data'])) diff --git a/ceilometer/tests/api/v2/test_alarm_scenarios.py b/ceilometer/tests/api/v2/test_alarm_scenarios.py index f07bcaf574..1deb9e8aa0 100644 --- a/ceilometer/tests/api/v2/test_alarm_scenarios.py +++ b/ceilometer/tests/api/v2/test_alarm_scenarios.py @@ -1909,11 +1909,11 @@ class TestAlarms(v2.FunctionalTest, resp = self._get_alarm_history(alarm, query=query, expect_errors=True, status=400) self.assertEqual('Unknown argument: "alarm_id": unrecognized' - ' field in query: [], valid keys: set(' - '[\'start_timestamp\', \'end_timestamp_op\',' - ' \'project\', \'user\', \'start_timestamp_op\'' - ', \'type\', \'end_timestamp\'])', + " field in query: [], valid keys: ['end_timestamp'," + " 'end_timestamp_op', 'project'," + " 'start_timestamp', 'start_timestamp_op'," + " 'type', 'user']", resp.json['error_message']['faultstring']) def test_get_alarm_history_constrained_by_not_supported_rule(self): @@ -1922,11 +1922,11 @@ class TestAlarms(v2.FunctionalTest, resp = self._get_alarm_history(alarm, query=query, expect_errors=True, status=400) self.assertEqual('Unknown argument: "abcd": unrecognized' - ' field in query: [], valid keys: set(' - '[\'start_timestamp\', \'end_timestamp_op\',' - ' \'project\', \'user\', \'start_timestamp_op\'' - ', \'type\', \'end_timestamp\'])', + " field in query: [], valid keys: ['end_timestamp'," + " 'end_timestamp_op', 'project'," + " 'start_timestamp', 'start_timestamp_op'," + " 'type', 'user']", resp.json['error_message']['faultstring']) def test_get_nonexistent_alarm_history(self): diff --git a/ceilometer/tests/test_utils.py b/ceilometer/tests/test_utils.py index beaa24f5b7..7ecd1052d0 100644 --- a/ceilometer/tests/test_utils.py +++ b/ceilometer/tests/test_utils.py @@ -139,9 +139,10 @@ class TestUtils(test.BaseTestCase): 'nested2': [{'c': 'A'}, {'c': 'B'}] } pairs = list(utils.dict_to_keyval(data)) - self.assertEqual(set([('a', 'A'), ('b', 'B'), - ('nested2[0].c', 'A'), - ('nested2[1].c', 'B'), - ('nested.a', 'A'), - ('nested.b', 'B')]), - set(pairs)) + self.assertEqual([('a', 'A'), + ('b', 'B'), + ('nested.a', 'A'), + ('nested.b', 'B'), + ('nested2[0].c', 'A'), + ('nested2[1].c', 'B')], + sorted(pairs, key=lambda x: x[0])) diff --git a/tox.ini b/tox.ini index 3f585a8b6a..21fc85c1c7 100644 --- a/tox.ini +++ b/tox.ini @@ -8,11 +8,8 @@ deps = -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt install_command = pip install -U --allow-external pytidylib --allow-insecure pytidylib --allow-external netifaces --allow-insecure netifaces {opts} {packages} usedevelop = True -# Note the hash seed is set to 0 until ceilometer can be tested with a -# random hash seed successfully. setenv = VIRTUAL_ENV={envdir} EVENTLET_NO_GREENDNS=yes - PYTHONHASHSEED=0 commands = bash -x {toxinidir}/setup-test-env.sh python setup.py testr --slowest --testr-args="{posargs}" downloadcache = {toxworkdir}/_download